├── data └── .gitkeep ├── analyses └── .gitkeep ├── macros └── .gitkeep ├── snapshots └── .gitkeep ├── tests └── .gitkeep ├── dbt_packages ├── dbt_expectations │ ├── data │ │ └── .gitkeep │ ├── macros │ │ ├── .gitkeep │ │ ├── math │ │ │ ├── median.sql │ │ │ ├── log_natural.sql │ │ │ ├── rand.sql │ │ │ └── percentile_cont.sql │ │ ├── schema_tests │ │ │ ├── string_matching │ │ │ │ ├── _get_like_pattern_expression.sql │ │ │ │ ├── expect_column_value_lengths_to_equal.sql │ │ │ │ ├── expect_column_values_to_match_regex.sql │ │ │ │ ├── expect_column_values_to_not_match_regex.sql │ │ │ │ ├── expect_column_values_to_match_like_pattern.sql │ │ │ │ ├── expect_column_values_to_not_match_like_pattern.sql │ │ │ │ ├── expect_column_values_to_not_match_regex_list.sql │ │ │ │ ├── expect_column_values_to_match_regex_list.sql │ │ │ │ ├── expect_column_value_lengths_to_be_between.sql │ │ │ │ ├── expect_column_values_to_match_like_pattern_list.sql │ │ │ │ └── expect_column_values_to_not_match_like_pattern_list.sql │ │ │ ├── column_values_basic │ │ │ │ ├── expect_column_values_to_be_of_type.sql │ │ │ │ ├── expect_column_values_to_be_unique.sql │ │ │ │ ├── expect_column_values_to_be_null.sql │ │ │ │ ├── expect_column_values_to_not_be_null.sql │ │ │ │ ├── expect_column_values_to_be_between.sql │ │ │ │ ├── expect_column_values_to_be_in_type_list.sql │ │ │ │ ├── expect_column_values_to_be_decreasing.sql │ │ │ │ ├── expect_column_values_to_be_increasing.sql │ │ │ │ ├── expect_column_values_to_not_be_in_set.sql │ │ │ │ └── expect_column_values_to_be_in_set.sql │ │ │ ├── _generalized │ │ │ │ └── _truth_expression.sql │ │ │ ├── table_shape │ │ │ │ ├── _get_column_list.sql │ │ │ │ ├── _list_intersect.sql │ │ │ │ ├── expect_table_row_count_to_equal_other_table_times_factor.sql │ │ │ │ ├── expect_table_column_count_to_equal.sql │ │ │ │ ├── expect_table_row_count_to_equal_other_table.sql │ │ │ │ ├── expect_table_column_count_to_equal_other_table.sql │ │ │ │ ├── expect_table_row_count_to_be_between.sql │ │ │ │ ├── expect_table_column_count_to_be_between.sql │ │ │ │ ├── expect_column_to_exist.sql │ │ │ │ ├── expect_table_columns_to_contain_set.sql │ │ │ │ ├── expect_table_columns_to_match_set.sql │ │ │ │ ├── expect_table_row_count_to_equal.sql │ │ │ │ ├── expect_table_columns_to_match_ordered_list.sql │ │ │ │ └── expect_row_values_to_have_recent_data.sql │ │ │ ├── multi-column │ │ │ │ ├── expect_column_pair_values_to_be_equal.sql │ │ │ │ ├── expect_multicolumn_sum_to_equal.sql │ │ │ │ ├── expect_column_pair_values_A_to_be_greater_than_B.sql │ │ │ │ └── expect_column_pair_values_to_be_in_set.sql │ │ │ └── aggregate_functions │ │ │ │ ├── expect_column_distinct_count_to_equal.sql │ │ │ │ ├── expect_column_distinct_count_to_be_greater_than.sql │ │ │ │ ├── expect_column_max_to_be_between.sql │ │ │ │ ├── expect_column_sum_to_be_between.sql │ │ │ │ ├── expect_column_mean_to_be_between.sql │ │ │ │ ├── expect_column_min_to_be_between.sql │ │ │ │ ├── expect_column_median_to_be_between.sql │ │ │ │ ├── expect_column_unique_value_count_to_be_between.sql │ │ │ │ ├── expect_column_distinct_count_to_equal_other_table.sql │ │ │ │ ├── expect_column_proportion_of_unique_values_to_be_between.sql │ │ │ │ ├── expect_column_quantile_values_to_be_between.sql │ │ │ │ ├── expect_column_distinct_values_to_be_in_set.sql │ │ │ │ ├── expect_column_distinct_values_to_equal_set.sql │ │ │ │ └── expect_column_distinct_values_to_contain_set.sql │ │ ├── utils │ │ │ └── datatypes.sql │ │ └── regex │ │ │ └── regexp_instr.sql │ ├── tests │ │ └── .gitkeep │ ├── integration_tests │ │ ├── packages.yml │ │ ├── models │ │ │ └── schema_tests │ │ │ │ ├── series_10.sql │ │ │ │ ├── series_4.sql │ │ │ │ ├── data_test_factored.sql │ │ │ │ ├── timeseries_base.sql │ │ │ │ ├── timeseries_hourly.sql │ │ │ │ ├── emails.sql │ │ │ │ ├── timeseries_data.sql │ │ │ │ ├── timeseries_data_extended.sql │ │ │ │ ├── timeseries_hourly_data_extended.sql │ │ │ │ ├── timeseries_data_grouped.sql │ │ │ │ └── data_test.sql │ │ ├── test.sh │ │ ├── macros │ │ │ └── get_custom_schema.sql │ │ ├── dbt_project.yml │ │ └── ci │ │ │ └── sample.profiles.yml │ ├── .gitignore │ ├── packages.yml │ ├── expectations.gif │ ├── .editorconfig │ ├── regen_docs.sh │ └── dbt_project.yml ├── dbt_utils │ ├── integration_tests │ │ ├── data │ │ │ ├── .gitkeep │ │ │ ├── schema_tests │ │ │ │ ├── data_test_accepted_range.csv │ │ │ │ ├── data_test_at_least_one.csv │ │ │ │ ├── data_test_not_constant.csv │ │ │ │ ├── data_cardinality_equality_a.csv │ │ │ │ ├── data_test_equal_rowcount.csv │ │ │ │ ├── data_test_fewer_rows_than_table_1.csv │ │ │ │ ├── data_test_relationships_where_table_1.csv │ │ │ │ ├── data_test_relationships_where_table_2.csv │ │ │ │ ├── data_test_fewer_rows_than_table_2.csv │ │ │ │ ├── data_cardinality_equality_b.csv │ │ │ │ ├── data_test_expression_is_true.csv │ │ │ │ ├── data_test_sequential_values.csv │ │ │ │ ├── data_test_unique_where.csv │ │ │ │ ├── data_test_not_null_where.csv │ │ │ │ ├── data_test_mutually_exclusive_ranges_no_gaps.csv │ │ │ │ ├── data_test_not_accepted_values.csv │ │ │ │ ├── data_not_null_proportion.csv │ │ │ │ ├── data_test_sequential_timestamps.csv │ │ │ │ ├── data_test_mutually_exclusive_ranges_with_gaps.csv │ │ │ │ ├── data_unique_combination_of_columns.csv │ │ │ │ ├── data_test_mutually_exclusive_ranges_with_gaps_zero_length.csv │ │ │ │ └── schema.yml │ │ │ ├── sql │ │ │ │ ├── data_pivot.csv │ │ │ │ ├── data_pivot_expected.csv │ │ │ │ ├── data_events_20180101.csv │ │ │ │ ├── data_events_20180102.csv │ │ │ │ ├── data_events_20180103.csv │ │ │ │ ├── data_star_expected.csv │ │ │ │ ├── data_star.csv │ │ │ │ ├── data_nullcheck_table.csv │ │ │ │ ├── data_union_table_2.csv │ │ │ │ ├── data_get_query_results_as_dict.csv │ │ │ │ ├── data_union_table_1.csv │ │ │ │ ├── data_safe_add.csv │ │ │ │ ├── data_generate_series.csv │ │ │ │ ├── data_get_column_values.csv │ │ │ │ ├── data_union_events_expected.csv │ │ │ │ ├── data_get_column_values_dropped.csv │ │ │ │ ├── data_star_prefix_suffix_expected.csv │ │ │ │ ├── data_union_expected.csv │ │ │ │ ├── data_unpivot.csv │ │ │ │ ├── data_unpivot_bool.csv │ │ │ │ ├── data_unpivot_expected.csv │ │ │ │ ├── data_unpivot_bool_expected.csv │ │ │ │ ├── data_unpivot_original_api_expected.csv │ │ │ │ └── data_surrogate_key.csv │ │ │ ├── cross_db │ │ │ │ ├── data_cast_bool_to_text │ │ │ │ ├── data_safe_cast.csv │ │ │ │ ├── data_concat.csv │ │ │ │ ├── data_date_trunc.csv │ │ │ │ ├── data_length.csv │ │ │ │ ├── data_split_part.csv │ │ │ │ ├── data_replace.csv │ │ │ │ ├── data_position.csv │ │ │ │ ├── data_right.csv │ │ │ │ ├── data_last_day.csv │ │ │ │ ├── data_hash.csv │ │ │ │ ├── data_dateadd.csv │ │ │ │ ├── data_width_bucket.csv │ │ │ │ └── data_datediff.csv │ │ │ ├── geo │ │ │ │ ├── data_haversine_km.csv │ │ │ │ └── data_haversine_mi.csv │ │ │ ├── datetime │ │ │ │ └── data_date_spine.csv │ │ │ ├── web │ │ │ │ ├── data_urls.csv │ │ │ │ ├── data_url_host.csv │ │ │ │ └── data_url_path.csv │ │ │ └── materializations │ │ │ │ └── data_insert_by_period.csv │ │ ├── macros │ │ │ ├── .gitkeep │ │ │ ├── limit_zero.sql │ │ │ └── tests.sql │ │ ├── packages.yml │ │ ├── .gitignore │ │ ├── models │ │ │ ├── sql │ │ │ │ ├── test_union.sql │ │ │ │ ├── test_union_base.sql │ │ │ │ ├── test_get_relations_by_prefix_and_union.sql │ │ │ │ ├── test_safe_add.sql │ │ │ │ ├── test_star.sql │ │ │ │ ├── test_get_relations_by_pattern.sql │ │ │ │ ├── test_star_prefix_suffix.sql │ │ │ │ ├── test_pivot.sql │ │ │ │ ├── test_groupby.sql │ │ │ │ ├── test_surrogate_key.sql │ │ │ │ ├── test_generate_series.sql │ │ │ │ ├── test_get_column_values.sql │ │ │ │ ├── test_nullcheck_table.sql │ │ │ │ ├── test_unpivot_bool.sql │ │ │ │ ├── test_unpivot.sql │ │ │ │ └── test_unpivot_original_api.sql │ │ │ ├── schema_tests │ │ │ │ ├── test_equal_rowcount.sql │ │ │ │ ├── test_fewer_rows_than.sql │ │ │ │ ├── test_equal_column_subset.sql │ │ │ │ └── test_recency.sql │ │ │ ├── datetime │ │ │ │ ├── schema.yml │ │ │ │ └── test_date_spine.sql │ │ │ ├── materializations │ │ │ │ ├── schema.yml │ │ │ │ ├── expected_insert_by_period.sql │ │ │ │ └── test_insert_by_period.sql │ │ │ ├── cross_db_utils │ │ │ │ ├── test_current_timestamp.sql │ │ │ │ ├── test_current_timestamp_in_utc.sql │ │ │ │ ├── test_hash.sql │ │ │ │ ├── test_length.sql │ │ │ │ ├── test_concat.sql │ │ │ │ ├── test_position.sql │ │ │ │ ├── test_right.sql │ │ │ │ ├── test_safe_cast.sql │ │ │ │ ├── test_width_bucket.sql │ │ │ │ ├── test_replace.sql │ │ │ │ ├── test_date_trunc.sql │ │ │ │ ├── test_last_day.sql │ │ │ │ ├── test_split_part.sql │ │ │ │ └── test_dateadd.sql │ │ │ ├── web │ │ │ │ ├── test_url_host.sql │ │ │ │ ├── test_url_path.sql │ │ │ │ ├── test_urls.sql │ │ │ │ └── schema.yml │ │ │ └── geo │ │ │ │ ├── schema.yml │ │ │ │ ├── test_haversine_distance_km.sql │ │ │ │ └── test_haversine_distance_mi.sql │ │ ├── tests │ │ │ ├── jinja_helpers │ │ │ │ ├── assert_pretty_time_is_string.sql │ │ │ │ ├── assert_pretty_output_msg_is_string.sql │ │ │ │ └── test_slugify.sql │ │ │ └── sql │ │ │ │ └── test_get_column_values_use_default.sql │ │ └── Makefile │ ├── .github │ │ ├── CODEOWNERS │ │ ├── ISSUE_TEMPLATE │ │ │ ├── feature_request.md │ │ │ ├── dbt_minor_release.md │ │ │ └── bug_report.md │ │ └── pull_request_template.md │ ├── .gitignore │ ├── etc │ │ └── dbt-logo.png │ ├── macros │ │ ├── cross_db_utils │ │ │ ├── literal.sql │ │ │ ├── concat.sql │ │ │ ├── except.sql │ │ │ ├── _is_relation.sql │ │ │ ├── intersect.sql │ │ │ ├── hash.sql │ │ │ ├── replace.sql │ │ │ ├── length.sql │ │ │ ├── date_trunc.sql │ │ │ ├── cast_bool_to_text.sql │ │ │ ├── position.sql │ │ │ ├── safe_cast.sql │ │ │ ├── identifier.sql │ │ │ ├── split_part.sql │ │ │ ├── _is_ephemeral.sql │ │ │ ├── right.sql │ │ │ ├── dateadd.sql │ │ │ ├── current_timestamp.sql │ │ │ └── last_day.sql │ │ ├── jinja_helpers │ │ │ ├── log_info.sql │ │ │ ├── pretty_log_format.sql │ │ │ ├── pretty_time.sql │ │ │ └── slugify.sql │ │ ├── sql │ │ │ ├── groupby.sql │ │ │ ├── nullcheck.sql │ │ │ ├── nullcheck_table.sql │ │ │ ├── get_tables_by_prefix_sql.sql │ │ │ ├── safe_add.sql │ │ │ ├── get_query_results_as_dict.sql │ │ │ ├── get_relations_by_prefix.sql │ │ │ ├── star.sql │ │ │ ├── get_relations_by_pattern.sql │ │ │ ├── generate_series.sql │ │ │ └── surrogate_key.sql │ │ ├── web │ │ │ ├── get_url_parameter.sql │ │ │ ├── get_url_host.sql │ │ │ └── get_url_path.sql │ │ └── schema_tests │ │ │ ├── not_constant.sql │ │ │ ├── test_unique_where.sql │ │ │ ├── test_not_null_where.sql │ │ │ ├── at_least_one.sql │ │ │ ├── recency.sql │ │ │ ├── expression_is_true.sql │ │ │ ├── not_null_proportion.sql │ │ │ ├── not_accepted_values.sql │ │ │ ├── equal_rowcount.sql │ │ │ ├── sequential_values.sql │ │ │ ├── cardinality_equality.sql │ │ │ ├── relationships_where.sql │ │ │ ├── unique_combination_of_columns.sql │ │ │ ├── accepted_range.sql │ │ │ └── fewer_rows_than.sql │ ├── dbt_project.yml │ ├── run_test.sh │ └── docker-compose.yml └── dbt_date │ ├── integration_tests │ ├── packages.yml │ ├── models │ │ ├── test_dates.sql │ │ ├── dates.sql │ │ ├── dim_week.sql │ │ ├── dim_hour.sql │ │ ├── dim_date_fiscal.sql │ │ └── dim_date.sql │ ├── test.sh │ ├── macros │ │ └── get_custom_schema.sql │ └── dbt_project.yml │ ├── .gitignore │ ├── packages.yml │ ├── macros │ └── calendar_date │ │ ├── last_week.sql │ │ ├── next_week.sql │ │ ├── today.sql │ │ ├── last_month.sql │ │ ├── next_month.sql │ │ ├── next_week_start_date.sql │ │ ├── now.sql │ │ ├── tomorrow.sql │ │ ├── yesterday.sql │ │ ├── last_month_start_date.sql │ │ ├── next_month_start_date.sql │ │ ├── this_week.sql │ │ ├── n_days_away.sql │ │ ├── last_month_number.sql │ │ ├── next_month_number.sql │ │ ├── last_month_name.sql │ │ ├── next_month_name.sql │ │ ├── periods_since.sql │ │ ├── n_days_ago.sql │ │ ├── n_weeks_away.sql │ │ ├── n_months_away.sql │ │ ├── n_weeks_ago.sql │ │ ├── n_months_ago.sql │ │ ├── day_of_month.sql │ │ ├── date_part.sql │ │ ├── day_of_year.sql │ │ ├── to_unixtimestamp.sql │ │ ├── week_end.sql │ │ ├── iso_week_end.sql │ │ ├── week_of_year.sql │ │ ├── iso_week_start.sql │ │ ├── iso_week_of_year.sql │ │ ├── week_start.sql │ │ ├── month_name.sql │ │ ├── day_name.sql │ │ └── convert_timezone.sql │ └── dbt_project.yml ├── requirements.txt ├── .gitpod.yml ├── packages.yml ├── models ├── staging │ ├── citibike_trips_macro_star.sql │ ├── citibike_trips_2013.sql │ ├── citibike_trips_2018.sql │ ├── citibike_trips_macro_surrogate.sql │ ├── schema.yml │ └── citibike_trips_macro_get_columns.sql └── sources │ └── schema.yml ├── Dockerfile ├── profiles └── profiles.yml └── dbt_project.yml /data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /analyses/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /macros/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /snapshots/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | dbt-core~=1.0.0 2 | dbt-bigquery 3 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/macros/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitpod.yml: -------------------------------------------------------------------------------- 1 | tasks: 2 | - init: pip install -r requirements.txt -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @clrcrl 2 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_modules/ 4 | logs/ 5 | venv/ -------------------------------------------------------------------------------- /dbt_packages/dbt_date/integration_tests/packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - local: ../ 3 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/packages.yml: -------------------------------------------------------------------------------- 1 | 2 | packages: 3 | - local: ../ 4 | -------------------------------------------------------------------------------- /packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: calogica/dbt_expectations 3 | version: 0.5.1 4 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | .python-version 6 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/integration_tests/packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - local: ../ 3 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_modules/ 4 | logs/ 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_test_accepted_range.csv: -------------------------------------------------------------------------------- 1 | id 2 | -1 3 | 11 -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_test_at_least_one.csv: -------------------------------------------------------------------------------- 1 | field 2 | a 3 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_packages/ 4 | logs/ 5 | .python-version 6 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_test_not_constant.csv: -------------------------------------------------------------------------------- 1 | field 2 | 1 3 | 1 4 | 2 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_pivot.csv: -------------------------------------------------------------------------------- 1 | size,color 2 | S,red 3 | S,blue 4 | M,red 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_pivot_expected.csv: -------------------------------------------------------------------------------- 1 | size,red,blue 2 | S,1,1 3 | M,1,0 4 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_cardinality_equality_a.csv: -------------------------------------------------------------------------------- 1 | same_name 2 | 1 3 | 2 4 | 3 -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_test_equal_rowcount.csv: -------------------------------------------------------------------------------- 1 | field 2 | 1 3 | 1 4 | 2 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_events_20180101.csv: -------------------------------------------------------------------------------- 1 | user_id,event 2 | 1,play 3 | 2,pause 4 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_events_20180102.csv: -------------------------------------------------------------------------------- 1 | user_id,event 2 | 3,play 3 | 4,pause 4 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_events_20180103.csv: -------------------------------------------------------------------------------- 1 | user_id,event 2 | 5,play 3 | 6,pause 4 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_star_expected.csv: -------------------------------------------------------------------------------- 1 | field_1,field_2 2 | a,b 3 | d,e 4 | g,h 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: [">=0.8.0", "<0.9.0"] 4 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/cross_db/data_cast_bool_to_text: -------------------------------------------------------------------------------- 1 | id,my_bool 2 | 1,true 3 | 2,false 4 | 3, 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/cross_db/data_safe_cast.csv: -------------------------------------------------------------------------------- 1 | field,output 2 | abc,abc 3 | 123,123 4 | , 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_star.csv: -------------------------------------------------------------------------------- 1 | field_1,field_2,field_3 2 | a,b,c 3 | d,e,f 4 | g,h,i 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/integration_tests/models/schema_tests/series_10.sql: -------------------------------------------------------------------------------- 1 | {{ dbt_utils.generate_series(upper_bound=10) }} -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/integration_tests/models/schema_tests/series_4.sql: -------------------------------------------------------------------------------- 1 | {{ dbt_utils.generate_series(upper_bound=4) }} -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_test_fewer_rows_than_table_1.csv: -------------------------------------------------------------------------------- 1 | field 2 | 1 3 | 2 4 | 3 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_test_relationships_where_table_1.csv: -------------------------------------------------------------------------------- 1 | id 2 | 1 3 | 2 4 | 3 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_test_relationships_where_table_2.csv: -------------------------------------------------------------------------------- 1 | id 2 | 1 3 | 2 4 | 4 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: calogica/dbt_date 3 | version: [">=0.5.0", "<0.6.0"] 4 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/etc/dbt-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdk/demo-getdbt/main/dbt_packages/dbt_utils/etc/dbt-logo.png -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/cross_db/data_concat.csv: -------------------------------------------------------------------------------- 1 | input_1,input_2,output 2 | a,b,ab 3 | a,,a 4 | ,b,b 5 | ,, 6 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_test_fewer_rows_than_table_2.csv: -------------------------------------------------------------------------------- 1 | field 2 | 1 3 | 2 4 | 3 5 | 4 6 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_cardinality_equality_b.csv: -------------------------------------------------------------------------------- 1 | same_name,different_name 2 | 1,2 3 | 2,3 4 | 3,1 -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_test_expression_is_true.csv: -------------------------------------------------------------------------------- 1 | col_a,col_b 2 | 0,1 3 | 1,0 4 | 0.5,0.5 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_nullcheck_table.csv: -------------------------------------------------------------------------------- 1 | field_1,field_2,field_3 2 | a,'',1 3 | '',b,2 4 | '','',3 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_union_table_2.csv: -------------------------------------------------------------------------------- 1 | id,favorite_color,favorite_number 2 | 1,green,7 3 | 2,pink,13 4 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/expectations.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sdk/demo-getdbt/main/dbt_packages/dbt_expectations/expectations.gif -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/integration_tests/models/schema_tests/data_test_factored.sql: -------------------------------------------------------------------------------- 1 | {{ dbt_utils.generate_series(upper_bound=8) }} 2 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_test_sequential_values.csv: -------------------------------------------------------------------------------- 1 | my_even_sequence 2 | 2 3 | 4 4 | 6 5 | 8 6 | 10 7 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_test_unique_where.csv: -------------------------------------------------------------------------------- 1 | id,_deleted 2 | 1,false 3 | 2,false 4 | 2,true 5 | 3,true -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_get_query_results_as_dict.csv: -------------------------------------------------------------------------------- 1 | col_1,col_2,col_3 2 | 1,a,True 3 | 2,b,False 4 | 3,c, 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_union_table_1.csv: -------------------------------------------------------------------------------- 1 | id,name,favorite_number 2 | 1,drew,pi 3 | 2,bob,e 4 | 3,alice,4 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/last_week.sql: -------------------------------------------------------------------------------- 1 | {%- macro last_week(tz=None) -%} 2 | {{ dbt_date.n_weeks_ago(1, tz) }} 3 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/next_week.sql: -------------------------------------------------------------------------------- 1 | {%- macro next_week(tz=None) -%} 2 | {{ dbt_date.n_weeks_away(1, tz) }} 3 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/today.sql: -------------------------------------------------------------------------------- 1 | {%- macro today(tz=None) -%} 2 | cast({{ dbt_date.now(tz) }} as date) 3 | {%- endmacro -%} 4 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_test_not_null_where.csv: -------------------------------------------------------------------------------- 1 | id,_deleted 2 | 1,false 3 | 2,false 4 | ,true 5 | 3,false -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_safe_add.csv: -------------------------------------------------------------------------------- 1 | field_1,field_2,field_3,expected 2 | 1,2,3,6 3 | 1,,3,4 4 | ,,2,2 5 | ,,,0 6 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/last_month.sql: -------------------------------------------------------------------------------- 1 | {%- macro last_month(tz=None) -%} 2 | {{ dbt_date.n_months_ago(1, tz) }} 3 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/next_month.sql: -------------------------------------------------------------------------------- 1 | {%- macro next_month(tz=None) -%} 2 | {{ dbt_date.n_months_away(1, tz) }} 3 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/cross_db/data_date_trunc.csv: -------------------------------------------------------------------------------- 1 | updated_at,day,month 2 | 2018-01-05 12:00:00,2018-01-05,2018-01-01 3 | ,, 4 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/geo/data_haversine_km.csv: -------------------------------------------------------------------------------- 1 | lat_1,lon_1,lat_2,lon_2,output 2 | 48.864716,2.349014,52.379189,4.899431,430 3 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/geo/data_haversine_mi.csv: -------------------------------------------------------------------------------- 1 | lat_1,lon_1,lat_2,lon_2,output 2 | 48.864716,2.349014,52.379189,4.899431,267 3 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/integration_tests/models/schema_tests/timeseries_base.sql: -------------------------------------------------------------------------------- 1 | {{ dbt_date.get_base_dates(n_dateparts=366, datepart='day') }} 2 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/math/median.sql: -------------------------------------------------------------------------------- 1 | {% macro median(field) %} 2 | {{ dbt_expectations.percentile_cont(field, 0.5) }} 3 | {% endmacro %} 4 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/cross_db/data_length.csv: -------------------------------------------------------------------------------- 1 | expression,output 2 | abcdef,6 3 | fishtown,8 4 | december,8 5 | www.google.com/path,19 -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_test_mutually_exclusive_ranges_no_gaps.csv: -------------------------------------------------------------------------------- 1 | lower_bound,upper_bound 2 | 0,1 3 | 1,2 4 | 2,4 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_test_not_accepted_values.csv: -------------------------------------------------------------------------------- 1 | id,city 2 | 1,Barcelona 3 | 2,London 4 | 3,Paris 5 | 4,New York 6 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/next_week_start_date.sql: -------------------------------------------------------------------------------- 1 | {%- macro next_week_start_date(tz=None) -%} 2 | {{ dbt_date.next_week(1, tz) }} 3 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/now.sql: -------------------------------------------------------------------------------- 1 | {%- macro now(tz=None) -%} 2 | {{ dbt_date.convert_timezone(dbt_utils.current_timestamp(), tz) }} 3 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/tomorrow.sql: -------------------------------------------------------------------------------- 1 | {%- macro tomorrow(date=None, tz=None) -%} 2 | {{ dbt_date.n_days_away(1, date, tz) }} 3 | {%- endmacro -%} 4 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/yesterday.sql: -------------------------------------------------------------------------------- 1 | {%- macro yesterday(date=None, tz=None) -%} 2 | {{ dbt_date.n_days_ago(1, date, tz) }} 3 | {%- endmacro -%} 4 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/cross_db/data_split_part.csv: -------------------------------------------------------------------------------- 1 | parts,split_on,result_1,result_2,result_3 2 | a|b|c,|,a,b,c 3 | 1|2|3,|,1,2,3 4 | ,|,,, 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_generate_series.csv: -------------------------------------------------------------------------------- 1 | generated_number 2 | 1 3 | 2 4 | 3 5 | 4 6 | 5 7 | 6 8 | 7 9 | 8 10 | 9 11 | 10 12 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_get_column_values.csv: -------------------------------------------------------------------------------- 1 | field 2 | a 3 | b 4 | c 5 | d 6 | e 7 | f 8 | g 9 | g 10 | g 11 | g 12 | g 13 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_union_events_expected.csv: -------------------------------------------------------------------------------- 1 | user_id,event 2 | 1,play 3 | 2,pause 4 | 3,play 5 | 4,pause 6 | 5,play 7 | 6,pause 8 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/last_month_start_date.sql: -------------------------------------------------------------------------------- 1 | {%- macro last_month_start_date(tz=None) -%} 2 | {{ dbt_date.last_month(1, tz) }} 3 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/next_month_start_date.sql: -------------------------------------------------------------------------------- 1 | {%- macro next_month_start_date(tz=None) -%} 2 | {{ dbt_date.next_month(1, tz) }} 3 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/this_week.sql: -------------------------------------------------------------------------------- 1 | {%- macro this_week(date=None, tz=None) -%} 2 | {{ dbt_utils.this_week_start(date, tz) }} 3 | {%- endmacro -%} 4 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/cross_db/data_replace.csv: -------------------------------------------------------------------------------- 1 | string_text,search_chars,replace_chars,result 2 | a,a,b,b 3 | http://google.com,http://,"",google.com -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_get_column_values_dropped.csv: -------------------------------------------------------------------------------- 1 | field 2 | a 3 | b 4 | c 5 | d 6 | e 7 | f 8 | g 9 | g 10 | g 11 | g 12 | g 13 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/n_days_away.sql: -------------------------------------------------------------------------------- 1 | {%- macro n_days_away(n, date=None, tz=None) -%} 2 | {{ dbt_date.n_days_ago(-1 * n, date, tz) }} 3 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/cross_db/data_position.csv: -------------------------------------------------------------------------------- 1 | substring_text,string_text,result 2 | def,abcdef,4 3 | land,earth,0 4 | town,fishtown,5 5 | ember,december,4 -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/cross_db/data_right.csv: -------------------------------------------------------------------------------- 1 | string_text,length_expression,output 2 | abcdef,3,def 3 | fishtown,4,town 4 | december,5,ember 5 | december,0, -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/sql/test_union.sql: -------------------------------------------------------------------------------- 1 | 2 | select 3 | id, 4 | name, 5 | favorite_color 6 | 7 | from {{ ref('test_union_base') }} 8 | 9 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_star_prefix_suffix_expected.csv: -------------------------------------------------------------------------------- 1 | prefix_field_1_suffix,prefix_field_2_suffix,prefix_field_3_suffix 2 | a,b,c 3 | d,e,f 4 | g,h,i 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/integration_tests/models/test_dates.sql: -------------------------------------------------------------------------------- 1 | {{ 2 | config( 3 | materialized = 'table' 4 | ) 5 | }} 6 | {{ dbt_date_integration_tests.get_test_dates() }} 7 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/last_month_number.sql: -------------------------------------------------------------------------------- 1 | {%- macro last_month_number(tz=None) -%} 2 | {{ dbt_date.date_part('month', dbt_date.last_month(1, tz)) }} 3 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/next_month_number.sql: -------------------------------------------------------------------------------- 1 | {%- macro next_month_number(tz=None) -%} 2 | {{ dbt_date.date_part('month', dbt_date.next_month(1, tz)) }} 3 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_not_null_proportion.csv: -------------------------------------------------------------------------------- 1 | point_5,point_9 2 | 1,1 3 | ,2 4 | ,3 5 | 4,4 6 | 5,5 7 | 6,6 8 | ,7 9 | ,8 10 | , 11 | 10,10 -------------------------------------------------------------------------------- /dbt_packages/dbt_date/integration_tests/models/dates.sql: -------------------------------------------------------------------------------- 1 | {{ 2 | config( 3 | materialized = "table" 4 | ) 5 | }} 6 | {{ dbt_date.get_date_dimension('2015-01-01', '2022-12-31') }} 7 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_test_sequential_timestamps.csv: -------------------------------------------------------------------------------- 1 | my_timestamp 2 | 2021-01-01 00:00 3 | 2021-01-01 01:00 4 | 2021-01-01 02:00 5 | 2021-01-01 03:00 6 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/sql/test_union_base.sql: -------------------------------------------------------------------------------- 1 | 2 | {{ dbt_utils.union_relations([ 3 | ref('data_union_table_1'), 4 | ref('data_union_table_2')] 5 | ) }} 6 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_union_expected.csv: -------------------------------------------------------------------------------- 1 | id,name,favorite_color,favorite_number 2 | 1,"drew",,pi 3 | 2,"bob",,e 4 | 3,"alice",,4 5 | 1,,"green",7 6 | 2,,"pink",13 7 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/last_month_name.sql: -------------------------------------------------------------------------------- 1 | {%- macro last_month_name(short=True, tz=None) -%} 2 | {{ dbt_date.month_name(dbt_date.lastnext_month(1, tz), short=short) }} 3 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/next_month_name.sql: -------------------------------------------------------------------------------- 1 | {%- macro next_month_name(short=True, tz=None) -%} 2 | {{ dbt_date.month_name(dbt_date.next_month(1, tz), short=short) }} 3 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/cross_db/data_last_day.csv: -------------------------------------------------------------------------------- 1 | date_day,date_part,result 2 | 2018-01-02,month,2018-01-31 3 | 2018-01-02,quarter,2018-03-31 4 | 2018-01-02,year,2018-12-31 5 | ,month, 6 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/periods_since.sql: -------------------------------------------------------------------------------- 1 | {%- macro periods_since(date_col, period_name='day', tz=None) -%} 2 | {{ dbt_utils.datediff(date_col, dbt_date.today(tz), period_name) }} 3 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/schema_tests/test_equal_rowcount.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | 3 | select * from {{ ref('data_test_equal_rowcount') }} 4 | 5 | ) 6 | 7 | select 8 | field 9 | from data -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/integration_tests/models/schema_tests/timeseries_hourly.sql: -------------------------------------------------------------------------------- 1 | {{ dbt_utils.date_spine('hour', 2 | start_date=dbt_date.n_days_ago(10), 3 | end_date=dbt_date.tomorrow() 4 | ) }} 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/datetime/data_date_spine.csv: -------------------------------------------------------------------------------- 1 | date_day 2 | 2018-01-01 3 | 2018-01-02 4 | 2018-01-03 5 | 2018-01-04 6 | 2018-01-05 7 | 2018-01-06 8 | 2018-01-07 9 | 2018-01-08 10 | 2018-01-09 -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/web/data_urls.csv: -------------------------------------------------------------------------------- 1 | url,medium,source 2 | http://drewbanin.com/milky?utm_medium=organic,organic, 3 | http://drewbanin.com/milky?utm_medium=organic&utm_source=github,organic,github 4 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/datetime/schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: test_date_spine 5 | tests: 6 | - dbt_utils.equality: 7 | compare_model: ref('data_date_spine') 8 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/schema_tests/test_fewer_rows_than.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | 3 | select * from {{ ref('data_test_fewer_rows_than_table_1') }} 4 | 5 | ) 6 | 7 | select 8 | field 9 | from data -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_unpivot.csv: -------------------------------------------------------------------------------- 1 | customer_id,created_at,status,segment,name 2 | 123,2017-01-01,active,tier 1,name 1 3 | 234,2017-02-01,active,tier 3,name 3 4 | 567,2017-03-01,churned,tier 2,name 2 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_unpivot_bool.csv: -------------------------------------------------------------------------------- 1 | customer_id,created_at,status,segment,is_updated 2 | 123,2017-01-01,active,tier 1,TRUE 3 | 234,2017-02-01,active,tier 3,FALSE 4 | 567,2017-03-01,churned,tier 2, 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/cross_db/data_hash.csv: -------------------------------------------------------------------------------- 1 | input_1,output 2 | ab,187ef4436122d1cc2f40dc2b92f0eba0 3 | a,0cc175b9c0f1b6a831c399e269772661 4 | 1,c4ca4238a0b923820dcc509a6f75849b 5 | ,d41d8cd98f00b204e9800998ecf8427e 6 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_test_mutually_exclusive_ranges_with_gaps.csv: -------------------------------------------------------------------------------- 1 | subscription_id,valid_from,valid_to 2 | 1,2019-01-01,2019-02-01 3 | 1,2019-03-03,2019-04-01 4 | 2,2019-05-06,2019-07-02 5 | 2,2019-07-03, 6 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/materializations/schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: test_insert_by_period 5 | tests: 6 | - dbt_utils.equality: 7 | compare_model: ref('expected_insert_by_period') 8 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/schema_tests/test_equal_column_subset.sql: -------------------------------------------------------------------------------- 1 | {{ config(materialized='ephemeral') }} 2 | 3 | select 4 | 5 | first_name, 6 | last_name, 7 | email 8 | 9 | from {{ ref('data_people') }} 10 | -------------------------------------------------------------------------------- /models/staging/citibike_trips_macro_star.sql: -------------------------------------------------------------------------------- 1 | with citibike_trips_macro_star as ( 2 | select 3 | {{ dbt_utils.star(ref('citibike_trips_2018')) }} 4 | from {{ ref('citibike_trips_2018') }} 5 | ) 6 | select * from citibike_trips_macro_star -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/cross_db_utils/test_current_timestamp.sql: -------------------------------------------------------------------------------- 1 | 2 | -- how can we test this better? 3 | select 4 | {{ dbt_utils.current_timestamp() }} as actual, 5 | {{ dbt_utils.current_timestamp() }} as expected 6 | 7 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 4 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | insert_final_newline = true 9 | 10 | [*.{yml,yaml}] 11 | indent_size = 2 12 | -------------------------------------------------------------------------------- /models/staging/citibike_trips_2013.sql: -------------------------------------------------------------------------------- 1 | with citibike_trips_2018 AS ( 2 | SELECT * from {{source('citybike','citibike_trips')}} 3 | WHERE starttime >= '2013-01-01' and 4 | starttime < '2014-01-01' 5 | ) 6 | 7 | select * from citibike_trips_2018 -------------------------------------------------------------------------------- /models/staging/citibike_trips_2018.sql: -------------------------------------------------------------------------------- 1 | with citibike_trips_2018 AS ( 2 | SELECT * from {{source('citybike','citibike_trips')}} 3 | WHERE starttime >= '2018-01-01' and 4 | starttime < '2019-01-01' 5 | ) 6 | 7 | select * from citibike_trips_2018 -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql: -------------------------------------------------------------------------------- 1 | 2 | -- how can we test this better? 3 | select 4 | {{ dbt_utils.current_timestamp_in_utc() }} as actual, 5 | {{ dbt_utils.current_timestamp_in_utc() }} as expected -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/materializations/data_insert_by_period.csv: -------------------------------------------------------------------------------- 1 | id,created_at 2 | 1,2017-12-02 3 | 2,2018-01-02 4 | 3,2018-02-02 5 | 4,2018-03-02 6 | 5,2018-04-02 7 | 6,2018-05-02 8 | 7,2018-06-02 9 | 8,2018-07-02 10 | 9,2018-08-02 11 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/n_days_ago.sql: -------------------------------------------------------------------------------- 1 | {%- macro n_days_ago(n, date=None, tz=None) -%} 2 | {%-set dt = date if date else dbt_date.today(tz) -%} 3 | {%- set n = n|int -%} 4 | cast({{ dbt_utils.dateadd('day', -1 * n, dt) }} as date) 5 | {%- endmacro -%} 6 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/n_weeks_away.sql: -------------------------------------------------------------------------------- 1 | {%- macro n_weeks_away(n, tz=None) -%} 2 | {%- set n = n|int -%} 3 | {{ dbt_utils.date_trunc('week', 4 | dbt_utils.dateadd('week', n, 5 | dbt_date.today(tz) 6 | ) 7 | ) }} 8 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql: -------------------------------------------------------------------------------- 1 | {{ config(materialized = 'table') }} 2 | 3 | {% set relations = dbt_utils.get_relations_by_prefix(target.schema, 'data_events_') %} 4 | {{ dbt_utils.union_relations(relations) }} 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/n_months_away.sql: -------------------------------------------------------------------------------- 1 | {%- macro n_months_away(n, tz=None) -%} 2 | {%- set n = n|int -%} 3 | {{ dbt_utils.date_trunc('month', 4 | dbt_utils.dateadd('month', n, 5 | dbt_date.today(tz) 6 | ) 7 | ) }} 8 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/n_weeks_ago.sql: -------------------------------------------------------------------------------- 1 | {%- macro n_weeks_ago(n, tz=None) -%} 2 | {%- set n = n|int -%} 3 | {{ dbt_utils.date_trunc('week', 4 | dbt_utils.dateadd('week', -1 * n, 5 | dbt_date.today(tz) 6 | ) 7 | ) }} 8 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/n_months_ago.sql: -------------------------------------------------------------------------------- 1 | {%- macro n_months_ago(n, tz=None) -%} 2 | {%- set n = n|int -%} 3 | {{ dbt_utils.date_trunc('month', 4 | dbt_utils.dateadd('month', -1 * n, 5 | dbt_date.today(tz) 6 | ) 7 | ) }} 8 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/string_matching/_get_like_pattern_expression.sql: -------------------------------------------------------------------------------- 1 | 2 | {% macro _get_like_pattern_expression(column_name, like_pattern, positive) %} 3 | {{ column_name }} {{ "not" if not positive else "" }} like '{{ like_pattern }}' 4 | {% endmacro %} 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_unique_combination_of_columns.csv: -------------------------------------------------------------------------------- 1 | month,product,revenue 2 | 2019-01-01,jaffle,500 3 | 2019-01-01,lamington,100 4 | 2019-01-01,pavlova,600 5 | 2019-02-01,jaffle,300 6 | 2019-02-01,lamington,300 7 | 2019-02-01,pavlova,400 8 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/cross_db_utils/test_hash.sql: -------------------------------------------------------------------------------- 1 | 2 | with data as ( 3 | 4 | select * from {{ ref('data_hash') }} 5 | 6 | ) 7 | 8 | select 9 | {{ dbt_utils.hash('input_1') }} as actual, 10 | output as expected 11 | 12 | from data 13 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/cross_db_utils/test_length.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | 3 | select * from {{ ref('data_length') }} 4 | 5 | ) 6 | 7 | select 8 | 9 | {{ dbt_utils.length('expression') }} as actual, 10 | output as expected 11 | 12 | from data -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/literal.sql: -------------------------------------------------------------------------------- 1 | 2 | {%- macro string_literal(value) -%} 3 | {{ return(adapter.dispatch('string_literal', 'dbt_utils') (value)) }} 4 | {%- endmacro -%} 5 | 6 | {% macro default__string_literal(value) -%} 7 | '{{ value }}' 8 | {%- endmacro %} 9 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/concat.sql: -------------------------------------------------------------------------------- 1 | 2 | 3 | {% macro concat(fields) -%} 4 | {{ return(adapter.dispatch('concat', 'dbt_utils')(fields)) }} 5 | {%- endmacro %} 6 | 7 | {% macro default__concat(fields) -%} 8 | {{ fields|join(' || ') }} 9 | {%- endmacro %} 10 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/sql/test_safe_add.sql: -------------------------------------------------------------------------------- 1 | 2 | with data as ( 3 | 4 | select * from {{ ref('data_safe_add') }} 5 | 6 | ) 7 | 8 | select 9 | {{ dbt_utils.safe_add('field_1', 'field_2', 'field_3') }} as actual, 10 | expected 11 | 12 | from data 13 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/integration_tests/models/dim_week.sql: -------------------------------------------------------------------------------- 1 | {{ 2 | config( 3 | materialized = "table" 4 | ) 5 | }} 6 | with periods_weeks as ( 7 | {{ dbt_date.get_base_dates(n_dateparts=52, datepart="week") }} 8 | ) 9 | select 10 | d.* 11 | from 12 | periods_weeks d 13 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/integration_tests/test.sh: -------------------------------------------------------------------------------- 1 | if [[ $# -gt 0 ]] 2 | then 3 | 4 | i=1; 5 | for t in "$@" 6 | do 7 | 8 | dbt build -t $t 9 | 10 | done 11 | 12 | else 13 | echo "Please specify one or more targets as command-line arguments, i.e. test.sh bq snowflake" 14 | fi 15 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/cross_db_utils/test_concat.sql: -------------------------------------------------------------------------------- 1 | 2 | with data as ( 3 | 4 | select * from {{ ref('data_concat') }} 5 | 6 | ) 7 | 8 | select 9 | {{ dbt_utils.concat(['input_1', 'input_2']) }} as actual, 10 | output as expected 11 | 12 | from data 13 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/materializations/expected_insert_by_period.sql: -------------------------------------------------------------------------------- 1 | {{ 2 | config( 3 | materialized = 'view', 4 | enabled=(target.type == 'redshift') 5 | ) 6 | }} 7 | 8 | select * 9 | from {{ ref('data_insert_by_period') }} 10 | where id in (2, 3, 4, 5, 6) 11 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/web/test_url_host.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | 3 | select * from {{ref('data_url_host')}} 4 | 5 | ) 6 | 7 | select 8 | 9 | {{ dbt_utils.get_url_host('original_url') }} as actual, 10 | parsed_url as expected 11 | 12 | from data -------------------------------------------------------------------------------- /dbt_packages/dbt_date/integration_tests/models/dim_hour.sql: -------------------------------------------------------------------------------- 1 | {{ 2 | config( 3 | materialized = "table" 4 | ) 5 | }} 6 | with periods_hours as ( 7 | {{ dbt_date.get_base_dates(n_dateparts=24*28, datepart="hour") }} 8 | ) 9 | select 10 | d.* 11 | from 12 | periods_hours d 13 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/day_of_month.sql: -------------------------------------------------------------------------------- 1 | {%- macro day_of_month(date) -%} 2 | {{ dbt_date.date_part('day', date) }} 3 | {%- endmacro %} 4 | 5 | {%- macro redshift__day_of_month(date) -%} 6 | cast({{ dbt_date.date_part('day', date) }} as {{ dbt_utils.type_bigint() }}) 7 | {%- endmacro %} 8 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/dbt_project.yml: -------------------------------------------------------------------------------- 1 | name: 'dbt_utils' 2 | version: '0.1.0' 3 | 4 | require-dbt-version: [">=1.0.0", "<2.0.0"] 5 | 6 | config-version: 2 7 | 8 | target-path: "target" 9 | clean-targets: ["target", "dbt_modules", "dbt_packages"] 10 | macro-paths: ["macros"] 11 | log-path: "logs" 12 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/integration_tests/models/dim_date_fiscal.sql: -------------------------------------------------------------------------------- 1 | {{ 2 | config( 3 | materialized = "table" 4 | ) 5 | }} 6 | with fp as ( 7 | {{ dbt_date.get_fiscal_periods(ref('dates'), year_end_month=1, week_start_day=1) }} 8 | ) 9 | select 10 | f.* 11 | from 12 | fp f 13 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/cross_db_utils/test_position.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | 3 | select * from {{ ref('data_position') }} 4 | 5 | ) 6 | 7 | select 8 | 9 | {{ dbt_utils.position('substring_text', 'string_text') }} as actual, 10 | result as expected 11 | 12 | from data -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/jinja_helpers/log_info.sql: -------------------------------------------------------------------------------- 1 | {% macro log_info(message) %} 2 | {{ return(adapter.dispatch('log_info', 'dbt_utils')(message)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__log_info(message) %} 6 | {{ log(dbt_utils.pretty_log_format(message), info=True) }} 7 | {% endmacro %} 8 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/data_test_mutually_exclusive_ranges_with_gaps_zero_length.csv: -------------------------------------------------------------------------------- 1 | subscription_id,valid_from,valid_to 2 | 3,2020-05-06,2020-05-07 3 | 3,2020-05-08,2020-05-08 4 | 3,2020-05-09,2020-05-10 5 | 4,2020-06-06,2020-06-07 6 | 4,2020-06-08,2020-06-08 7 | 4,2020-06-09,2020-06-10 -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/cross_db_utils/test_right.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | 3 | select * from {{ ref('data_right') }} 4 | 5 | ) 6 | 7 | select 8 | 9 | {{ dbt_utils.right('string_text', 'length_expression') }} as actual, 10 | coalesce(output, '') as expected 11 | 12 | from data -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/column_values_basic/expect_column_values_to_be_of_type.sql: -------------------------------------------------------------------------------- 1 | {%- test expect_column_values_to_be_of_type(model, column_name, column_type) -%} 2 | {{ dbt_expectations.test_expect_column_values_to_be_in_type_list(model, column_name, [column_type]) }} 3 | {%- endtest -%} 4 | 5 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/cross_db_utils/test_safe_cast.sql: -------------------------------------------------------------------------------- 1 | 2 | with data as ( 3 | 4 | select * from {{ ref('data_safe_cast') }} 5 | 6 | ) 7 | 8 | select 9 | {{ dbt_utils.safe_cast('field', dbt_utils.type_string()) }} as actual, 10 | output as expected 11 | 12 | from data 13 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/sql/test_star.sql: -------------------------------------------------------------------------------- 1 | {% set exclude_field = 'field_3' %} 2 | 3 | 4 | with data as ( 5 | 6 | select 7 | {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} 8 | 9 | from {{ ref('data_star') }} 10 | 11 | ) 12 | 13 | select * from data 14 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/cross_db/data_dateadd.csv: -------------------------------------------------------------------------------- 1 | from_time,interval_length,datepart,result 2 | 2018-01-01 01:00:00,1,day,2018-01-02 01:00:00 3 | 2018-01-01 01:00:00,1,month,2018-02-01 01:00:00 4 | 2018-01-01 01:00:00,1,year,2019-01-01 01:00:00 5 | 2018-01-01 01:00:00,1,hour,2018-01-01 02:00:00 6 | ,1,day, 7 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/column_values_basic/expect_column_values_to_be_unique.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_values_to_be_unique(model, column_name, row_condition=None) %} 2 | {{ dbt_expectations.test_expect_compound_columns_to_be_unique(model, [column_name], row_condition=row_condition) }} 3 | {% endtest %} 4 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/web/test_url_path.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | 3 | select * from {{ref('data_url_path')}} 4 | 5 | ) 6 | 7 | select 8 | 9 | coalesce({{ dbt_utils.get_url_path('original_url') }}, '') as actual, 10 | coalesce(parsed_path, '') as expected 11 | 12 | from data -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql: -------------------------------------------------------------------------------- 1 | {% if dbt_utils.pretty_time() is string %} 2 | {# Return 0 rows for the test to pass #} 3 | select 1 as col_name {{ limit_zero() }} 4 | {% else %} 5 | {# Return >0 rows for the test to fail #} 6 | select 1 7 | {% endif %} 8 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/regen_docs.sh: -------------------------------------------------------------------------------- 1 | git checkout -B docs-regen 2 | cd integration_tests 3 | dbt docs generate 4 | mv -f target/*.json ../docs 5 | mv -f target/*.html ../docs 6 | git add . 7 | git commit -am"updating docs site" 8 | git push --set-upstream origin docs-regen 9 | git checkout 10 | git branch -D docs-regen 11 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql: -------------------------------------------------------------------------------- 1 | {% if dbt_utils.pretty_log_format() is string %} 2 | {# Return 0 rows for the test to pass #} 3 | select 1 as col_name {{ limit_zero() }} 4 | {% else %} 5 | {# Return >0 rows for the test to fail #} 6 | select 1 7 | {% endif %} 8 | -------------------------------------------------------------------------------- /models/staging/citibike_trips_macro_surrogate.sql: -------------------------------------------------------------------------------- 1 | with citibike_trips_macro_star as ( 2 | select 3 | {{ dbt_utils.surrogate_key(['bikeid', 'starttime']) }} as my_surrogate_key, 4 | {{ dbt_utils.star(ref('citibike_trips_2018')) }} 5 | from {{ ref('citibike_trips_2018') }} 6 | ) 7 | select * from citibike_trips_macro_star -------------------------------------------------------------------------------- /dbt_packages/dbt_date/integration_tests/test.sh: -------------------------------------------------------------------------------- 1 | if [[ $# -gt 0 ]] 2 | then 3 | 4 | i=1; 5 | for t in "$@" 6 | do 7 | 8 | dbt seed -t $t && 9 | dbt run -t $t && 10 | dbt test -t $t 11 | 12 | done 13 | 14 | else 15 | echo "Please specify one or more targets as command-line arguments, i.e. test.sh bq snowflake" 16 | fi 17 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/_generalized/_truth_expression.sql: -------------------------------------------------------------------------------- 1 | 2 | {% macro truth_expression(expression) %} 3 | {{ adapter.dispatch('truth_expression', 'dbt_expectations') (expression) }} 4 | {% endmacro %} 5 | 6 | {% macro default__truth_expression(expression) %} 7 | {{ expression }} as expression 8 | {% endmacro %} 9 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_unpivot_expected.csv: -------------------------------------------------------------------------------- 1 | customer_id,created_at,prop,val 2 | 123,"2017-01-01","segment","tier 1" 3 | 123,"2017-01-01","status","active" 4 | 234,"2017-02-01","segment","tier 3" 5 | 234,"2017-02-01","status","active" 6 | 567,"2017-03-01","status","churned" 7 | 567,"2017-03-01","segment","tier 2" 8 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/macros/limit_zero.sql: -------------------------------------------------------------------------------- 1 | {% macro my_custom_macro() %} 2 | whatever 3 | {% endmacro %} 4 | 5 | {% macro limit_zero() %} 6 | {{ return(adapter.dispatch('limit_zero', 'dbt_utils')()) }} 7 | {% endmacro %} 8 | 9 | {% macro default__limit_zero() %} 10 | {{ return('limit 0') }} 11 | {% endmacro %} -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/cross_db_utils/test_width_bucket.sql: -------------------------------------------------------------------------------- 1 | 2 | with data as ( 3 | 4 | select * from {{ ref('data_width_bucket') }} 5 | 6 | ) 7 | 8 | select 9 | {{ dbt_utils.width_bucket('amount', 'min_value', 'max_value', 'num_buckets') }} as actual, 10 | bucket as expected 11 | 12 | from data 13 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/jinja_helpers/pretty_log_format.sql: -------------------------------------------------------------------------------- 1 | {% macro pretty_log_format(message) %} 2 | {{ return(adapter.dispatch('pretty_log_format', 'dbt_utils')(message)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__pretty_log_format(message) %} 6 | {{ return( dbt_utils.pretty_time() ~ ' + ' ~ message) }} 7 | {% endmacro %} 8 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/jinja_helpers/pretty_time.sql: -------------------------------------------------------------------------------- 1 | {% macro pretty_time(format='%H:%M:%S') %} 2 | {{ return(adapter.dispatch('pretty_time', 'dbt_utils')(format)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__pretty_time(format='%H:%M:%S') %} 6 | {{ return(modules.datetime.datetime.now().strftime(format)) }} 7 | {% endmacro %} 8 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/tests/jinja_helpers/test_slugify.sql: -------------------------------------------------------------------------------- 1 | {% if dbt_utils.slugify('!Hell0 world-hi') == 'hell0_world_hi' %} 2 | {# Return 0 rows for the test to pass #} 3 | select 1 as col_name {{ limit_zero() }} 4 | {% else %} 5 | {# Return >0 rows for the test to fail #} 6 | select 1 as col_name 7 | {% endif %} 8 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/except.sql: -------------------------------------------------------------------------------- 1 | {% macro except() %} 2 | {{ return(adapter.dispatch('except', 'dbt_utils')()) }} 3 | {% endmacro %} 4 | 5 | 6 | {% macro default__except() %} 7 | 8 | except 9 | 10 | {% endmacro %} 11 | 12 | {% macro bigquery__except() %} 13 | 14 | except distinct 15 | 16 | {% endmacro %} -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/macros/tests.sql: -------------------------------------------------------------------------------- 1 | 2 | {% test assert_equal(model, actual, expected) %} 3 | select * from {{ model }} where {{ actual }} != {{ expected }} 4 | 5 | {% endtest %} 6 | 7 | 8 | {% test not_empty_string(model, column_name) %} 9 | 10 | select * from {{ model }} where {{ column_name }} = '' 11 | 12 | {% endtest %} 13 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/sql/groupby.sql: -------------------------------------------------------------------------------- 1 | {%- macro group_by(n) -%} 2 | {{ return(adapter.dispatch('group_by', 'dbt_utils')(n)) }} 3 | {% endmacro %} 4 | 5 | {%- macro default__group_by(n) -%} 6 | 7 | group by {% for i in range(1, n + 1) -%} 8 | {{ i }}{{ ',' if not loop.last }} 9 | {%- endfor -%} 10 | 11 | {%- endmacro -%} 12 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/_is_relation.sql: -------------------------------------------------------------------------------- 1 | {% macro _is_relation(obj, macro) %} 2 | {%- if not (obj is mapping and obj.get('metadata', {}).get('type', '').endswith('Relation')) -%} 3 | {%- do exceptions.raise_compiler_error("Macro " ~ macro ~ " expected a Relation but received the value: " ~ obj) -%} 4 | {%- endif -%} 5 | {% endmacro %} 6 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/intersect.sql: -------------------------------------------------------------------------------- 1 | {% macro intersect() %} 2 | {{ return(adapter.dispatch('intersect', 'dbt_utils')()) }} 3 | {% endmacro %} 4 | 5 | 6 | {% macro default__intersect() %} 7 | 8 | intersect 9 | 10 | {% endmacro %} 11 | 12 | {% macro bigquery__intersect() %} 13 | 14 | intersect distinct 15 | 16 | {% endmacro %} 17 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/table_shape/_get_column_list.sql: -------------------------------------------------------------------------------- 1 | {%- macro _get_column_list(model, transform="upper") -%} 2 | {%- set relation_columns = adapter.get_columns_in_relation(model) -%} 3 | {%- set relation_column_names = relation_columns | map(attribute="name") | map(transform) | list -%} 4 | {%- do return(relation_column_names) -%} 5 | {%- endmacro -%} 6 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/table_shape/_list_intersect.sql: -------------------------------------------------------------------------------- 1 | 2 | {%- macro _list_intersect(list1, list2) -%} 3 | {%- set matching_items = [] -%} 4 | {%- for itm in list1 -%} 5 | {%- if itm in list2 -%} 6 | {%- do matching_items.append(itm) -%} 7 | {%- endif -%} 8 | {%- endfor -%} 9 | {%- do return(matching_items) -%} 10 | {%- endmacro -%} 11 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/schema_tests/test_recency.sql: -------------------------------------------------------------------------------- 1 | 2 | {% if target.type == 'postgres' %} 3 | 4 | select 5 | {{ dbt_utils.date_trunc('day', dbt_utils.current_timestamp()) }} as today 6 | 7 | {% else %} 8 | 9 | select 10 | cast({{ dbt_utils.date_trunc('day', dbt_utils.current_timestamp()) }} as datetime) as today 11 | 12 | {% endif %} 13 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/geo/schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: test_haversine_distance_km 5 | tests: 6 | - assert_equal: 7 | actual: actual 8 | expected: expected 9 | - name: test_haversine_distance_mi 10 | tests: 11 | - assert_equal: 12 | actual: actual 13 | expected: expected 14 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/integration_tests/macros/get_custom_schema.sql: -------------------------------------------------------------------------------- 1 | {% macro generate_schema_name(custom_schema_name, node) -%} 2 | 3 | {%- set default_schema = target.schema -%} 4 | {%- if custom_schema_name is none -%} 5 | 6 | {{ default_schema }} 7 | 8 | {%- else -%} 9 | 10 | {{ custom_schema_name | trim }} 11 | 12 | {%- endif -%} 13 | 14 | {%- endmacro %} 15 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/integration_tests/macros/get_custom_schema.sql: -------------------------------------------------------------------------------- 1 | {% macro generate_schema_name(custom_schema_name, node) -%} 2 | 3 | {%- set default_schema = target.schema -%} 4 | {%- if custom_schema_name is none -%} 5 | 6 | {{ default_schema }} 7 | 8 | {%- else -%} 9 | 10 | {{ custom_schema_name | trim }} 11 | 12 | {%- endif -%} 13 | 14 | {%- endmacro %} 15 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/sql/test_get_relations_by_pattern.sql: -------------------------------------------------------------------------------- 1 | {{ config(materialized = 'table') }} 2 | 3 | {% set relations = dbt_utils.get_relations_by_pattern(target.schema ~ '%', 'data_events_%') %} 4 | 5 | with unioned as ( 6 | 7 | {{ dbt_utils.union_relations(relations) }} 8 | 9 | ) 10 | 11 | select 12 | 13 | user_id, 14 | event 15 | 16 | from unioned 17 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.8.8-slim-buster 2 | USER root 3 | RUN \ 4 | apt-get update &&\ 5 | apt-get install -y git 6 | COPY ./requirements.txt /app/requirements.txt 7 | RUN pip3 install --use-feature=2020-resolver -r /app/requirements.txt --no-deps 8 | RUN pip3 install --use-feature=2020-resolver -r /app/requirements.txt 9 | WORKDIR /root 10 | COPY . . 11 | COPY profiles/ .dbt/ 12 | CMD ["dbt","test"] 13 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_unpivot_bool_expected.csv: -------------------------------------------------------------------------------- 1 | customer_id,created_at,prop,val 2 | 123,2017-01-01,segment,tier 1 3 | 123,2017-01-01,status,active 4 | 123,2017-01-01,is_updated,true 5 | 234,2017-02-01,segment,tier 3 6 | 234,2017-02-01,status,active 7 | 234,2017-02-01,is_updated,false 8 | 567,2017-03-01,status,churned 9 | 567,2017-03-01,is_updated, 10 | 567,2017-03-01,segment,tier 2 11 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_unpivot_original_api_expected.csv: -------------------------------------------------------------------------------- 1 | customer_id,created_at,field_name,value 2 | 123,2017-01-01,status,active 3 | 123,2017-01-01,segment,tier 1 4 | 234,2017-02-01,status,active 5 | 234,2017-02-01,segment,tier 3 6 | 567,2017-03-01,status,churned 7 | 567,2017-03-01,segment,tier 2 8 | 123,2017-01-01,name,name 1 9 | 234,2017-02-01,name,name 3 10 | 567,2017-03-01,name,name 2 -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/hash.sql: -------------------------------------------------------------------------------- 1 | {% macro hash(field) -%} 2 | {{ return(adapter.dispatch('hash', 'dbt_utils') (field)) }} 3 | {%- endmacro %} 4 | 5 | 6 | {% macro default__hash(field) -%} 7 | md5(cast({{field}} as {{dbt_utils.type_string()}})) 8 | {%- endmacro %} 9 | 10 | 11 | {% macro bigquery__hash(field) -%} 12 | to_hex({{dbt_utils.default__hash(field)}}) 13 | {%- endmacro %} 14 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/Makefile: -------------------------------------------------------------------------------- 1 | TARGETS = postgres redshift snowflake bigquery 2 | .PHONY : test test-all $(TARGETS) 3 | 4 | test: export TARGET = $(target) 5 | test: export MODELS = $(models) 6 | test: export SEEDS = $(seeds) 7 | test: 8 | docker-compose -f ../docker-compose.yml up dbt 9 | 10 | $(TARGETS): 11 | $(MAKE) test target=$@ 12 | 13 | test-all: $(TARGETS) 14 | echo "Tested all targets" 15 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/sql/data_surrogate_key.csv: -------------------------------------------------------------------------------- 1 | column_1,column_2,column_3,expected_column_1_only,expected_all_columns 2 | a,b,c,0cc175b9c0f1b6a831c399e269772661,7b193b3d33184464106f41ddf733783b 3 | a,,c,0cc175b9c0f1b6a831c399e269772661,c5fd1b92380c6222ab0ef67839208624 4 | ,,c,d41d8cd98f00b204e9800998ecf8427e,267743defab4558f1940311b66274e26 5 | ,,,d41d8cd98f00b204e9800998ecf8427e,cfab1ba8c67c7c838db98d666f02a132 6 | -------------------------------------------------------------------------------- /profiles/profiles.yml: -------------------------------------------------------------------------------- 1 | demo_getdbt: 2 | outputs: 3 | dev: 4 | dataset: getdbt_demo 5 | fixed_retries: 1 6 | keyfile: /home/gitpod/.dbt/dogwood-day-336418-28272a6c70e7.json 7 | location: US 8 | method: service-account 9 | priority: interactive 10 | project: dogwood-day-336418 11 | threads: 4 12 | timeout_seconds: 300 13 | type: bigquery 14 | target: dev 15 | 16 | 17 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/replace.sql: -------------------------------------------------------------------------------- 1 | {% macro replace(field, old_chars, new_chars) -%} 2 | {{ return(adapter.dispatch('replace', 'dbt_utils') (field, old_chars, new_chars)) }} 3 | {% endmacro %} 4 | 5 | 6 | {% macro default__replace(field, old_chars, new_chars) %} 7 | 8 | replace( 9 | {{ field }}, 10 | {{ old_chars }}, 11 | {{ new_chars }} 12 | ) 13 | 14 | 15 | {% endmacro %} -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/date_part.sql: -------------------------------------------------------------------------------- 1 | {% macro date_part(datepart, date) -%} 2 | {{ adapter.dispatch('date_part', 'dbt_date') (datepart, date) }} 3 | {%- endmacro %} 4 | 5 | {% macro default__date_part(datepart, date) -%} 6 | date_part('{{ datepart }}', {{ date }}) 7 | {%- endmacro %} 8 | 9 | {% macro bigquery__date_part(datepart, date) -%} 10 | extract({{ datepart }} from {{ date }}) 11 | {%- endmacro %} 12 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/web/test_urls.sql: -------------------------------------------------------------------------------- 1 | 2 | with data as ( 3 | 4 | select * from {{ ref('data_urls') }} 5 | 6 | ) 7 | 8 | select 9 | {{ dbt_utils.get_url_parameter('url', 'utm_medium') }} as actual, 10 | medium as expected 11 | 12 | from data 13 | 14 | union all 15 | 16 | select 17 | {{ dbt_utils.get_url_parameter('url', 'utm_source') }} as actual, 18 | source as expected 19 | 20 | from data 21 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/jinja_helpers/slugify.sql: -------------------------------------------------------------------------------- 1 | {% macro slugify(string) %} 2 | 3 | {#- Lower case the string -#} 4 | {% set string = string | lower %} 5 | {#- Replace spaces and dashes with underscores -#} 6 | {% set string = modules.re.sub('[ -]+', '_', string) %} 7 | {#- Only take letters, numbers, and underscores -#} 8 | {% set string = modules.re.sub('[^a-z0-9_]+', '', string) %} 9 | 10 | {{ return(string) }} 11 | 12 | {% endmacro %} 13 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/sql/test_star_prefix_suffix.sql: -------------------------------------------------------------------------------- 1 | {% set prefix_with = 'prefix_' if target.type != 'snowflake' else 'PREFIX_' %} 2 | {% set suffix_with = '_suffix' if target.type != 'snowflake' else '_SUFFIX' %} 3 | 4 | with data as ( 5 | 6 | select 7 | {{ dbt_utils.star(from=ref('data_star'), prefix=prefix_with, suffix=suffix_with) }} 8 | 9 | from {{ ref('data_star') }} 10 | 11 | ) 12 | 13 | select * from data -------------------------------------------------------------------------------- /dbt_packages/dbt_date/dbt_project.yml: -------------------------------------------------------------------------------- 1 | name: 'dbt_date' 2 | version: '0.5.0' 3 | 4 | config-version: 2 5 | 6 | target-path: "target" 7 | clean-targets: ["target", "dbt_packages"] 8 | macro-paths: ["macros"] 9 | log-path: "logs" 10 | 11 | require-dbt-version: [">=1.0.0", "<2.0.0"] 12 | profile: integration_tests 13 | 14 | quoting: 15 | identifier: false 16 | schema: false 17 | 18 | vars: 19 | 'dbt_date:time_zone': 'America/Los_Angeles' 20 | 21 | -------------------------------------------------------------------------------- /models/sources/schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: citybike 5 | database: bigquery-public-data 6 | schema: new_york_citibike 7 | meta: 8 | contains_pii: false 9 | owner: "sgonzalez" 10 | tables: 11 | - name: citibike_stations 12 | columns: 13 | - name: station_id 14 | tests: 15 | - unique 16 | - not_null 17 | 18 | - name: citibike_trips 19 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/cross_db_utils/test_replace.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | 3 | select 4 | 5 | *, 6 | coalesce(search_chars, '') as old_chars, 7 | coalesce(replace_chars, '') as new_chars 8 | 9 | from {{ ref('data_replace') }} 10 | 11 | ) 12 | 13 | select 14 | 15 | {{ dbt_utils.replace('string_text', 'old_chars', 'new_chars') }} as actual, 16 | result as expected 17 | 18 | from data 19 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/math/log_natural.sql: -------------------------------------------------------------------------------- 1 | {% macro log_natural(x) -%} 2 | {{ adapter.dispatch('log_natural', 'dbt_expectations') (x) }} 3 | {% endmacro %} 4 | 5 | {% macro default__log_natural(x) %} 6 | 7 | ln({{ x }}) 8 | 9 | {%- endmacro -%} 10 | 11 | {% macro bigquery__log_natural(x) %} 12 | 13 | ln({{ x }}) 14 | 15 | {%- endmacro -%} 16 | 17 | {% macro snowflake__log_natural(x) %} 18 | 19 | ln({{ x }}) 20 | 21 | {%- endmacro -%} 22 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/length.sql: -------------------------------------------------------------------------------- 1 | {% macro length(expression) -%} 2 | {{ return(adapter.dispatch('length', 'dbt_utils') (expression)) }} 3 | {% endmacro %} 4 | 5 | 6 | {% macro default__length(expression) %} 7 | 8 | length( 9 | {{ expression }} 10 | ) 11 | 12 | {%- endmacro -%} 13 | 14 | 15 | {% macro redshift__length(expression) %} 16 | 17 | len( 18 | {{ expression }} 19 | ) 20 | 21 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/integration_tests/models/schema_tests/emails.sql: -------------------------------------------------------------------------------- 1 | select 2 | 'ab@gmail.com' as email_address, 3 | '@[^.]*' as reg_exp 4 | 5 | union all 6 | 7 | select 8 | 'ab@mail.com' as email_address, 9 | '@[^.]*' as reg_exp 10 | 11 | union all 12 | 13 | select 14 | 'abc@gmail.com' as email_address, 15 | '@[^.]*' as reg_exp 16 | 17 | union all 18 | 19 | select 20 | 'abc.com@gmail.com' as email_address, 21 | '@[^.]*' as reg_exp 22 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/cross_db_utils/test_date_trunc.sql: -------------------------------------------------------------------------------- 1 | 2 | with data as ( 3 | 4 | select * from {{ ref('data_date_trunc') }} 5 | 6 | ) 7 | 8 | select 9 | cast({{dbt_utils.date_trunc('day', 'updated_at') }} as date) as actual, 10 | day as expected 11 | 12 | from data 13 | 14 | union all 15 | 16 | select 17 | cast({{ dbt_utils.date_trunc('month', 'updated_at') }} as date) as actual, 18 | month as expected 19 | 20 | from data 21 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/materializations/test_insert_by_period.sql: -------------------------------------------------------------------------------- 1 | {{ 2 | config( 3 | materialized = 'insert_by_period', 4 | period = 'month', 5 | timestamp_field = 'created_at', 6 | start_date = '2018-01-01', 7 | stop_date = '2018-06-01', 8 | enabled=(target.type == 'redshift') 9 | ) 10 | }} 11 | 12 | with events as ( 13 | select * 14 | from {{ ref('data_insert_by_period') }} 15 | where __PERIOD_FILTER__ 16 | ) 17 | 18 | select * from events 19 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/table_shape/expect_table_row_count_to_equal_other_table_times_factor.sql: -------------------------------------------------------------------------------- 1 | {%- test expect_table_row_count_to_equal_other_table_times_factor(model, compare_model, factor, row_condition=None, compare_row_condition=None) -%} 2 | {{ dbt_expectations.test_expect_table_row_count_to_equal_other_table(model, compare_model, 3 | factor=factor, 4 | row_condition=row_condition, 5 | compare_row_condition=compare_row_condition 6 | ) }} 7 | {%- endtest -%} 8 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/web/data_url_host.csv: -------------------------------------------------------------------------------- 1 | original_url,parsed_url 2 | www.google.co.uk?utm_source=google&utm_medium=cpc&utm_campaign=spring-summer,www.google.co.uk 3 | http://witanddelight.com/2018/01/tips-tricks-how-run-half-marathon-first-time/,witanddelight.com 4 | https://www.nytimes.com/2018/01/01/blog,www.nytimes.com 5 | android-app://m.facebook.com/,m.facebook.com 6 | docs.nytimes.com/2021/01/01/index.js?utm_source=google,docs.nytimes.com 7 | https://m.facebook.com/,m.facebook.com -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/cross_db/data_width_bucket.csv: -------------------------------------------------------------------------------- 1 | date_col,amount,num_buckets,min_value,max_value,bucket 2 | 2012-08-01,190000.00,4,200000.0,600000.0,0 3 | 2013-08-01,290000.00,4,200000.0,600000.0,1 4 | 2014-02-01,320000.00,4,200000.0,600000.0,2 5 | 2015-04-01,399999.99,4,200000.0,600000.0,2 6 | 2016-04-01,400000.00,4,200000.0,600000.0,3 7 | 2017-04-01,470000.00,4,200000.0,600000.0,3 8 | 2018-04-01,510000.00,4,200000.0,600000.0,4 9 | 2019-04-01,610000.00,4,200000.0,600000.0,5 10 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/web/schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: test_urls 5 | tests: 6 | - assert_equal: 7 | actual: actual 8 | expected: expected 9 | 10 | - name: test_url_host 11 | tests: 12 | - assert_equal: 13 | actual: actual 14 | expected: expected 15 | 16 | - name: test_url_path 17 | tests: 18 | - assert_equal: 19 | actual: actual 20 | expected: expected -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/date_trunc.sql: -------------------------------------------------------------------------------- 1 | {% macro date_trunc(datepart, date) -%} 2 | {{ return(adapter.dispatch('date_trunc', 'dbt_utils') (datepart, date)) }} 3 | {%- endmacro %} 4 | 5 | {% macro default__date_trunc(datepart, date) %} 6 | date_trunc('{{datepart}}', {{date}}) 7 | {% endmacro %} 8 | 9 | {% macro bigquery__date_trunc(datepart, date) %} 10 | timestamp_trunc( 11 | cast({{date}} as timestamp), 12 | {{datepart}} 13 | ) 14 | 15 | {% endmacro %} 16 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/sql/nullcheck.sql: -------------------------------------------------------------------------------- 1 | {% macro nullcheck(cols) %} 2 | {{ return(adapter.dispatch('nullcheck', 'dbt_utils')(cols)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__nullcheck(cols) %} 6 | {%- for col in cols %} 7 | 8 | {% if col.is_string() -%} 9 | 10 | nullif({{col.name}},'') as {{col.name}} 11 | 12 | {%- else -%} 13 | 14 | {{col.name}} 15 | 16 | {%- endif -%} 17 | 18 | {%- if not loop.last -%} , {%- endif -%} 19 | 20 | {%- endfor -%} 21 | {% endmacro %} 22 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/sql/test_pivot.sql: -------------------------------------------------------------------------------- 1 | 2 | -- TODO: How do we make this work nicely on Snowflake too? 3 | 4 | {% if target.type == 'snowflake' %} 5 | {% set column_values = ['RED', 'BLUE'] %} 6 | {% set cmp = 'ilike' %} 7 | {% else %} 8 | {% set column_values = ['red', 'blue'] %} 9 | {% set cmp = '=' %} 10 | {% endif %} 11 | 12 | select 13 | size, 14 | {{ dbt_utils.pivot('color', column_values, cmp=cmp) }} 15 | 16 | from {{ ref('data_pivot') }} 17 | group by size 18 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/schema_tests/schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | seeds: 4 | - name: data_test_sequential_values 5 | columns: 6 | - name: my_even_sequence 7 | tests: 8 | - dbt_utils.sequential_values: 9 | interval: 2 10 | 11 | 12 | - name: data_test_sequential_timestamps 13 | columns: 14 | - name: my_timestamp 15 | tests: 16 | - dbt_utils.sequential_values: 17 | interval: 1 18 | datepart: 'hour' 19 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/sql/test_groupby.sql: -------------------------------------------------------------------------------- 1 | with test_data as ( 2 | 3 | select 4 | 5 | {{ dbt_utils.safe_cast("'a'", dbt_utils.type_string() )}} as column_1, 6 | {{ dbt_utils.safe_cast("'b'", dbt_utils.type_string() )}} as column_2 7 | 8 | ), 9 | 10 | grouped as ( 11 | 12 | select 13 | *, 14 | count(*) as total 15 | 16 | from test_data 17 | {{ dbt_utils.group_by(2) }} 18 | 19 | ) 20 | 21 | select * from grouped 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/cast_bool_to_text.sql: -------------------------------------------------------------------------------- 1 | {% macro cast_bool_to_text(field) %} 2 | {{ adapter.dispatch('cast_bool_to_text', 'dbt_utils') (field) }} 3 | {% endmacro %} 4 | 5 | 6 | {% macro default__cast_bool_to_text(field) %} 7 | cast({{ field }} as {{ dbt_utils.type_string() }}) 8 | {% endmacro %} 9 | 10 | {% macro redshift__cast_bool_to_text(field) %} 11 | case 12 | when {{ field }} is true then 'true' 13 | when {{ field }} is false then 'false' 14 | end::text 15 | {% endmacro %} 16 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/web/get_url_parameter.sql: -------------------------------------------------------------------------------- 1 | {% macro get_url_parameter(field, url_parameter) -%} 2 | {{ return(adapter.dispatch('get_url_parameter', 'dbt_utils')(field, url_parameter)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_url_parameter(field, url_parameter) -%} 6 | 7 | {%- set formatted_url_parameter = "'" + url_parameter + "='" -%} 8 | 9 | {%- set split = dbt_utils.split_part(dbt_utils.split_part(field, formatted_url_parameter, 2), "'&'", 1) -%} 10 | 11 | nullif({{ split }},'') 12 | 13 | {%- endmacro %} 14 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/web/data_url_path.csv: -------------------------------------------------------------------------------- 1 | original_url,parsed_path 2 | www.google.co.uk?utm_source=google&utm_medium=cpc&utm_campaign=spring-summer, 3 | http://witanddelight.com/2018/01/tips-tricks-how-run-half-marathon-first-time/,2018/01/tips-tricks-how-run-half-marathon-first-time/ 4 | https://www.nytimes.com/2018/01/01/blog,2018/01/01/blog 5 | http://witanddelight.com/2018/01/tips-tricks-how-run-half-marathon-first-time/?utm_source=google&utm_medium=cpc&utm_campaign=spring-summer,2018/01/tips-tricks-how-run-half-marathon-first-time/ -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/sql/test_surrogate_key.sql: -------------------------------------------------------------------------------- 1 | 2 | with data as ( 3 | 4 | select * from {{ ref('data_surrogate_key') }} 5 | 6 | ) 7 | 8 | select 9 | {{ dbt_utils.surrogate_key('column_1') }} as actual_column_1_only, 10 | expected_column_1_only, 11 | {{ dbt_utils.surrogate_key('column_1', 'column_2', 'column_3') }} as actual_all_columns_arguments, 12 | {{ dbt_utils.surrogate_key(['column_1', 'column_2', 'column_3']) }} as actual_all_columns_list, 13 | expected_all_columns 14 | 15 | from data 16 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/table_shape/expect_table_column_count_to_equal.sql: -------------------------------------------------------------------------------- 1 | {%- test expect_table_column_count_to_equal(model, value) -%} 2 | {%- if execute -%} 3 | {%- set number_actual_columns = (adapter.get_columns_in_relation(model) | length) -%} 4 | with test_data as ( 5 | 6 | select 7 | {{ number_actual_columns }} as number_actual_columns, 8 | {{ value }} as value 9 | 10 | ) 11 | select * 12 | from test_data 13 | where 14 | number_actual_columns != value 15 | {%- endif -%} 16 | {%- endtest -%} 17 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/table_shape/expect_table_row_count_to_equal_other_table.sql: -------------------------------------------------------------------------------- 1 | {%- test expect_table_row_count_to_equal_other_table(model, compare_model, factor=1, row_condition=None, compare_row_condition=None) -%} 2 | {{ dbt_expectations.test_equal_expression(model, "count(*)", 3 | compare_model=compare_model, 4 | compare_expression="count(*) * " + factor|string, 5 | row_condition=row_condition, 6 | compare_row_condition=compare_row_condition, 7 | return_difference=True 8 | ) }} 9 | {%- endtest -%} 10 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/cross_db_utils/test_last_day.sql: -------------------------------------------------------------------------------- 1 | 2 | with data as ( 3 | 4 | select * from {{ ref('data_last_day') }} 5 | 6 | ) 7 | 8 | select 9 | case 10 | when date_part = 'month' then {{ dbt_utils.last_day('date_day', 'month') }} 11 | when date_part = 'quarter' then {{ dbt_utils.last_day('date_day', 'quarter') }} 12 | when date_part = 'year' then {{ dbt_utils.last_day('date_day', 'year') }} 13 | else null 14 | end as actual, 15 | result as expected 16 | 17 | from data 18 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/sql/nullcheck_table.sql: -------------------------------------------------------------------------------- 1 | {% macro nullcheck_table(relation) %} 2 | {{ return(adapter.dispatch('nullcheck_table', 'dbt_utils')(relation)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__nullcheck_table(relation) %} 6 | 7 | {%- do dbt_utils._is_relation(relation, 'nullcheck_table') -%} 8 | {%- do dbt_utils._is_ephemeral(relation, 'nullcheck_table') -%} 9 | {% set cols = adapter.get_columns_in_relation(relation) %} 10 | 11 | select {{ dbt_utils.nullcheck(cols) }} 12 | from {{relation}} 13 | 14 | {% endmacro %} 15 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/day_of_year.sql: -------------------------------------------------------------------------------- 1 | {%- macro day_of_year(date) -%} 2 | {{ adapter.dispatch('day_of_year', 'dbt_date') (date) }} 3 | {%- endmacro %} 4 | 5 | {%- macro default__day_of_year(date) -%} 6 | {{ dbt_date.date_part('dayofyear', date) }} 7 | {%- endmacro %} 8 | 9 | {%- macro postgres__day_of_year(date) -%} 10 | {{ dbt_date.date_part('doy', date) }} 11 | {%- endmacro %} 12 | 13 | {%- macro redshift__day_of_year(date) -%} 14 | cast({{ dbt_date.date_part('dayofyear', date) }} as {{ dbt_utils.type_bigint() }}) 15 | {%- endmacro %} 16 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/sql/test_generate_series.sql: -------------------------------------------------------------------------------- 1 | 2 | -- snowflake doesn't like this as a view because the `generate_series` 3 | -- call creates a CTE called `unioned`, as does the `equality` schema test. 4 | -- Ideally, Snowflake would be smart enough to know that these CTE names are 5 | -- different, as they live in different relations. TODO: use a less common cte name 6 | 7 | {{ config(materialized='table') }} 8 | 9 | with data as ( 10 | 11 | {{ dbt_utils.generate_series(10) }} 12 | 13 | ) 14 | 15 | select generated_number from data 16 | -------------------------------------------------------------------------------- /models/staging/schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: citibike_trips_2018 5 | description: "Citibike trips only for 2018" 6 | columns: 7 | - name: birth_year 8 | tests: 9 | - not_null 10 | - name: start_station_id 11 | tests: 12 | - relationships: 13 | to: source('citybike','citibike_stations') 14 | field: station_id 15 | - name: citibike_trips_2013 16 | description: "Citibike trips for 2013" 17 | columns: 18 | - name: birth_year 19 | tests: 20 | - not_null -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/to_unixtimestamp.sql: -------------------------------------------------------------------------------- 1 | {%- macro to_unixtimestamp(timestamp) -%} 2 | {{ adapter.dispatch('to_unixtimestamp', 'dbt_date') (timestamp) }} 3 | {%- endmacro %} 4 | 5 | {%- macro default__to_unixtimestamp(timestamp) -%} 6 | {{ dbt_date.date_part('epoch', timestamp) }} 7 | {%- endmacro %} 8 | 9 | {%- macro snowflake__to_unixtimestamp(timestamp) -%} 10 | {{ dbt_date.date_part('epoch_seconds', timestamp) }} 11 | {%- endmacro %} 12 | 13 | {%- macro bigquery__to_unixtimestamp(timestamp) -%} 14 | unix_seconds({{ timestamp }}) 15 | {%- endmacro %} 16 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/schema_tests/not_constant.sql: -------------------------------------------------------------------------------- 1 | 2 | {% test not_constant(model, column_name) %} 3 | {{ return(adapter.dispatch('test_not_constant', 'dbt_utils')(model, column_name)) }} 4 | {% endtest %} 5 | 6 | {% macro default__test_not_constant(model, column_name) %} 7 | 8 | 9 | select 10 | {# In TSQL, subquery aggregate columns need aliases #} 11 | {# thus: a filler col name, 'filler_column' #} 12 | count(distinct {{ column_name }}) as filler_column 13 | 14 | from {{ model }} 15 | 16 | having count(distinct {{ column_name }}) = 1 17 | 18 | 19 | {% endmacro %} 20 | -------------------------------------------------------------------------------- /models/staging/citibike_trips_macro_get_columns.sql: -------------------------------------------------------------------------------- 1 | 2 | {% set usertypes = dbt_utils.get_column_values(table=ref('citibike_trips_2018'), column='usertype') %} 3 | --{# set usertypes = ['asd','foo','bla','bar'] #} 4 | 5 | with 6 | {% for usertype in usertypes %} 7 | table_{{usertype}} as ( 8 | select count(1) as count_{{usertype}} from {{ref('citibike_trips_2018')}} where usertype = '{{usertype}}' 9 | ){% if not loop.last%} ,{% endif %} 10 | {% endfor %} 11 | 12 | select * from {% for usertype in usertypes %} 13 | table_{{usertype}}{% if not loop.last%} ,{% endif %} 14 | {% endfor %} -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/column_values_basic/expect_column_values_to_be_null.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_values_to_be_null(model, column_name, row_condition=None) %} 2 | 3 | {% set expression = column_name ~ " is null" %} 4 | 5 | {{ dbt_expectations.expression_is_true(model, 6 | expression=expression, 7 | group_by_columns=None, 8 | row_condition=row_condition 9 | ) 10 | }} 11 | {% endtest %} 12 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/cross_db_utils/test_split_part.sql: -------------------------------------------------------------------------------- 1 | 2 | with data as ( 3 | 4 | select * from {{ ref('data_split_part') }} 5 | 6 | ) 7 | 8 | select 9 | {{ dbt_utils.split_part('parts', 'split_on', 1) }} as actual, 10 | result_1 as expected 11 | 12 | from data 13 | 14 | union all 15 | 16 | select 17 | {{ dbt_utils.split_part('parts', 'split_on', 2) }} as actual, 18 | result_2 as expected 19 | 20 | from data 21 | 22 | union all 23 | 24 | select 25 | {{ dbt_utils.split_part('parts', 'split_on', 3) }} as actual, 26 | result_3 as expected 27 | 28 | from data 29 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/math/rand.sql: -------------------------------------------------------------------------------- 1 | {% macro rand() -%} 2 | {{ adapter.dispatch('rand', 'dbt_expectations') () }} 3 | {% endmacro %} 4 | 5 | {% macro default__rand() %} 6 | 7 | rand() 8 | 9 | {%- endmacro -%} 10 | 11 | {% macro bigquery__rand() %} 12 | 13 | rand() 14 | 15 | {%- endmacro -%} 16 | 17 | {% macro snowflake__rand(seed) %} 18 | 19 | uniform(0::float, 1::float, random()) 20 | 21 | {%- endmacro -%} 22 | 23 | {% macro postgres__rand() %} 24 | 25 | random() 26 | 27 | {%- endmacro -%} 28 | 29 | {% macro redshift__rand() %} 30 | 31 | random() 32 | 33 | {%- endmacro -%} 34 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/column_values_basic/expect_column_values_to_not_be_null.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_values_to_not_be_null(model, column_name, row_condition=None) %} 2 | 3 | {% set expression = column_name ~ " is not null" %} 4 | 5 | {{ dbt_expectations.expression_is_true(model, 6 | expression=expression, 7 | group_by_columns=None, 8 | row_condition=row_condition 9 | ) 10 | }} 11 | {% endtest %} 12 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/position.sql: -------------------------------------------------------------------------------- 1 | {% macro position(substring_text, string_text) -%} 2 | {{ return(adapter.dispatch('position', 'dbt_utils') (substring_text, string_text)) }} 3 | {% endmacro %} 4 | 5 | 6 | {% macro default__position(substring_text, string_text) %} 7 | 8 | position( 9 | {{ substring_text }} in {{ string_text }} 10 | ) 11 | 12 | {%- endmacro -%} 13 | 14 | {% macro bigquery__position(substring_text, string_text) %} 15 | 16 | strpos( 17 | {{ string_text }}, 18 | {{ substring_text }} 19 | 20 | ) 21 | 22 | {%- endmacro -%} 23 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/schema_tests/test_unique_where.sql: -------------------------------------------------------------------------------- 1 | {% test unique_where(model, column_name) %} 2 | {%- set deprecation_warning = ' 3 | Warning: `dbt_utils.unique_where` is no longer supported. 4 | Starting in dbt v0.20.0, the built-in `unique` test supports a `where` config. 5 | ' -%} 6 | {%- do exceptions.warn(deprecation_warning) -%} 7 | {{ return(adapter.dispatch('test_unique_where', 'dbt_utils')(model, column_name)) }} 8 | {% endtest %} 9 | 10 | {% macro default__test_unique_where(model, column_name) %} 11 | {{ return(test_unique(model, column_name)) }} 12 | {% endmacro %} 13 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/dbt_project.yml: -------------------------------------------------------------------------------- 1 | 2 | # Name your project! Project names should contain only lowercase characters 3 | # and underscores. A good package name should reflect your organization's 4 | # name or the intended use of these models 5 | name: 'dbt_expectations' 6 | version: '0.5.0' 7 | 8 | require-dbt-version: [">=1.0.0", "<2.0.0"] 9 | config-version: 2 10 | 11 | target-path: "target" 12 | clean-targets: ["target", "dbt_packages"] 13 | macro-paths: ["macros"] 14 | log-path: "logs" 15 | 16 | dispatch: 17 | - macro_namespace: dbt_utils 18 | search_order: ['dbt_expectations', 'dbt_utils'] # enable override 19 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/sql/get_tables_by_prefix_sql.sql: -------------------------------------------------------------------------------- 1 | {% macro get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} 2 | {{ return(adapter.dispatch('get_tables_by_prefix_sql', 'dbt_utils')(schema, prefix, exclude, database)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} 6 | 7 | {{ dbt_utils.get_tables_by_pattern_sql( 8 | schema_pattern = schema, 9 | table_pattern = prefix ~ '%', 10 | exclude = exclude, 11 | database = database 12 | ) }} 13 | 14 | {% endmacro %} 15 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/schema_tests/test_not_null_where.sql: -------------------------------------------------------------------------------- 1 | {% test not_null_where(model, column_name) %} 2 | {%- set deprecation_warning = ' 3 | Warning: `dbt_utils.not_null_where` is no longer supported. 4 | Starting in dbt v0.20.0, the built-in `not_null` test supports a `where` config. 5 | ' -%} 6 | {%- do exceptions.warn(deprecation_warning) -%} 7 | {{ return(adapter.dispatch('test_not_null_where', 'dbt_utils')(model, column_name)) }} 8 | {% endtest %} 9 | 10 | {% macro default__test_not_null_where(model, column_name) %} 11 | {{ return(test_not_null(model, column_name)) }} 12 | {% endmacro %} 13 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/safe_cast.sql: -------------------------------------------------------------------------------- 1 | {% macro safe_cast(field, type) %} 2 | {{ return(adapter.dispatch('safe_cast', 'dbt_utils') (field, type)) }} 3 | {% endmacro %} 4 | 5 | 6 | {% macro default__safe_cast(field, type) %} 7 | {# most databases don't support this function yet 8 | so we just need to use cast #} 9 | cast({{field}} as {{type}}) 10 | {% endmacro %} 11 | 12 | 13 | {% macro snowflake__safe_cast(field, type) %} 14 | try_cast({{field}} as {{type}}) 15 | {% endmacro %} 16 | 17 | 18 | {% macro bigquery__safe_cast(field, type) %} 19 | safe_cast({{field}} as {{type}}) 20 | {% endmacro %} 21 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/schema_tests/at_least_one.sql: -------------------------------------------------------------------------------- 1 | {% test at_least_one(model, column_name) %} 2 | {{ return(adapter.dispatch('test_at_least_one', 'dbt_utils')(model, column_name)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_at_least_one(model, column_name) %} 6 | 7 | select * 8 | from ( 9 | select 10 | {# In TSQL, subquery aggregate columns need aliases #} 11 | {# thus: a filler col name, 'filler_column' #} 12 | count({{ column_name }}) as filler_column 13 | 14 | from {{ model }} 15 | 16 | having count({{ column_name }}) = 0 17 | 18 | ) validation_errors 19 | 20 | {% endmacro %} 21 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/week_end.sql: -------------------------------------------------------------------------------- 1 | {%- macro week_end(date=None, tz=None) -%} 2 | {%-set dt = date if date else dbt_date.today(tz) -%} 3 | {{ adapter.dispatch('week_end', 'dbt_date') (dt) }} 4 | {%- endmacro -%} 5 | 6 | {%- macro default__week_end(date) -%} 7 | {{ dbt_utils.last_day(date, 'week') }} 8 | {%- endmacro %} 9 | 10 | {%- macro snowflake__week_end(date) -%} 11 | {%- set dt = dbt_date.week_start(date) -%} 12 | {{ dbt_date.n_days_away(6, dt) }} 13 | {%- endmacro %} 14 | 15 | {%- macro postgres__week_end(date) -%} 16 | {%- set dt = dbt_date.week_start(date) -%} 17 | {{ dbt_date.n_days_away(6, dt) }} 18 | {%- endmacro %} 19 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/integration_tests/models/dim_date.sql: -------------------------------------------------------------------------------- 1 | {{ 2 | config( 3 | materialized = "table" 4 | ) 5 | }} 6 | with date_dimension as ( 7 | select * from {{ ref('dates') }} 8 | ), 9 | fiscal_periods as ( 10 | {{ dbt_date.get_fiscal_periods(ref('dates'), year_end_month=1, week_start_day=1, shift_year=1) }} 11 | ) 12 | select 13 | d.*, 14 | f.fiscal_week_of_year, 15 | f.fiscal_week_of_period, 16 | f.fiscal_period_number, 17 | f.fiscal_quarter_number, 18 | f.fiscal_period_of_quarter 19 | from 20 | date_dimension d 21 | left join 22 | fiscal_periods f 23 | on d.date_day = f.date_day 24 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/sql/safe_add.sql: -------------------------------------------------------------------------------- 1 | {%- macro safe_add() -%} 2 | {# needed for safe_add to allow for non-keyword arguments see SO post #} 3 | {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} 4 | {% set frustrating_jinja_feature = varargs %} 5 | {{ return(adapter.dispatch('safe_add', 'dbt_utils')(*varargs)) }} 6 | {% endmacro %} 7 | 8 | {%- macro default__safe_add() -%} 9 | 10 | {% set fields = [] %} 11 | 12 | {%- for field in varargs -%} 13 | 14 | {% do fields.append("coalesce(" ~ field ~ ", 0)") %} 15 | 16 | {%- endfor -%} 17 | 18 | {{ fields|join(' +\n ') }} 19 | 20 | {%- endmacro -%} 21 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/iso_week_end.sql: -------------------------------------------------------------------------------- 1 | {%- macro iso_week_end(date=None, tz=None) -%} 2 | {%-set dt = date if date else dbt_date.today(tz) -%} 3 | {{ adapter.dispatch('iso_week_end', 'dbt_date') (dt) }} 4 | {%- endmacro -%} 5 | 6 | {%- macro _iso_week_end(date, week_type) -%} 7 | {%- set dt = dbt_date.iso_week_start(date) -%} 8 | {{ dbt_date.n_days_away(6, dt) }} 9 | {%- endmacro %} 10 | 11 | {%- macro default__iso_week_end(date) -%} 12 | {{ dbt_date._iso_week_end(date, 'isoweek') }} 13 | {%- endmacro %} 14 | 15 | {%- macro snowflake__iso_week_end(date) -%} 16 | {{ dbt_date._iso_week_end(date, 'weekiso') }} 17 | {%- endmacro %} 18 | 19 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/week_of_year.sql: -------------------------------------------------------------------------------- 1 | {%- macro week_of_year(date=None, tz=None) -%} 2 | {%-set dt = date if date else dbt_date.today(tz) -%} 3 | {{ adapter.dispatch('week_of_year', 'dbt_date') (dt) }} 4 | {%- endmacro -%} 5 | 6 | {%- macro default__week_of_year(date) -%} 7 | cast({{ dbt_date.date_part('week', date) }} as {{ dbt_utils.type_int() }}) 8 | {%- endmacro %} 9 | 10 | {%- macro postgres__week_of_year(date) -%} 11 | {# postgresql 'week' returns isoweek. Use to_char instead. 12 | WW = the first week starts on the first day of the year #} 13 | cast(to_char({{ date }}, 'WW') as {{ dbt_utils.type_int() }}) 14 | {%- endmacro %} 15 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/utils/datatypes.sql: -------------------------------------------------------------------------------- 1 | {% macro postgres__type_timestamp() -%} 2 | timestamp without time zone 3 | {%- endmacro %} 4 | 5 | 6 | 7 | {%- macro type_datetime() -%} 8 | {{ return(adapter.dispatch('type_datetime', 'dbt_expectations')()) }} 9 | {%- endmacro -%} 10 | 11 | {% macro default__type_datetime() -%} 12 | datetime 13 | {%- endmacro %} 14 | 15 | {# see: https://docs.snowflake.com/en/sql-reference/data-types-datetime.html#datetime #} 16 | {% macro snowflake__type_datetime() -%} 17 | timestamp_ntz 18 | {%- endmacro %} 19 | 20 | {% macro postgres__type_datetime() -%} 21 | timestamp without time zone 22 | {%- endmacro %} 23 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/math/percentile_cont.sql: -------------------------------------------------------------------------------- 1 | {% macro percentile_cont(field, quantile, partition=None) %} 2 | {{ adapter.dispatch('quantile', 'dbt_expectations') (field, quantile, partition) }} 3 | {% endmacro %} 4 | 5 | {% macro default__quantile(field, quantile, partition) -%} 6 | percentile_cont({{ quantile }}) within group (order by {{ field }}) 7 | {%- if partition %}over(partition by {{ partition }}){% endif -%} 8 | {%- endmacro %} 9 | 10 | {% macro bigquery__quantile(field, quantile, partition) %} 11 | percentile_cont({{ field }}, {{ quantile }}) 12 | over({%- if partition %}partition by {{ partition }}{% endif -%}) 13 | {% endmacro %} 14 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/integration_tests/models/schema_tests/timeseries_data.sql: -------------------------------------------------------------------------------- 1 | with dates as ( 2 | 3 | select * from {{ ref('timeseries_base') }} 4 | 5 | ), 6 | add_row_values as ( 7 | 8 | select 9 | cast(d.date_day as {{ dbt_expectations.type_datetime() }}) as date_day, 10 | cast(abs({{ dbt_expectations.rand() }}) as {{ dbt_utils.type_float() }}) as row_value 11 | from 12 | dates d 13 | 14 | ), 15 | add_logs as ( 16 | 17 | select 18 | *, 19 | {{ dbt_expectations.log_natural('nullif(row_value, 0)') }} as row_value_log 20 | from 21 | add_row_values 22 | ) 23 | select 24 | * 25 | from 26 | add_logs 27 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/data/cross_db/data_datediff.csv: -------------------------------------------------------------------------------- 1 | first_date,second_date,datepart,result 2 | 2018-01-01 01:00:00,2018-01-02 01:00:00,day,1 3 | 2018-01-01 01:00:00,2018-02-01 01:00:00,month,1 4 | 2018-01-01 01:00:00,2019-01-01 01:00:00,year,1 5 | 2018-01-01 01:00:00,2018-01-01 02:00:00,hour,1 6 | 2018-01-01 01:00:00,2018-01-01 02:01:00,minute,61 7 | 2018-01-01 01:00:00,2018-01-01 02:00:01,second,3601 8 | 2019-12-31 00:00:00,2019-12-27 00:00:00,week,-1 9 | 2019-12-31 00:00:00,2019-12-30 00:00:00,week,0 10 | 2019-12-31 00:00:00,2020-01-02 00:00:00,week,0 11 | 2019-12-31 00:00:00,2020-01-06 02:00:00,week,1 12 | ,2018-01-01 02:00:00,hour, 13 | 2018-01-01 02:00:00,,hour, 14 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/schema_tests/recency.sql: -------------------------------------------------------------------------------- 1 | {% test recency(model, field, datepart, interval) %} 2 | {{ return(adapter.dispatch('test_recency', 'dbt_utils')(model, field, datepart, interval)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_recency(model, field, datepart, interval) %} 6 | 7 | {% set threshold = dbt_utils.dateadd(datepart, interval * -1, dbt_utils.current_timestamp()) %} 8 | 9 | with recency as ( 10 | 11 | select max({{field}}) as most_recent 12 | from {{ model }} 13 | 14 | ) 15 | 16 | select 17 | 18 | most_recent, 19 | {{ threshold }} as threshold 20 | 21 | from recency 22 | where most_recent < {{ threshold }} 23 | 24 | {% endmacro %} 25 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/table_shape/expect_table_column_count_to_equal_other_table.sql: -------------------------------------------------------------------------------- 1 | {%- test expect_table_column_count_to_equal_other_table(model, compare_model) -%} 2 | {%- if execute -%} 3 | {%- set number_columns = (adapter.get_columns_in_relation(model) | length) -%} 4 | {%- set compare_number_columns = (adapter.get_columns_in_relation(compare_model) | length) -%} 5 | with test_data as ( 6 | 7 | select 8 | {{ number_columns }} as number_columns, 9 | {{ compare_number_columns }} as compare_number_columns 10 | 11 | ) 12 | select * 13 | from test_data 14 | where 15 | number_columns != compare_number_columns 16 | {%- endif -%} 17 | {%- endtest -%} 18 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/geo/test_haversine_distance_km.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | select * from {{ ref('data_haversine_km') }} 3 | ), 4 | final as ( 5 | select 6 | output as expected, 7 | cast( 8 | {{ 9 | dbt_utils.haversine_distance( 10 | lat1='lat_1', 11 | lon1='lon_1', 12 | lat2='lat_2', 13 | lon2='lon_2', 14 | unit='km' 15 | ) 16 | }} as {{ dbt_utils.type_numeric() }} 17 | ) as actual 18 | from data 19 | ) 20 | select 21 | expected, 22 | round(actual,0) as actual 23 | from final 24 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/identifier.sql: -------------------------------------------------------------------------------- 1 | {% macro identifier(value) %} 2 | {%- set error_message = ' 3 | Warning: the `identifier` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ 4 | Use `adapter.quote` instead. The {}.{} model triggered this warning. \ 5 | '.format(model.package_name, model.name) -%} 6 | {%- do exceptions.warn(error_message) -%} 7 | {{ return(adapter.dispatch('identifier', 'dbt_utils') (value)) }} 8 | {% endmacro %} 9 | 10 | {% macro default__identifier(value) -%} 11 | "{{ value }}" 12 | {%- endmacro %} 13 | 14 | {% macro bigquery__identifier(value) -%} 15 | `{{ value }}` 16 | {%- endmacro %} 17 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/split_part.sql: -------------------------------------------------------------------------------- 1 | {% macro split_part(string_text, delimiter_text, part_number) %} 2 | {{ return(adapter.dispatch('split_part', 'dbt_utils') (string_text, delimiter_text, part_number)) }} 3 | {% endmacro %} 4 | 5 | 6 | {% macro default__split_part(string_text, delimiter_text, part_number) %} 7 | 8 | split_part( 9 | {{ string_text }}, 10 | {{ delimiter_text }}, 11 | {{ part_number }} 12 | ) 13 | 14 | {% endmacro %} 15 | 16 | 17 | {% macro bigquery__split_part(string_text, delimiter_text, part_number) %} 18 | 19 | split( 20 | {{ string_text }}, 21 | {{ delimiter_text }} 22 | )[safe_offset({{ part_number - 1 }})] 23 | 24 | {% endmacro %} 25 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/iso_week_start.sql: -------------------------------------------------------------------------------- 1 | {%- macro iso_week_start(date=None, tz=None) -%} 2 | {%-set dt = date if date else dbt_date.today(tz) -%} 3 | {{ adapter.dispatch('iso_week_start', 'dbt_date') (dt) }} 4 | {%- endmacro -%} 5 | 6 | {%- macro _iso_week_start(date, week_type) -%} 7 | cast({{ dbt_utils.date_trunc(week_type, date) }} as date) 8 | {%- endmacro %} 9 | 10 | {%- macro default__iso_week_start(date) -%} 11 | {{ dbt_date._iso_week_start(date, 'isoweek') }} 12 | {%- endmacro %} 13 | 14 | {%- macro snowflake__iso_week_start(date) -%} 15 | {{ dbt_date._iso_week_start(date, 'week') }} 16 | {%- endmacro %} 17 | 18 | {%- macro postgres__iso_week_start(date) -%} 19 | {{ dbt_date._iso_week_start(date, 'week') }} 20 | {%- endmacro %} 21 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/web/get_url_host.sql: -------------------------------------------------------------------------------- 1 | {% macro get_url_host(field) -%} 2 | {{ return(adapter.dispatch('get_url_host', 'dbt_utils')(field)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_url_host(field) -%} 6 | 7 | {%- set parsed = 8 | dbt_utils.split_part( 9 | dbt_utils.split_part( 10 | dbt_utils.replace( 11 | dbt_utils.replace( 12 | dbt_utils.replace(field, "'android-app://'", "''" 13 | ), "'http://'", "''" 14 | ), "'https://'", "''" 15 | ), "'/'", 1 16 | ), "'?'", 1 17 | ) 18 | 19 | -%} 20 | 21 | 22 | {{ dbt_utils.safe_cast( 23 | parsed, 24 | dbt_utils.type_string() 25 | )}} 26 | 27 | {%- endmacro %} 28 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/_is_ephemeral.sql: -------------------------------------------------------------------------------- 1 | {% macro _is_ephemeral(obj, macro) %} 2 | {%- if obj.is_cte -%} 3 | {% set ephemeral_prefix = api.Relation.add_ephemeral_prefix('') %} 4 | {% if obj.name.startswith(ephemeral_prefix) %} 5 | {% set model_name = obj.name[(ephemeral_prefix|length):] %} 6 | {% else %} 7 | {% set model_name = obj.name %} 8 | {%- endif -%} 9 | {% set error_message %} 10 | The `{{ macro }}` macro cannot be used with ephemeral models, as it relies on the information schema. 11 | 12 | `{{ model_name }}` is an ephemeral model. Consider making it a view or table instead. 13 | {% endset %} 14 | {%- do exceptions.raise_compiler_error(error_message) -%} 15 | {%- endif -%} 16 | {% endmacro %} 17 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/integration_tests/models/schema_tests/timeseries_data_extended.sql: -------------------------------------------------------------------------------- 1 | with dates as ( 2 | 3 | select * from {{ ref('timeseries_base') }} 4 | 5 | ), 6 | row_values as ( 7 | select * from {{ ref('series_10') }} 8 | ), 9 | add_row_values as ( 10 | 11 | select 12 | cast(d.date_day as {{ dbt_expectations.type_datetime() }}) as date_day, 13 | cast(abs({{ dbt_expectations.rand() }}) as {{ dbt_utils.type_float() }}) as row_value 14 | from 15 | dates d 16 | cross join 17 | row_values r 18 | 19 | ), 20 | add_logs as ( 21 | 22 | select 23 | *, 24 | {{ dbt_expectations.log_natural('nullif(row_value, 0)') }} as row_value_log 25 | from 26 | add_row_values 27 | ) 28 | select 29 | * 30 | from 31 | add_logs 32 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/string_matching/expect_column_value_lengths_to_equal.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_value_lengths_to_equal(model, column_name, 2 | value, 3 | row_condition=None 4 | ) %} 5 | 6 | {% set expression = dbt_utils.length(column_name) ~ " = " ~ value %} 7 | 8 | {{ dbt_expectations.expression_is_true(model, 9 | expression=expression, 10 | group_by_columns=None, 11 | row_condition=row_condition 12 | ) 13 | }} 14 | 15 | {% endtest %} 16 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/integration_tests/models/schema_tests/timeseries_hourly_data_extended.sql: -------------------------------------------------------------------------------- 1 | with dates as ( 2 | 3 | select * from {{ ref('timeseries_hourly') }} 4 | 5 | ), 6 | row_values as ( 7 | select * from {{ ref('series_10') }} 8 | ), 9 | add_row_values as ( 10 | 11 | select 12 | cast(d.date_hour as {{ dbt_expectations.type_datetime() }}) as date_hour, 13 | cast(abs({{ dbt_expectations.rand() }}) as {{ dbt_utils.type_float() }}) as row_value 14 | from 15 | dates d 16 | cross join 17 | row_values r 18 | 19 | ), 20 | add_logs as ( 21 | 22 | select 23 | *, 24 | {{ dbt_expectations.log_natural('nullif(row_value, 0)') }} as row_value_log 25 | from 26 | add_row_values 27 | ) 28 | select 29 | * 30 | from 31 | add_logs 32 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/schema_tests/expression_is_true.sql: -------------------------------------------------------------------------------- 1 | {% test expression_is_true(model, expression, column_name=None, condition='1=1') %} 2 | {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} 3 | {# ref https://stackoverflow.com/a/7170753/3842610 #} 4 | {{ return(adapter.dispatch('test_expression_is_true', 'dbt_utils')(model, expression, column_name, condition)) }} 5 | {% endtest %} 6 | 7 | {% macro default__test_expression_is_true(model, expression, column_name, condition) %} 8 | 9 | with meet_condition as ( 10 | select * from {{ model }} where {{ condition }} 11 | ) 12 | 13 | select 14 | * 15 | from meet_condition 16 | {% if column_name is none %} 17 | where not({{ expression }}) 18 | {%- else %} 19 | where not({{ column_name }} {{ expression }}) 20 | {%- endif %} 21 | 22 | {% endmacro %} 23 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/string_matching/expect_column_values_to_match_regex.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_values_to_match_regex(model, column_name, 2 | regex, 3 | row_condition=None 4 | ) %} 5 | 6 | {% set expression %} 7 | {{ dbt_expectations.regexp_instr(column_name, regex) }} > 0 8 | {% endset %} 9 | 10 | {{ dbt_expectations.expression_is_true(model, 11 | expression=expression, 12 | group_by_columns=None, 13 | row_condition=row_condition 14 | ) 15 | }} 16 | 17 | {% endtest %} 18 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/cross_db_utils/test_dateadd.sql: -------------------------------------------------------------------------------- 1 | 2 | with data as ( 3 | 4 | select * from {{ ref('data_dateadd') }} 5 | 6 | ) 7 | 8 | select 9 | case 10 | when datepart = 'hour' then cast({{ dbt_utils.dateadd('hour', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) 11 | when datepart = 'day' then cast({{ dbt_utils.dateadd('day', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) 12 | when datepart = 'month' then cast({{ dbt_utils.dateadd('month', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) 13 | when datepart = 'year' then cast({{ dbt_utils.dateadd('year', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) 14 | else null 15 | end as actual, 16 | result as expected 17 | 18 | from data 19 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/run_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | VENV="venv/bin/activate" 3 | 4 | if [[ ! -f $VENV ]]; then 5 | python3 -m venv venv 6 | . $VENV 7 | 8 | pip install --upgrade pip setuptools 9 | pip install --pre "dbt-$1" 10 | fi 11 | 12 | . $VENV 13 | cd integration_tests 14 | 15 | if [[ ! -e ~/.dbt/profiles.yml ]]; then 16 | mkdir -p ~/.dbt 17 | cp ci/sample.profiles.yml ~/.dbt/profiles.yml 18 | fi 19 | 20 | _models="" 21 | _seeds="--full-refresh" 22 | if [[ ! -z $2 ]]; then _models="--models $2"; fi 23 | if [[ ! -z $3 ]]; then _seeds="--select $3 --full-refresh"; fi 24 | 25 | dbt deps --target $1 26 | dbt seed --target $1 $_seeds 27 | if [ $1 == 'redshift' ]; then 28 | dbt run -x -m test_insert_by_period --full-refresh --target redshift 29 | fi 30 | dbt run -x --target $1 $_models 31 | dbt test -x --target $1 $_models 32 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/string_matching/expect_column_values_to_not_match_regex.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_values_to_not_match_regex(model, column_name, 2 | regex, 3 | row_condition=None 4 | ) %} 5 | 6 | {% set expression %} 7 | {{ dbt_expectations.regexp_instr(column_name, regex) }} = 0 8 | {% endset %} 9 | 10 | {{ dbt_expectations.expression_is_true(model, 11 | expression=expression, 12 | group_by_columns=None, 13 | row_condition=row_condition 14 | ) 15 | }} 16 | 17 | {% endtest %} 18 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/integration_tests/dbt_project.yml: -------------------------------------------------------------------------------- 1 | name: "dbt_date_integration_tests" 2 | version: "1.0" 3 | 4 | profile: "integration_tests" 5 | 6 | config-version: 2 7 | 8 | model-paths: ["models"] 9 | test-paths: ["tests"] 10 | seed-paths: ["data"] 11 | macro-paths: ["macros"] 12 | 13 | target-path: "target" 14 | clean-targets: ["target", "dbt_modules", "dbt_packages"] 15 | 16 | dispatch: 17 | - macro_namespace: dbt_date 18 | search_order: ['dbt_date_integration_tests', 'dbt_date'] # enable override 19 | 20 | vars: 21 | dbt_date_dispatch_list: ['dbt_date_integration_tests'] 22 | "dbt_date:time_zone": "America/Los_Angeles" 23 | 24 | quoting: 25 | database: false 26 | identifier: false 27 | schema: false 28 | 29 | models: 30 | dbt_date_integration_tests: 31 | schema: dbt_date_integration_tests 32 | materialized: table 33 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/string_matching/expect_column_values_to_match_like_pattern.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_values_to_match_like_pattern(model, column_name, 2 | like_pattern, 3 | row_condition=None 4 | ) %} 5 | 6 | {% set expression = dbt_expectations._get_like_pattern_expression(column_name, like_pattern, positive=True) %} 7 | 8 | {{ dbt_expectations.expression_is_true(model, 9 | expression=expression, 10 | group_by_columns=None, 11 | row_condition=row_condition 12 | ) 13 | }} 14 | 15 | {% endtest %} 16 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/string_matching/expect_column_values_to_not_match_like_pattern.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_values_to_not_match_like_pattern(model, column_name, 2 | like_pattern, 3 | row_condition=None 4 | ) %} 5 | 6 | {% set expression = dbt_expectations._get_like_pattern_expression(column_name, like_pattern, positive=False) %} 7 | 8 | {{ dbt_expectations.expression_is_true(model, 9 | expression=expression, 10 | group_by_columns=None, 11 | row_condition=row_condition 12 | ) 13 | }} 14 | 15 | {% endtest %} 16 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/iso_week_of_year.sql: -------------------------------------------------------------------------------- 1 | {%- macro iso_week_of_year(date=None, tz=None) -%} 2 | {%-set dt = date if date else dbt_date.today(tz) -%} 3 | {{ adapter.dispatch('iso_week_of_year', 'dbt_date') (dt) }} 4 | {%- endmacro -%} 5 | 6 | {%- macro _iso_week_of_year(date, week_type) -%} 7 | cast({{ dbt_date.date_part(week_type, date) }} as {{ dbt_utils.type_int() }}) 8 | {%- endmacro %} 9 | 10 | {%- macro default__iso_week_of_year(date) -%} 11 | {{ dbt_date._iso_week_of_year(date, 'isoweek') }} 12 | {%- endmacro %} 13 | 14 | {%- macro snowflake__iso_week_of_year(date) -%} 15 | {{ dbt_date._iso_week_of_year(date, 'weekiso') }} 16 | {%- endmacro %} 17 | 18 | {%- macro postgres__iso_week_of_year(date) -%} 19 | -- postgresql week is isoweek, the first week of a year containing January 4 of that year. 20 | {{ dbt_date._iso_week_of_year(date, 'week') }} 21 | {%- endmacro %} 22 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/sql/test_get_column_values.sql: -------------------------------------------------------------------------------- 1 | 2 | {% set column_values = dbt_utils.get_column_values(ref('data_get_column_values'), 'field', default=[], order_by="field") %} 3 | 4 | 5 | {% if target.type == 'snowflake' %} 6 | 7 | select 8 | {% for val in column_values -%} 9 | 10 | sum(case when field = '{{ val }}' then 1 else 0 end) as count_{{ val }} 11 | {%- if not loop.last %},{% endif -%} 12 | 13 | {%- endfor %} 14 | 15 | from {{ ref('data_get_column_values') }} 16 | 17 | {% else %} 18 | 19 | select 20 | {% for val in column_values -%} 21 | 22 | {{dbt_utils.safe_cast("sum(case when field = '" ~ val ~ "' then 1 else 0 end)", dbt_utils.type_string()) }} as count_{{ val }} 23 | {%- if not loop.last %},{% endif -%} 24 | 25 | {%- endfor %} 26 | 27 | from {{ ref('data_get_column_values') }} 28 | 29 | {% endif %} 30 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/sql/test_nullcheck_table.sql: -------------------------------------------------------------------------------- 1 | {{ config( materialized = "table" ) }} 2 | 3 | -- TO DO: remove if-statement 4 | 5 | {% set tbl = ref('data_nullcheck_table') %} 6 | 7 | 8 | with nulled as ( 9 | 10 | {{ dbt_utils.nullcheck_table(tbl) }} 11 | 12 | ) 13 | 14 | {% if target.type == 'snowflake' %} 15 | 16 | select 17 | field_1::varchar as field_1, 18 | field_2::varchar as field_2, 19 | field_3::varchar as field_3 20 | 21 | from nulled 22 | 23 | {% else %} 24 | 25 | select 26 | 27 | {{ dbt_utils.safe_cast('field_1', 28 | dbt_utils.type_string() 29 | )}} as field_1, 30 | 31 | {{ dbt_utils.safe_cast('field_2', 32 | dbt_utils.type_string() 33 | )}} as field_2, 34 | 35 | {{ dbt_utils.safe_cast('field_3', 36 | dbt_utils.type_string() 37 | )}} as field_3 38 | 39 | from nulled 40 | 41 | {% endif %} 42 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/week_start.sql: -------------------------------------------------------------------------------- 1 | {%- macro week_start(date=None, tz=None) -%} 2 | {%-set dt = date if date else dbt_date.today(tz) -%} 3 | {{ adapter.dispatch('week_start', 'dbt_date') (dt) }} 4 | {%- endmacro -%} 5 | 6 | {%- macro default__week_start(date) -%} 7 | cast({{ dbt_utils.date_trunc('week', date) }} as date) 8 | {%- endmacro %} 9 | 10 | {%- macro snowflake__week_start(date) -%} 11 | case 12 | when {{ dbt_date.day_of_week(dbt_utils.date_trunc('week', date), isoweek=False) }} = 1 13 | then {{ dbt_date.n_days_ago( 14 | dbt_date.day_of_week(date, isoweek=False) ~ " - 1", 15 | date 16 | ) }} 17 | end 18 | {%- endmacro %} 19 | 20 | {%- macro postgres__week_start(date) -%} 21 | -- Sunday as week start date 22 | cast({{ dbt_utils.dateadd('day', -1, dbt_utils.date_trunc('week', dbt_utils.dateadd('day', 1, date))) }} as date) 23 | {%- endmacro %} 24 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/month_name.sql: -------------------------------------------------------------------------------- 1 | {%- macro month_name(date, short=True) -%} 2 | {{ adapter.dispatch('month_name', 'dbt_date') (date, short) }} 3 | {%- endmacro %} 4 | 5 | {%- macro default__month_name(date, short) -%} 6 | {%- set f = 'MON' if short else 'MONTH' -%} 7 | to_char({{ date }}, '{{ f }}') 8 | {%- endmacro %} 9 | 10 | {%- macro bigquery__month_name(date, short) -%} 11 | {%- set f = '%b' if short else '%B' -%} 12 | format_date('{{ f }}', cast({{ date }} as date)) 13 | {%- endmacro %} 14 | 15 | {%- macro snowflake__month_name(date, short) -%} 16 | {%- set f = 'MON' if short else 'MMMM' -%} 17 | to_char({{ date }}, '{{ f }}') 18 | {%- endmacro %} 19 | 20 | {%- macro postgres__month_name(date, short) -%} 21 | {# FM = Fill mode, which suppresses padding blanks #} 22 | {%- set f = 'FMMon' if short else 'FMMonth' -%} 23 | to_char({{ date }}, '{{ f }}') 24 | {%- endmacro %} 25 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/multi-column/expect_column_pair_values_to_be_equal.sql: -------------------------------------------------------------------------------- 1 | 2 | {% test expect_column_pair_values_to_be_equal(model, 3 | column_A, 4 | column_B, 5 | row_condition=None 6 | ) %} 7 | 8 | {% set operator = "=" %} 9 | {% set expression = column_A ~ " " ~ operator ~ " " ~ column_B %} 10 | 11 | {{ dbt_expectations.expression_is_true(model, 12 | expression=expression, 13 | group_by_columns=None, 14 | row_condition=row_condition 15 | ) 16 | }} 17 | 18 | {% endtest %} 19 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/sql/get_query_results_as_dict.sql: -------------------------------------------------------------------------------- 1 | {% macro get_query_results_as_dict(query) %} 2 | {{ return(adapter.dispatch('get_query_results_as_dict', 'dbt_utils')(query)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_query_results_as_dict(query) %} 6 | 7 | {# This macro returns a dictionary of the form {column_name: (tuple_of_results)} #} 8 | 9 | {%- call statement('get_query_results', fetch_result=True,auto_begin=false) -%} 10 | 11 | {{ query }} 12 | 13 | {%- endcall -%} 14 | 15 | {% set sql_results={} %} 16 | 17 | {%- if execute -%} 18 | {% set sql_results_table = load_result('get_query_results').table.columns %} 19 | {% for column_name, column in sql_results_table.items() %} 20 | {% do sql_results.update({column_name: column.values()}) %} 21 | {% endfor %} 22 | {%- endif -%} 23 | 24 | {{ return(sql_results) }} 25 | 26 | {% endmacro %} 27 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/schema_tests/not_null_proportion.sql: -------------------------------------------------------------------------------- 1 | {% macro test_not_null_proportion(model) %} 2 | {{ return(adapter.dispatch('test_not_null_proportion', 'dbt_utils')(model, **kwargs)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__test_not_null_proportion(model) %} 6 | 7 | {% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} 8 | {% set at_least = kwargs.get('at_least', kwargs.get('arg')) %} 9 | {% set at_most = kwargs.get('at_most', kwargs.get('arg', 1)) %} 10 | 11 | with validation as ( 12 | select 13 | sum(case when {{ column_name }} is null then 0 else 1 end) / cast(count(*) as numeric) as not_null_proportion 14 | from {{ model }} 15 | ), 16 | validation_errors as ( 17 | select 18 | not_null_proportion 19 | from validation 20 | where not_null_proportion < {{ at_least }} or not_null_proportion > {{ at_most }} 21 | ) 22 | select 23 | * 24 | from validation_errors 25 | 26 | {% endmacro %} 27 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/integration_tests/dbt_project.yml: -------------------------------------------------------------------------------- 1 | 2 | name: 'dbt_expectations_integration_tests' 3 | version: '1.0' 4 | 5 | profile: 'integration_tests' 6 | 7 | config-version: 2 8 | 9 | model-paths: ["models"] 10 | analysis-paths: ["analysis"] 11 | test-paths: ["tests"] 12 | seed-paths: ["data"] 13 | macro-paths: ["macros"] 14 | 15 | target-path: "target" 16 | clean-targets: ["target", "dbt_modules", "dbt_packages"] 17 | 18 | dispatch: 19 | - macro_namespace: dbt_expectations 20 | search_order: ['dbt_expectations_integration_tests', 'dbt_expectations'] 21 | - macro_namespace: dbt_utils 22 | search_order: ['dbt_expectations', 'dbt_utils'] 23 | vars: 24 | 'dbt_date:time_zone': 'America/Los_Angeles' 25 | 26 | quoting: 27 | database: false 28 | identifier: false 29 | schema: false 30 | 31 | models: 32 | dbt_expectations_integration_tests: 33 | +schema: dbt_expectations_integration_tests 34 | +materialized: table 35 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | services: 3 | 4 | dbt: 5 | image: circleci/python:3.6.3-stretch 6 | depends_on: 7 | - ${TARGET} 8 | env_file: "./integration_tests/.env/${TARGET}.env" 9 | entrypoint: "/repo/run_test.sh ${TARGET} ${MODELS} ${SEEDS}" 10 | working_dir: /repo 11 | volumes: 12 | - ".:/repo" 13 | 14 | postgres: 15 | image: circleci/postgres:9.6.5-alpine-ram 16 | ports: 17 | - "5432:5432" 18 | 19 | # dummy container, since snowflake is a managed service 20 | snowflake: 21 | image: circleci/python:3.6.3-stretch 22 | entrypoint: "/bin/true" 23 | 24 | # dummy container, since bigquery is a managed service 25 | bigquery: 26 | image: circleci/python:3.6.3-stretch 27 | entrypoint: "/bin/true" 28 | 29 | # dummy container, since redshift is a managed service 30 | redshift: 31 | image: circleci/python:3.6.3-stretch 32 | entrypoint: "/bin/true" 33 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this package 4 | title: '' 5 | labels: enhancement, triage 6 | assignees: '' 7 | 8 | --- 9 | 10 | ### Describe the feature 11 | A clear and concise description of what you want to happen. 12 | 13 | ### Describe alternatives you've considered 14 | A clear and concise description of any alternative solutions or features you've considered. 15 | 16 | ### Additional context 17 | Is this feature database-specific? Which database(s) is/are relevant? Please include any other relevant context here. 18 | 19 | ### Who will this benefit? 20 | What kind of use case will this feature be useful for? Please be specific and provide examples, this will help us prioritize properly. 21 | 22 | ### Are you interested in contributing this feature? 23 | 26 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/aggregate_functions/expect_column_distinct_count_to_equal.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_distinct_count_to_equal(model, 2 | column_name, 3 | value, 4 | quote_values=False, 5 | group_by=None, 6 | row_condition=None 7 | ) %} 8 | {% set expression %} 9 | count(distinct {{ column_name }}) = {{ value }} 10 | {% endset %} 11 | {{ dbt_expectations.expression_is_true(model, 12 | expression=expression, 13 | group_by_columns=group_by, 14 | row_condition=row_condition) 15 | }} 16 | {%- endtest -%} 17 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/integration_tests/models/schema_tests/timeseries_data_grouped.sql: -------------------------------------------------------------------------------- 1 | with dates as ( 2 | select * from {{ ref('timeseries_base') }} 3 | ), 4 | groupings as ( 5 | select * from {{ ref('series_4') }} 6 | ), 7 | row_values as ( 8 | select * from {{ ref('series_10') }} 9 | ), 10 | add_row_values as ( 11 | 12 | select 13 | cast(d.date_day as {{ dbt_expectations.type_datetime() }}) as date_day, 14 | cast(g.generated_number as {{ dbt_utils.type_int() }}) as group_id, 15 | cast(floor(100 * r.generated_number) as {{ dbt_utils.type_int() }}) as row_value 16 | from 17 | dates d 18 | cross join 19 | groupings g 20 | cross join 21 | row_values r 22 | 23 | ), 24 | add_logs as ( 25 | 26 | select 27 | *, 28 | {{ dbt_expectations.log_natural('nullif(row_value, 0)') }} as row_value_log 29 | from 30 | add_row_values 31 | ) 32 | select 33 | * 34 | from 35 | add_logs 36 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/datetime/test_date_spine.sql: -------------------------------------------------------------------------------- 1 | 2 | -- snowflake doesn't like this as a view because the `generate_series` 3 | -- call creates a CTE called `unioned`, as does the `equality` schema test. 4 | -- Ideally, Snowflake would be smart enough to know that these CTE names are 5 | -- different, as they live in different relations. TODO: use a less common cte name 6 | 7 | {{ config(materialized='table') }} 8 | 9 | with date_spine as ( 10 | 11 | {% if target.type == 'postgres' %} 12 | {{ dbt_utils.date_spine("day", "'2018-01-01'::date", "'2018-01-10'::date") }} 13 | 14 | {% elif target.type == 'bigquery' %} 15 | select cast(date_day as date) as date_day 16 | from ({{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }}) 17 | 18 | {% else %} 19 | {{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }} 20 | {% endif %} 21 | 22 | ) 23 | 24 | select date_day 25 | from date_spine 26 | 27 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/tests/sql/test_get_column_values_use_default.sql: -------------------------------------------------------------------------------- 1 | 2 | {% set column_values = dbt_utils.get_column_values(ref('data_get_column_values_dropped'), 'field', default=['y', 'z'], order_by="field") %} 3 | 4 | with expected as ( 5 | select {{ dbt_utils.safe_cast("'y'", dbt_utils.type_string()) }} as expected_column_value union all 6 | select {{ dbt_utils.safe_cast("'z'", dbt_utils.type_string()) }} as expected_column_value 7 | ), 8 | 9 | actual as ( 10 | 11 | {% for val in column_values %} 12 | select {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} as actual_column_value 13 | {% if not loop.last %} 14 | union all 15 | {% endif %} 16 | {% endfor %} 17 | ), 18 | 19 | failures as ( 20 | select * from actual 21 | where actual.actual_column_value not in ( 22 | select expected.expected_column_value from expected 23 | ) 24 | ) 25 | 26 | select * from failures -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/schema_tests/not_accepted_values.sql: -------------------------------------------------------------------------------- 1 | {% test not_accepted_values(model, column_name, values, quote=True) %} 2 | {{ return(adapter.dispatch('test_not_accepted_values', 'dbt_utils')(model, column_name, values, quote)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_not_accepted_values(model, column_name, values, quote=True) %} 6 | with all_values as ( 7 | 8 | select distinct 9 | {{ column_name }} as value_field 10 | 11 | from {{ model }} 12 | 13 | ), 14 | 15 | validation_errors as ( 16 | 17 | select 18 | value_field 19 | 20 | from all_values 21 | where value_field in ( 22 | {% for value in values -%} 23 | {% if quote -%} 24 | '{{ value }}' 25 | {%- else -%} 26 | {{ value }} 27 | {%- endif -%} 28 | {%- if not loop.last -%},{%- endif %} 29 | {%- endfor %} 30 | ) 31 | 32 | ) 33 | 34 | select * 35 | from validation_errors 36 | 37 | {% endmacro %} 38 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/schema_tests/equal_rowcount.sql: -------------------------------------------------------------------------------- 1 | {% test equal_rowcount(model, compare_model) %} 2 | {{ return(adapter.dispatch('test_equal_rowcount', 'dbt_utils')(model, compare_model)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_equal_rowcount(model, compare_model) %} 6 | 7 | {#-- Needs to be set at parse time, before we return '' below --#} 8 | {{ config(fail_calc = 'coalesce(diff_count, 0)') }} 9 | 10 | {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} 11 | {%- if not execute -%} 12 | {{ return('') }} 13 | {% endif %} 14 | 15 | with a as ( 16 | 17 | select count(*) as count_a from {{ model }} 18 | 19 | ), 20 | b as ( 21 | 22 | select count(*) as count_b from {{ compare_model }} 23 | 24 | ), 25 | final as ( 26 | 27 | select 28 | count_a, 29 | count_b, 30 | abs(count_a - count_b) as diff_count 31 | from a 32 | cross join b 33 | 34 | ) 35 | 36 | select * from final 37 | 38 | {% endmacro %} 39 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/sql/test_unpivot_bool.sql: -------------------------------------------------------------------------------- 1 | 2 | -- snowflake messes with these tests pretty badly since the 3 | -- output of the macro considers the casing of the source 4 | -- table columns. Using some hacks here to get this to work, 5 | -- but we should consider lowercasing the unpivot macro output 6 | -- at some point in the future for consistency 7 | 8 | {% if target.name == 'snowflake' %} 9 | {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} 10 | {% else %} 11 | {% set exclude = ['customer_id', 'created_at'] %} 12 | {% endif %} 13 | 14 | 15 | select 16 | customer_id, 17 | created_at, 18 | case 19 | when '{{ target.name }}' = 'snowflake' then lower(prop) 20 | else prop 21 | end as prop, 22 | val 23 | 24 | from ( 25 | {{ dbt_utils.unpivot( 26 | relation=ref('data_unpivot_bool'), 27 | cast_to=dbt_utils.type_string(), 28 | exclude=exclude, 29 | field_name='prop', 30 | value_name='val' 31 | ) }} 32 | ) as sbq 33 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/right.sql: -------------------------------------------------------------------------------- 1 | {% macro right(string_text, length_expression) -%} 2 | {{ return(adapter.dispatch('right', 'dbt_utils') (string_text, length_expression)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__right(string_text, length_expression) %} 6 | 7 | right( 8 | {{ string_text }}, 9 | {{ length_expression }} 10 | ) 11 | 12 | {%- endmacro -%} 13 | 14 | {% macro bigquery__right(string_text, length_expression) %} 15 | 16 | case when {{ length_expression }} = 0 17 | then '' 18 | else 19 | substr( 20 | {{ string_text }}, 21 | -1 * ({{ length_expression }}) 22 | ) 23 | end 24 | 25 | {%- endmacro -%} 26 | 27 | {% macro snowflake__right(string_text, length_expression) %} 28 | 29 | case when {{ length_expression }} = 0 30 | then '' 31 | else 32 | right( 33 | {{ string_text }}, 34 | {{ length_expression }} 35 | ) 36 | end 37 | 38 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/regex/regexp_instr.sql: -------------------------------------------------------------------------------- 1 | {% macro regexp_instr(source_value, regexp, position=1, occurrence=1) %} 2 | 3 | {{ adapter.dispatch('regexp_instr', 'dbt_expectations')( 4 | source_value, regexp, position, occurrence 5 | ) }} 6 | 7 | {% endmacro %} 8 | 9 | {% macro default__regexp_instr(source_value, regexp, position, occurrence) %} 10 | regexp_instr({{ source_value }}, '{{ regexp }}', {{ position }}, {{ occurrence }}) 11 | {% endmacro %} 12 | 13 | {% macro redshift__regexp_instr(source_value, regexp, position, occurrence) %} 14 | regexp_instr({{ source_value }}, '{{ regexp }}', {{ position }}, {{ occurrence }}) 15 | {% endmacro %} 16 | 17 | {% macro postgres__regexp_instr(source_value, regexp, position, occurrence) %} 18 | array_length((select regexp_matches({{ source_value }}, '{{ regexp }}')), 1) 19 | {% endmacro %} 20 | 21 | 22 | {% macro spark__regexp_instr(source_value, regexp, position, occurrence) %} 23 | case when {{ source_value }} rlike '{{ regexp }}' then 1 else 0 end 24 | {% endmacro %} 25 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/sql/test_unpivot.sql: -------------------------------------------------------------------------------- 1 | 2 | -- snowflake messes with these tests pretty badly since the 3 | -- output of the macro considers the casing of the source 4 | -- table columns. Using some hacks here to get this to work, 5 | -- but we should consider lowercasing the unpivot macro output 6 | -- at some point in the future for consistency 7 | 8 | {% if target.name == 'snowflake' %} 9 | {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} 10 | {% else %} 11 | {% set exclude = ['customer_id', 'created_at'] %} 12 | {% endif %} 13 | 14 | 15 | select 16 | customer_id, 17 | created_at, 18 | case 19 | when '{{ target.name }}' = 'snowflake' then lower(prop) 20 | else prop 21 | end as prop, 22 | val 23 | 24 | from ( 25 | {{ dbt_utils.unpivot( 26 | relation=ref('data_unpivot'), 27 | cast_to=dbt_utils.type_string(), 28 | exclude=exclude, 29 | remove=['name'], 30 | field_name='prop', 31 | value_name='val' 32 | ) }} 33 | ) as sbq 34 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/multi-column/expect_multicolumn_sum_to_equal.sql: -------------------------------------------------------------------------------- 1 | 2 | {% test expect_multicolumn_sum_to_equal(model, 3 | column_list, 4 | sum_total, 5 | group_by=None, 6 | row_condition=None 7 | ) %} 8 | 9 | {% set operator = "=" %} 10 | {% set expression %} 11 | {% for column in column_list %} 12 | sum({{ column }}){% if not loop.last %} + {% endif %} 13 | {% endfor %} = {{ sum_total }} 14 | {% endset %} 15 | 16 | {{ dbt_expectations.expression_is_true(model, 17 | expression=expression, 18 | group_by_columns=group_by, 19 | row_condition=row_condition 20 | ) 21 | }} 22 | 23 | {% endtest %} 24 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/web/get_url_path.sql: -------------------------------------------------------------------------------- 1 | {% macro get_url_path(field) -%} 2 | {{ return(adapter.dispatch('get_url_path', 'dbt_utils')(field)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_url_path(field) -%} 6 | 7 | {%- set stripped_url = 8 | dbt_utils.replace( 9 | dbt_utils.replace(field, "'http://'", "''"), "'https://'", "''") 10 | -%} 11 | 12 | {%- set first_slash_pos -%} 13 | coalesce( 14 | nullif({{dbt_utils.position("'/'", stripped_url)}}, 0), 15 | {{dbt_utils.position("'?'", stripped_url)}} - 1 16 | ) 17 | {%- endset -%} 18 | 19 | {%- set parsed_path = 20 | dbt_utils.split_part( 21 | dbt_utils.right( 22 | stripped_url, 23 | dbt_utils.length(stripped_url) ~ "-" ~ first_slash_pos 24 | ), 25 | "'?'", 1 26 | ) 27 | -%} 28 | 29 | {{ dbt_utils.safe_cast( 30 | parsed_path, 31 | dbt_utils.type_string() 32 | )}} 33 | 34 | {%- endmacro %} 35 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/multi-column/expect_column_pair_values_A_to_be_greater_than_B.sql: -------------------------------------------------------------------------------- 1 | 2 | {% test expect_column_pair_values_A_to_be_greater_than_B(model, 3 | column_A, 4 | column_B, 5 | or_equal=False, 6 | row_condition=None 7 | ) %} 8 | 9 | {% set operator = ">=" if or_equal else ">" %} 10 | {% set expression = column_A ~ " " ~ operator ~ " " ~ column_B %} 11 | 12 | {{ dbt_expectations.expression_is_true(model, 13 | expression=expression, 14 | group_by_columns=None, 15 | row_condition=row_condition 16 | ) 17 | }} 18 | 19 | {% endtest %} 20 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/integration_tests/models/schema_tests/data_test.sql: -------------------------------------------------------------------------------- 1 | select 2 | 1 as idx, 3 | '2020-10-21' as date_col, 4 | cast(0 as {{ dbt_utils.type_float() }}) as col_numeric_a, 5 | cast(1 as {{ dbt_utils.type_float() }}) as col_numeric_b, 6 | 'a' as col_string_a, 7 | 'b' as col_string_b, 8 | cast(null as {{ dbt_utils.type_string() }}) as col_null 9 | 10 | union all 11 | 12 | select 13 | 2 as idx, 14 | '2020-10-22' as date_col, 15 | 1 as col_numeric_a, 16 | 0 as col_numeric_b, 17 | 'b' as col_string_a, 18 | 'ab' as col_string_b, 19 | null as col_null 20 | 21 | union all 22 | 23 | select 24 | 3 as idx, 25 | '2020-10-23' as date_col, 26 | 0.5 as col_numeric_a, 27 | 0.5 as col_numeric_b, 28 | 'c' as col_string_a, 29 | 'abc' as col_string_b, 30 | null as col_null 31 | 32 | union all 33 | 34 | select 35 | 4 as idx, 36 | '2020-10-23' as date_col, 37 | 0.5 as col_numeric_a, 38 | 0.5 as col_numeric_b, 39 | 'c' as col_string_a, 40 | 'abcd' as col_string_b, 41 | null as col_null 42 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/aggregate_functions/expect_column_distinct_count_to_be_greater_than.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_distinct_count_to_be_greater_than(model, 2 | column_name, 3 | value, 4 | quote_values=False, 5 | group_by=None, 6 | row_condition=None 7 | ) %} 8 | {% set expression %} 9 | count(distinct {{ column_name }}) > {{ value }} 10 | {% endset %} 11 | {{ dbt_expectations.expression_is_true(model, 12 | expression=expression, 13 | group_by_columns=group_by, 14 | row_condition=row_condition) 15 | }} 16 | {%- endtest -%} 17 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/sql/test_unpivot_original_api.sql: -------------------------------------------------------------------------------- 1 | 2 | -- unpivot() was enhanced with 3 new parameters 3 | -- This test targets the original API. 4 | 5 | -- snowflake messes with these tests pretty badly since the 6 | -- output of the macro considers the casing of the source 7 | -- table columns. Using some hacks here to get this to work, 8 | -- but we should consider lowercasing the unpivot macro output 9 | -- at some point in the future for consistency 10 | 11 | {% if target.name == 'snowflake' %} 12 | {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} 13 | {% else %} 14 | {% set exclude = ['customer_id', 'created_at'] %} 15 | {% endif %} 16 | 17 | select 18 | customer_id, 19 | created_at, 20 | case 21 | when '{{ target.name }}' = 'snowflake' then lower(FIELD_NAME) 22 | else field_name 23 | end as field_name, 24 | value 25 | 26 | from ( 27 | {{ dbt_utils.unpivot( 28 | table=ref('data_unpivot'), 29 | cast_to=dbt_utils.type_string(), 30 | exclude=exclude 31 | ) }} 32 | ) as sbq 33 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/string_matching/expect_column_values_to_not_match_regex_list.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_values_to_not_match_regex_list(model, column_name, 2 | regex_list, 3 | match_on="any", 4 | row_condition=None 5 | ) %} 6 | 7 | {% set expression %} 8 | {% for regex in regex_list %} 9 | {{ dbt_expectations.regexp_instr(column_name, regex) }} = 0 10 | {%- if not loop.last %} 11 | {{ " and " if match_on == "all" else " or "}} 12 | {% endif -%} 13 | {% endfor %} 14 | {% endset %} 15 | 16 | {{ dbt_expectations.expression_is_true(model, 17 | expression=expression, 18 | group_by_columns=None, 19 | row_condition=row_condition 20 | ) 21 | }} 22 | 23 | {% endtest %} 24 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/string_matching/expect_column_values_to_match_regex_list.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_values_to_match_regex_list(model, column_name, 2 | regex_list, 3 | match_on="any", 4 | row_condition=None 5 | ) %} 6 | 7 | {% set expression %} 8 | {% for regex in regex_list %} 9 | {{ dbt_expectations.regexp_instr(column_name, regex) }} > 0 10 | {%- if not loop.last %} 11 | {{ " and " if match_on == "all" else " or "}} 12 | {% endif -%} 13 | {% endfor %} 14 | {% endset %} 15 | 16 | {{ dbt_expectations.expression_is_true(model, 17 | expression=expression, 18 | group_by_columns=None, 19 | row_condition=row_condition 20 | ) 21 | }} 22 | 23 | {% endtest %} 24 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/column_values_basic/expect_column_values_to_be_between.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_values_to_be_between(model, column_name, 2 | min_value=None, 3 | max_value=None, 4 | row_condition=None, 5 | strictly=False 6 | ) %} 7 | 8 | {% set expression %} 9 | {{ column_name }} 10 | {% endset %} 11 | 12 | {{ dbt_expectations.expression_between(model, 13 | expression=expression, 14 | min_value=min_value, 15 | max_value=max_value, 16 | group_by_columns=None, 17 | row_condition=row_condition, 18 | strictly=strictly 19 | ) }} 20 | 21 | 22 | {% endtest %} 23 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/schema_tests/sequential_values.sql: -------------------------------------------------------------------------------- 1 | {% test sequential_values(model, column_name, interval=1, datepart=None) %} 2 | 3 | {{ return(adapter.dispatch('test_sequential_values', 'dbt_utils')(model, column_name, interval, datepart)) }} 4 | 5 | {% endtest %} 6 | 7 | {% macro default__test_sequential_values(model, column_name, interval=1, datepart=None) %} 8 | 9 | with windowed as ( 10 | 11 | select 12 | {{ column_name }}, 13 | lag({{ column_name }}) over ( 14 | order by {{ column_name }} 15 | ) as previous_{{ column_name }} 16 | from {{ model }} 17 | ), 18 | 19 | validation_errors as ( 20 | select 21 | * 22 | from windowed 23 | {% if datepart %} 24 | where not(cast({{ column_name }} as {{ dbt_utils.type_timestamp() }})= cast({{ dbt_utils.dateadd(datepart, interval, 'previous_' + column_name) }} as {{ dbt_utils.type_timestamp() }})) 25 | {% else %} 26 | where not({{ column_name }} = previous_{{ column_name }} + {{ interval }}) 27 | {% endif %} 28 | ) 29 | 30 | select * 31 | from validation_errors 32 | 33 | {% endmacro %} 34 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/aggregate_functions/expect_column_max_to_be_between.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_max_to_be_between(model, column_name, 2 | min_value=None, 3 | max_value=None, 4 | group_by=None, 5 | row_condition=None, 6 | strictly=False 7 | ) %} 8 | {% set expression %} 9 | max({{ column_name }}) 10 | {% endset %} 11 | {{ dbt_expectations.expression_between(model, 12 | expression=expression, 13 | min_value=min_value, 14 | max_value=max_value, 15 | group_by_columns=group_by, 16 | row_condition=row_condition, 17 | strictly=strictly 18 | ) }} 19 | {% endtest %} 20 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/aggregate_functions/expect_column_sum_to_be_between.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_sum_to_be_between(model, column_name, 2 | min_value=None, 3 | max_value=None, 4 | group_by=None, 5 | row_condition=None, 6 | strictly=False 7 | ) %} 8 | {% set expression %} 9 | sum({{ column_name }}) 10 | {% endset %} 11 | {{ dbt_expectations.expression_between(model, 12 | expression=expression, 13 | min_value=min_value, 14 | max_value=max_value, 15 | group_by_columns=group_by, 16 | row_condition=row_condition, 17 | strictly=strictly 18 | ) }} 19 | {% endtest %} 20 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/integration_tests/models/geo/test_haversine_distance_mi.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | select * from {{ ref('data_haversine_mi') }} 3 | ), 4 | final as ( 5 | select 6 | output as expected, 7 | cast( 8 | {{ 9 | dbt_utils.haversine_distance( 10 | lat1='lat_1', 11 | lon1='lon_1', 12 | lat2='lat_2', 13 | lon2='lon_2', 14 | unit='mi' 15 | ) 16 | }} as {{ dbt_utils.type_numeric() }} 17 | ) as actual 18 | from data 19 | 20 | union all 21 | 22 | select 23 | output as expected, 24 | cast( 25 | {{ 26 | dbt_utils.haversine_distance( 27 | lat1='lat_1', 28 | lon1='lon_1', 29 | lat2='lat_2', 30 | lon2='lon_2', 31 | ) 32 | }} as {{ dbt_utils.type_numeric() }} 33 | ) as actual 34 | from data 35 | ) 36 | select 37 | expected, 38 | round(actual,0) as actual 39 | from final 40 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/table_shape/expect_table_row_count_to_be_between.sql: -------------------------------------------------------------------------------- 1 | {%- test expect_table_row_count_to_be_between(model, 2 | min_value=None, 3 | max_value=None, 4 | group_by=None, 5 | row_condition=None, 6 | strictly=False 7 | ) -%} 8 | {% set expression %} 9 | count(*) 10 | {% endset %} 11 | {{ dbt_expectations.expression_between(model, 12 | expression=expression, 13 | min_value=min_value, 14 | max_value=max_value, 15 | group_by_columns=group_by, 16 | row_condition=row_condition, 17 | strictly=strictly 18 | ) }} 19 | {%- endtest -%} 20 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/string_matching/expect_column_value_lengths_to_be_between.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_value_lengths_to_be_between(model, column_name, 2 | min_value=None, 3 | max_value=None, 4 | row_condition=None, 5 | strictly=False 6 | ) %} 7 | {% set expression %} 8 | {{ dbt_utils.length(column_name) }} 9 | {% endset %} 10 | 11 | {{ dbt_expectations.expression_between(model, 12 | expression=expression, 13 | min_value=min_value, 14 | max_value=max_value, 15 | group_by_columns=None, 16 | row_condition=row_condition, 17 | strictly=strictly 18 | ) }} 19 | 20 | {% endtest %} 21 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/aggregate_functions/expect_column_mean_to_be_between.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_mean_to_be_between(model, column_name, 2 | min_value=None, 3 | max_value=None, 4 | group_by=None, 5 | row_condition=None, 6 | strictly=False 7 | ) %} 8 | {% set expression %} 9 | avg({{ column_name }}) 10 | {% endset %} 11 | {{ dbt_expectations.expression_between(model, 12 | expression=expression, 13 | min_value=min_value, 14 | max_value=max_value, 15 | group_by_columns=group_by, 16 | row_condition=row_condition, 17 | strictly=strictly 18 | ) }} 19 | {% endtest %} 20 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/aggregate_functions/expect_column_min_to_be_between.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_min_to_be_between(model, column_name, 2 | min_value=None, 3 | max_value=None, 4 | group_by=None, 5 | row_condition=None, 6 | strictly=False 7 | ) %} 8 | {% set expression %} 9 | min({{ column_name }}) 10 | {% endset %} 11 | {{ dbt_expectations.expression_between(model, 12 | expression=expression, 13 | min_value=min_value, 14 | max_value=max_value, 15 | group_by_columns=group_by, 16 | row_condition=row_condition, 17 | strictly=strictly 18 | ) }} 19 | 20 | {% endtest %} 21 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/string_matching/expect_column_values_to_match_like_pattern_list.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_values_to_match_like_pattern_list(model, column_name, 2 | like_pattern_list, 3 | match_on="any", 4 | row_condition=None 5 | ) %} 6 | 7 | {% set expression %} 8 | {% for like_pattern in like_pattern_list %} 9 | {{ dbt_expectations._get_like_pattern_expression(column_name, like_pattern, positive=True) }} 10 | {%- if not loop.last %} 11 | {{ " and " if match_on == "all" else " or "}} 12 | {% endif -%} 13 | {% endfor %} 14 | {% endset %} 15 | 16 | {{ dbt_expectations.expression_is_true(model, 17 | expression=expression, 18 | group_by_columns=None, 19 | row_condition=row_condition 20 | ) 21 | }} 22 | 23 | {% endtest %} 24 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/string_matching/expect_column_values_to_not_match_like_pattern_list.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_values_to_not_match_like_pattern_list(model, column_name, 2 | like_pattern_list, 3 | match_on="any", 4 | row_condition=None 5 | ) %} 6 | 7 | {% set expression %} 8 | {% for like_pattern in like_pattern_list %} 9 | {{ dbt_expectations._get_like_pattern_expression(column_name, like_pattern, positive=False) }} 10 | {%- if not loop.last %} 11 | {{ " and " if match_on == "all" else " or "}} 12 | {% endif -%} 13 | {% endfor %} 14 | {% endset %} 15 | 16 | {{ dbt_expectations.expression_is_true(model, 17 | expression=expression, 18 | group_by_columns=None, 19 | row_condition=row_condition 20 | ) 21 | }} 22 | 23 | {% endtest %} 24 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/schema_tests/cardinality_equality.sql: -------------------------------------------------------------------------------- 1 | {% test cardinality_equality(model, column_name, to, field) %} 2 | {{ return(adapter.dispatch('test_cardinality_equality', 'dbt_utils')(model, column_name, to, field)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_cardinality_equality(model, column_name, to, field) %} 6 | 7 | {# T-SQL does not let you use numbers as aliases for columns #} 8 | {# Thus, no "GROUP BY 1" #} 9 | 10 | with table_a as ( 11 | select 12 | {{ column_name }}, 13 | count(*) as num_rows 14 | from {{ model }} 15 | group by {{ column_name }} 16 | ), 17 | 18 | table_b as ( 19 | select 20 | {{ field }}, 21 | count(*) as num_rows 22 | from {{ to }} 23 | group by {{ field }} 24 | ), 25 | 26 | except_a as ( 27 | select * 28 | from table_a 29 | {{ dbt_utils.except() }} 30 | select * 31 | from table_b 32 | ), 33 | 34 | except_b as ( 35 | select * 36 | from table_b 37 | {{ dbt_utils.except() }} 38 | select * 39 | from table_a 40 | ), 41 | 42 | unioned as ( 43 | select * 44 | from except_a 45 | union all 46 | select * 47 | from except_b 48 | ) 49 | 50 | select * 51 | from unioned 52 | 53 | {% endmacro %} 54 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/aggregate_functions/expect_column_median_to_be_between.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_median_to_be_between(model, column_name, 2 | min_value=None, 3 | max_value=None, 4 | group_by=None, 5 | row_condition=None, 6 | strictly=False 7 | ) %} 8 | 9 | {% set expression %} 10 | {{ dbt_expectations.median(column_name) }} 11 | {% endset %} 12 | {{ dbt_expectations.expression_between(model, 13 | expression=expression, 14 | min_value=min_value, 15 | max_value=max_value, 16 | group_by_columns=group_by, 17 | row_condition=row_condition, 18 | strictly=strictly 19 | ) }} 20 | {% endtest %} 21 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/dateadd.sql: -------------------------------------------------------------------------------- 1 | {% macro dateadd(datepart, interval, from_date_or_timestamp) %} 2 | {{ return(adapter.dispatch('dateadd', 'dbt_utils')(datepart, interval, from_date_or_timestamp)) }} 3 | {% endmacro %} 4 | 5 | 6 | {% macro default__dateadd(datepart, interval, from_date_or_timestamp) %} 7 | 8 | dateadd( 9 | {{ datepart }}, 10 | {{ interval }}, 11 | {{ from_date_or_timestamp }} 12 | ) 13 | 14 | {% endmacro %} 15 | 16 | 17 | {% macro bigquery__dateadd(datepart, interval, from_date_or_timestamp) %} 18 | 19 | datetime_add( 20 | cast( {{ from_date_or_timestamp }} as datetime), 21 | interval {{ interval }} {{ datepart }} 22 | ) 23 | 24 | {% endmacro %} 25 | 26 | {% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %} 27 | 28 | {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }})) 29 | 30 | {% endmacro %} 31 | 32 | {# redshift should use default instead of postgres #} 33 | {% macro redshift__dateadd(datepart, interval, from_date_or_timestamp) %} 34 | 35 | {{ return(dbt_utils.default__dateadd(datepart, interval, from_date_or_timestamp)) }} 36 | 37 | {% endmacro %} 38 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/column_values_basic/expect_column_values_to_be_in_type_list.sql: -------------------------------------------------------------------------------- 1 | {%- test expect_column_values_to_be_in_type_list(model, column_name, column_type_list) -%} 2 | {%- if execute -%} 3 | 4 | {%- set column_name = column_name | upper -%} 5 | {%- set columns_in_relation = adapter.get_columns_in_relation(model) -%} 6 | {%- set column_type_list = column_type_list| map("upper") | list -%} 7 | with relation_columns as ( 8 | 9 | {% for column in columns_in_relation %} 10 | select 11 | cast('{{ column.name | upper }}' as {{ dbt_utils.type_string() }}) as relation_column, 12 | cast('{{ column.dtype | upper }}' as {{ dbt_utils.type_string() }}) as relation_column_type 13 | {% if not loop.last %}union all{% endif %} 14 | {% endfor %} 15 | ), 16 | test_data as ( 17 | 18 | select 19 | * 20 | from 21 | relation_columns 22 | where 23 | relation_column = '{{ column_name }}' 24 | and 25 | relation_column_type not in ('{{ column_type_list | join("', '") }}') 26 | 27 | ) 28 | select * 29 | from test_data 30 | 31 | {%- endif -%} 32 | {%- endtest -%} 33 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/aggregate_functions/expect_column_unique_value_count_to_be_between.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_unique_value_count_to_be_between(model, column_name, 2 | min_value=None, 3 | max_value=None, 4 | group_by=None, 5 | row_condition=None, 6 | strictly=False 7 | ) %} 8 | {% set expression %} 9 | count(distinct {{ column_name }}) 10 | {% endset %} 11 | {{ dbt_expectations.expression_between(model, 12 | expression=expression, 13 | min_value=min_value, 14 | max_value=max_value, 15 | group_by_columns=group_by, 16 | row_condition=row_condition, 17 | strictly=strictly 18 | ) }} 19 | {% endtest %} 20 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/table_shape/expect_table_column_count_to_be_between.sql: -------------------------------------------------------------------------------- 1 | {%- test expect_table_column_count_to_be_between(model, 2 | min_value=None, 3 | max_value=None 4 | ) -%} 5 | {%- if min_value is none and max_value is none -%} 6 | {{ exceptions.raise_compiler_error( 7 | "You have to provide either a min_value, max_value or both." 8 | ) }} 9 | {%- endif -%} 10 | {%- if execute -%} 11 | {%- set number_actual_columns = (adapter.get_columns_in_relation(model) | length) -%} 12 | 13 | {%- set expression %} 14 | ( 1=1 15 | {%- if min_value %} and number_actual_columns >= min_value{% endif %} 16 | {%- if max_value %} and number_actual_columns <= max_value{% endif %} 17 | ) 18 | {% endset -%} 19 | 20 | with test_data as ( 21 | 22 | select 23 | {{ number_actual_columns }} as number_actual_columns, 24 | {{ min_value if min_value else 0 }} as min_value, 25 | {{ max_value if max_value else 0 }} as max_value 26 | 27 | ) 28 | select * 29 | from test_data 30 | where 31 | not {{ expression }} 32 | {%- endif -%} 33 | {%- endtest -%} 34 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/sql/get_relations_by_prefix.sql: -------------------------------------------------------------------------------- 1 | {% macro get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %} 2 | {{ return(adapter.dispatch('get_relations_by_prefix', 'dbt_utils')(schema, prefix, exclude, database)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %} 6 | 7 | {%- call statement('get_tables', fetch_result=True) %} 8 | 9 | {{ dbt_utils.get_tables_by_prefix_sql(schema, prefix, exclude, database) }} 10 | 11 | {%- endcall -%} 12 | 13 | {%- set table_list = load_result('get_tables') -%} 14 | 15 | {%- if table_list and table_list['table'] -%} 16 | {%- set tbl_relations = [] -%} 17 | {%- for row in table_list['table'] -%} 18 | {%- set tbl_relation = api.Relation.create( 19 | database=database, 20 | schema=row.table_schema, 21 | identifier=row.table_name, 22 | type=row.table_type 23 | ) -%} 24 | {%- do tbl_relations.append(tbl_relation) -%} 25 | {%- endfor -%} 26 | 27 | {{ return(tbl_relations) }} 28 | {%- else -%} 29 | {{ return([]) }} 30 | {%- endif -%} 31 | 32 | {% endmacro %} 33 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/schema_tests/relationships_where.sql: -------------------------------------------------------------------------------- 1 | {% test relationships_where(model, column_name, to, field, from_condition="1=1", to_condition="1=1") %} 2 | {{ return(adapter.dispatch('test_relationships_where', 'dbt_utils')(model, column_name, to, field, from_condition, to_condition)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_relationships_where(model, column_name, to, field, from_condition="1=1", to_condition="1=1") %} 6 | 7 | {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} 8 | {# ref https://stackoverflow.com/a/7170753/3842610 #} 9 | 10 | with left_table as ( 11 | 12 | select 13 | {{column_name}} as id 14 | 15 | from {{model}} 16 | 17 | where {{column_name}} is not null 18 | and {{from_condition}} 19 | 20 | ), 21 | 22 | right_table as ( 23 | 24 | select 25 | {{field}} as id 26 | 27 | from {{to}} 28 | 29 | where {{field}} is not null 30 | and {{to_condition}} 31 | 32 | ), 33 | 34 | exceptions as ( 35 | 36 | select 37 | left_table.id, 38 | right_table.id as right_id 39 | 40 | from left_table 41 | 42 | left join right_table 43 | on left_table.id = right_table.id 44 | 45 | where right_table.id is null 46 | 47 | ) 48 | 49 | select * from exceptions 50 | 51 | {% endmacro %} 52 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/aggregate_functions/expect_column_distinct_count_to_equal_other_table.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_distinct_count_to_equal_other_table(model, 2 | compare_model, 3 | column_name, 4 | compare_column_name, 5 | row_condition=None, 6 | compare_row_condition=None 7 | ) %} 8 | {%- set expression -%} 9 | count(distinct {{ column_name }}) 10 | {%- endset -%} 11 | {%- set compare_expression -%} 12 | {%- if compare_column_name -%} 13 | count(distinct {{ compare_column_name }}) 14 | {%- else -%} 15 | {{ expression }} 16 | {%- endif -%} 17 | {%- endset -%} 18 | {{ dbt_expectations.test_equal_expression( 19 | model, 20 | expression=expression, 21 | compare_model=compare_model, 22 | compare_expression=compare_expression, 23 | row_condition=row_condition, 24 | compare_row_condition=compare_row_condition, 25 | return_difference=True 26 | ) }} 27 | {%- endtest -%} 28 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/schema_tests/unique_combination_of_columns.sql: -------------------------------------------------------------------------------- 1 | {% test unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %} 2 | {{ return(adapter.dispatch('test_unique_combination_of_columns', 'dbt_utils')(model, combination_of_columns, quote_columns)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %} 6 | 7 | {% if not quote_columns %} 8 | {%- set column_list=combination_of_columns %} 9 | {% elif quote_columns %} 10 | {%- set column_list=[] %} 11 | {% for column in combination_of_columns -%} 12 | {% set column_list = column_list.append( adapter.quote(column) ) %} 13 | {%- endfor %} 14 | {% else %} 15 | {{ exceptions.raise_compiler_error( 16 | "`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '" ~ quote ~"'.'" 17 | ) }} 18 | {% endif %} 19 | 20 | {%- set columns_csv=column_list | join(', ') %} 21 | 22 | 23 | with validation_errors as ( 24 | 25 | select 26 | {{ columns_csv }} 27 | from {{ model }} 28 | group by {{ columns_csv }} 29 | having count(*) > 1 30 | 31 | ) 32 | 33 | select * 34 | from validation_errors 35 | 36 | 37 | {% endmacro %} 38 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/aggregate_functions/expect_column_proportion_of_unique_values_to_be_between.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_proportion_of_unique_values_to_be_between(model, column_name, 2 | min_value=None, 3 | max_value=None, 4 | group_by=None, 5 | row_condition=None, 6 | strictly=False 7 | ) %} 8 | {% set expression %} 9 | count(distinct {{ column_name }})/count({{ column_name }}) 10 | {% endset %} 11 | {{ dbt_expectations.expression_between(model, 12 | expression=expression, 13 | min_value=min_value, 14 | max_value=max_value, 15 | group_by_columns=group_by, 16 | row_condition=row_condition, 17 | strictly=strictly 18 | ) }} 19 | 20 | {% endtest %} 21 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/schema_tests/accepted_range.sql: -------------------------------------------------------------------------------- 1 | {% test accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %} 2 | {{ return(adapter.dispatch('test_accepted_range', 'dbt_utils')(model, column_name, min_value, max_value, inclusive)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %} 6 | 7 | with meet_condition as( 8 | select * 9 | from {{ model }} 10 | ), 11 | 12 | validation_errors as ( 13 | select * 14 | from meet_condition 15 | where 16 | -- never true, defaults to an empty result set. Exists to ensure any combo of the `or` clauses below succeeds 17 | 1 = 2 18 | 19 | {%- if min_value is not none %} 20 | -- records with a value >= min_value are permitted. The `not` flips this to find records that don't meet the rule. 21 | or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} 22 | {%- endif %} 23 | 24 | {%- if max_value is not none %} 25 | -- records with a value <= max_value are permitted. The `not` flips this to find records that don't meet the rule. 26 | or not {{ column_name }} < {{- "=" if inclusive }} {{ max_value }} 27 | {%- endif %} 28 | ) 29 | 30 | select * 31 | from validation_errors 32 | 33 | {% endmacro %} 34 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/day_name.sql: -------------------------------------------------------------------------------- 1 | {%- macro day_name(date, short=True) -%} 2 | {{ adapter.dispatch('day_name', 'dbt_date') (date, short) }} 3 | {%- endmacro %} 4 | 5 | {%- macro default__day_name(date, short) -%} 6 | {%- set f = 'Dy' if short else 'Day' -%} 7 | to_char({{ date }}, '{{ f }}') 8 | {%- endmacro %} 9 | 10 | {%- macro snowflake__day_name(date, short) -%} 11 | {%- if short -%} 12 | dayname({{ date }}) 13 | {%- else -%} 14 | -- long version not implemented on Snowflake so we're doing it manually :/ 15 | case dayname({{ date }}) 16 | when 'Mon' then 'Monday' 17 | when 'Tue' then 'Tuesday' 18 | when 'Wed' then 'Wednesday' 19 | when 'Thu' then 'Thursday' 20 | when 'Fri' then 'Friday' 21 | when 'Sat' then 'Saturday' 22 | when 'Sun' then 'Sunday' 23 | end 24 | {%- endif -%} 25 | 26 | {%- endmacro %} 27 | 28 | {%- macro bigquery__day_name(date, short) -%} 29 | {%- set f = '%a' if short else '%A' -%} 30 | format_date('{{ f }}', cast({{ date }} as date)) 31 | {%- endmacro %} 32 | 33 | {%- macro postgres__day_name(date, short) -%} 34 | {# FM = Fill mode, which suppresses padding blanks #} 35 | {%- set f = 'FMDy' if short else 'FMDay' -%} 36 | to_char({{ date }}, '{{ f }}') 37 | {%- endmacro %} 38 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/column_values_basic/expect_column_values_to_be_decreasing.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_values_to_be_decreasing(model, column_name, 2 | sort_column=None, 3 | strictly=True, 4 | row_condition=None) %} 5 | 6 | {%- set sort_column = column_name if not sort_column else sort_column -%} 7 | {%- set operator = "<" if strictly else "<=" %} 8 | with all_values as ( 9 | 10 | select 11 | {{ sort_column }} as sort_column, 12 | {{ column_name }} as value_field 13 | 14 | from {{ model }} 15 | {% if row_condition %} 16 | where {{ row_condition }} 17 | {% endif %} 18 | 19 | ), 20 | add_lag_values as ( 21 | 22 | select 23 | sort_column, 24 | value_field, 25 | lag(value_field) over(order by sort_column) as value_field_lag 26 | from 27 | all_values 28 | 29 | ), 30 | validation_errors as ( 31 | 32 | select 33 | * 34 | from 35 | add_lag_values 36 | where 37 | value_field_lag is not null 38 | and 39 | not (value_field {{ operator }} value_field_lag) 40 | 41 | ) 42 | select * 43 | from validation_errors 44 | {% endtest %} 45 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/column_values_basic/expect_column_values_to_be_increasing.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_values_to_be_increasing(model, column_name, 2 | sort_column=None, 3 | strictly=True, 4 | row_condition=None) %} 5 | 6 | {%- set sort_column = column_name if not sort_column else sort_column -%} 7 | {%- set operator = ">" if strictly else ">=" %} 8 | with all_values as ( 9 | 10 | select 11 | {{ sort_column }} as sort_column, 12 | {{ column_name }} as value_field 13 | 14 | from {{ model }} 15 | {% if row_condition %} 16 | where {{ row_condition }} 17 | {% endif %} 18 | 19 | ), 20 | add_lag_values as ( 21 | 22 | select 23 | sort_column, 24 | value_field, 25 | lag(value_field) over(order by sort_column) as value_field_lag 26 | from 27 | all_values 28 | 29 | ), 30 | validation_errors as ( 31 | 32 | select 33 | * 34 | from 35 | add_lag_values 36 | where 37 | value_field_lag is not null 38 | and 39 | not (value_field {{ operator }} value_field_lag) 40 | 41 | ) 42 | select * 43 | from validation_errors 44 | {% endtest %} 45 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/current_timestamp.sql: -------------------------------------------------------------------------------- 1 | {% macro current_timestamp() -%} 2 | {{ return(adapter.dispatch('current_timestamp', 'dbt_utils')()) }} 3 | {%- endmacro %} 4 | 5 | {% macro default__current_timestamp() %} 6 | current_timestamp::{{dbt_utils.type_timestamp()}} 7 | {% endmacro %} 8 | 9 | {% macro redshift__current_timestamp() %} 10 | getdate() 11 | {% endmacro %} 12 | 13 | {% macro bigquery__current_timestamp() %} 14 | current_timestamp 15 | {% endmacro %} 16 | 17 | 18 | 19 | {% macro current_timestamp_in_utc() -%} 20 | {{ return(adapter.dispatch('current_timestamp_in_utc', 'dbt_utils')()) }} 21 | {%- endmacro %} 22 | 23 | {% macro default__current_timestamp_in_utc() %} 24 | {{dbt_utils.current_timestamp()}} 25 | {% endmacro %} 26 | 27 | {% macro snowflake__current_timestamp_in_utc() %} 28 | convert_timezone('UTC', {{dbt_utils.current_timestamp()}})::{{dbt_utils.type_timestamp()}} 29 | {% endmacro %} 30 | 31 | {% macro postgres__current_timestamp_in_utc() %} 32 | (current_timestamp at time zone 'utc')::{{dbt_utils.type_timestamp()}} 33 | {% endmacro %} 34 | 35 | {# redshift should use default instead of postgres #} 36 | {% macro redshift__current_timestamp_in_utc() %} 37 | {{ return(dbt_utils.default__current_timestamp_in_utc()) }} 38 | {% endmacro %} 39 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/table_shape/expect_column_to_exist.sql: -------------------------------------------------------------------------------- 1 | {%- test expect_column_to_exist(model, column_name, column_index=None, transform="upper") -%} 2 | {%- if execute -%} 3 | 4 | {%- set column_name = column_name | map(transform) | join -%} 5 | {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%} 6 | 7 | {%- set matching_column_index = relation_column_names.index(column_name) if column_name in relation_column_names else -1 %} 8 | 9 | {%- if column_index -%} 10 | 11 | {%- set column_index_0 = column_index - 1 if column_index > 0 else 0 -%} 12 | 13 | {%- set column_index_matches = true if matching_column_index == column_index_0 else false %} 14 | 15 | {%- else -%} 16 | 17 | {%- set column_index_matches = true -%} 18 | 19 | {%- endif %} 20 | 21 | with test_data as ( 22 | 23 | select 24 | cast('{{ column_name }}' as {{ dbt_utils.type_string() }}) as column_name, 25 | {{ matching_column_index }} as matching_column_index, 26 | {{ column_index_matches }} as column_index_matches 27 | 28 | ) 29 | select * 30 | from test_data 31 | where 32 | not(matching_column_index >= 0 and column_index_matches) 33 | 34 | {%- endif -%} 35 | {%- endtest -%} 36 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/sql/star.sql: -------------------------------------------------------------------------------- 1 | {% macro star(from, relation_alias=False, except=[], prefix='', suffix='') -%} 2 | {{ return(adapter.dispatch('star', 'dbt_utils')(from, relation_alias, except, prefix, suffix)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__star(from, relation_alias=False, except=[], prefix='', suffix='') -%} 6 | {%- do dbt_utils._is_relation(from, 'star') -%} 7 | {%- do dbt_utils._is_ephemeral(from, 'star') -%} 8 | 9 | {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} 10 | {%- if not execute -%} 11 | {{ return('') }} 12 | {% endif %} 13 | 14 | {%- set include_cols = [] %} 15 | {%- set cols = adapter.get_columns_in_relation(from) -%} 16 | {%- set except = except | map("lower") | list %} 17 | {%- for col in cols -%} 18 | 19 | {%- if col.column|lower not in except -%} 20 | {% do include_cols.append(col.column) %} 21 | 22 | {%- endif %} 23 | {%- endfor %} 24 | 25 | {%- for col in include_cols %} 26 | 27 | {%- if relation_alias %}{{ relation_alias }}.{% else %}{%- endif -%}{{ adapter.quote(col)|trim }} as {{ adapter.quote(prefix ~ col ~ suffix)|trim }} 28 | {%- if not loop.last %},{{ '\n ' }}{% endif %} 29 | 30 | {%- endfor -%} 31 | {%- endmacro %} 32 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/column_values_basic/expect_column_values_to_not_be_in_set.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_values_to_not_be_in_set(model, column_name, 2 | value_set, 3 | quote_values=True, 4 | row_condition=None 5 | ) %} 6 | 7 | with all_values as ( 8 | 9 | select 10 | {{ column_name }} as value_field 11 | 12 | from {{ model }} 13 | {% if row_condition %} 14 | where {{ row_condition }} 15 | {% endif %} 16 | 17 | ), 18 | set_values as ( 19 | 20 | {% for value in value_set -%} 21 | select 22 | {% if quote_values -%} 23 | cast('{{ value }}' as {{ dbt_utils.type_string() }}) 24 | {%- else -%} 25 | {{ value }} 26 | {%- endif %} as value_field 27 | {% if not loop.last %}union all{% endif %} 28 | {% endfor %} 29 | ), 30 | validation_errors as ( 31 | -- values from the model that match the set 32 | select 33 | v.value_field 34 | from 35 | all_values v 36 | join 37 | set_values s on v.value_field = s.value_field 38 | 39 | ) 40 | 41 | select * 42 | from validation_errors 43 | 44 | {% endtest %} 45 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/sql/get_relations_by_pattern.sql: -------------------------------------------------------------------------------- 1 | {% macro get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} 2 | {{ return(adapter.dispatch('get_relations_by_pattern', 'dbt_utils')(schema_pattern, table_pattern, exclude, database)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} 6 | 7 | {%- call statement('get_tables', fetch_result=True) %} 8 | 9 | {{ dbt_utils.get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude, database) }} 10 | 11 | {%- endcall -%} 12 | 13 | {%- set table_list = load_result('get_tables') -%} 14 | 15 | {%- if table_list and table_list['table'] -%} 16 | {%- set tbl_relations = [] -%} 17 | {%- for row in table_list['table'] -%} 18 | {%- set tbl_relation = api.Relation.create( 19 | database=database, 20 | schema=row.table_schema, 21 | identifier=row.table_name, 22 | type=row.table_type 23 | ) -%} 24 | {%- do tbl_relations.append(tbl_relation) -%} 25 | {%- endfor -%} 26 | 27 | {{ return(tbl_relations) }} 28 | {%- else -%} 29 | {{ return([]) }} 30 | {%- endif -%} 31 | 32 | {% endmacro %} 33 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/table_shape/expect_table_columns_to_contain_set.sql: -------------------------------------------------------------------------------- 1 | {%- test expect_table_columns_to_contain_set(model, column_list, transform="upper") -%} 2 | {%- if execute -%} 3 | {%- set column_list = column_list | map(transform) | list -%} 4 | {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%} 5 | {%- set matching_columns = dbt_expectations._list_intersect(column_list, relation_column_names) -%} 6 | with relation_columns as ( 7 | 8 | {% for col_name in relation_column_names %} 9 | select cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as relation_column 10 | {% if not loop.last %}union all{% endif %} 11 | {% endfor %} 12 | ), 13 | input_columns as ( 14 | 15 | {% for col_name in column_list %} 16 | select cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as input_column 17 | {% if not loop.last %}union all{% endif %} 18 | {% endfor %} 19 | ) 20 | select * 21 | from 22 | input_columns i 23 | left join 24 | relation_columns r on r.relation_column = i.input_column 25 | where 26 | -- catch any column in input list that is not in the list of table columns 27 | r.relation_column is null 28 | {%- endif -%} 29 | {%- endtest -%} 30 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/aggregate_functions/expect_column_quantile_values_to_be_between.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_quantile_values_to_be_between(model, column_name, 2 | quantile, 3 | min_value=None, 4 | max_value=None, 5 | group_by=None, 6 | row_condition=None, 7 | strictly=False 8 | ) %} 9 | 10 | {% set expression %} 11 | {{ dbt_expectations.percentile_cont(column_name, quantile) }} 12 | {% endset %} 13 | {{ dbt_expectations.expression_between(model, 14 | expression=expression, 15 | min_value=min_value, 16 | max_value=max_value, 17 | group_by_columns=group_by, 18 | row_condition=row_condition, 19 | strictly=strictly 20 | ) }} 21 | {% endtest %} 22 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/schema_tests/fewer_rows_than.sql: -------------------------------------------------------------------------------- 1 | {% test fewer_rows_than(model, compare_model) %} 2 | {{ return(adapter.dispatch('test_fewer_rows_than', 'dbt_utils')(model, compare_model)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_fewer_rows_than(model, compare_model) %} 6 | 7 | {{ config(fail_calc = 'coalesce(row_count_delta, 0)') }} 8 | 9 | with a as ( 10 | 11 | select count(*) as count_our_model from {{ model }} 12 | 13 | ), 14 | b as ( 15 | 16 | select count(*) as count_comparison_model from {{ compare_model }} 17 | 18 | ), 19 | counts as ( 20 | 21 | select 22 | count_our_model, 23 | count_comparison_model 24 | from a 25 | cross join b 26 | 27 | ), 28 | final as ( 29 | 30 | select *, 31 | case 32 | -- fail the test if we have more rows than the reference model and return the row count delta 33 | when count_our_model > count_comparison_model then (count_our_model - count_comparison_model) 34 | -- fail the test if they are the same number 35 | when count_our_model = count_comparison_model then 1 36 | -- pass the test if the delta is positive (i.e. return the number 0) 37 | else 0 38 | end as row_count_delta 39 | from counts 40 | 41 | ) 42 | 43 | select * from final 44 | 45 | {% endmacro %} 46 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/multi-column/expect_column_pair_values_to_be_in_set.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_pair_values_to_be_in_set(model, 2 | column_A, 3 | column_B, 4 | value_pairs_set, 5 | row_condition=None 6 | ) %} 7 | 8 | {% set expression %} 9 | {% for pair in value_pairs_set %} 10 | {%- if (pair | length) == 2 %} 11 | ({{ column_A }} = {{ pair[0] }} and {{ column_B }} = {{ pair[1] }}){% if not loop.last %} or {% endif %} 12 | {% else %} 13 | {{ exceptions.raise_compiler_error( 14 | "`value_pairs_set` argument for expect_column_pair_values_to_be_in_set test cannot have more than 2 item per element. 15 | Got: '" ~ pair ~ "'.'" 16 | ) }} 17 | {% endif %} 18 | {% endfor %} 19 | {% endset %} 20 | {{ dbt_expectations.expression_is_true(model, 21 | expression=expression, 22 | group_by_columns=None, 23 | row_condition=row_condition 24 | ) 25 | }} 26 | 27 | {% endtest %} 28 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/column_values_basic/expect_column_values_to_be_in_set.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_values_to_be_in_set(model, column_name, 2 | value_set, 3 | quote_values=True, 4 | row_condition=None 5 | ) %} 6 | 7 | with all_values as ( 8 | 9 | select 10 | {{ column_name }} as value_field 11 | 12 | from {{ model }} 13 | {% if row_condition %} 14 | where {{ row_condition }} 15 | {% endif %} 16 | 17 | ), 18 | set_values as ( 19 | 20 | {% for value in value_set -%} 21 | select 22 | {% if quote_values -%} 23 | cast('{{ value }}' as {{ dbt_utils.type_string() }}) 24 | {%- else -%} 25 | {{ value }} 26 | {%- endif %} as value_field 27 | {% if not loop.last %}union all{% endif %} 28 | {% endfor %} 29 | ), 30 | validation_errors as ( 31 | -- values from the model that are not in the set 32 | select 33 | v.value_field 34 | from 35 | all_values v 36 | left join 37 | set_values s on v.value_field = s.value_field 38 | where 39 | s.value_field is null 40 | 41 | ) 42 | 43 | select * 44 | from validation_errors 45 | 46 | {% endtest %} 47 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/.github/ISSUE_TEMPLATE/dbt_minor_release.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: dbt Minor Release Follow-Up 3 | about: A checklist of tasks to complete after a minor release is made to dbt 4 | title: 'dbt Minor Release Follow up for dbt v0.x.0' 5 | labels: 6 | assignees: '' 7 | --- 8 | 9 | 13 | 14 | First, check if this is a breaking change 15 | - [ ] Increase the upper bound of the `require-dbt-version` config in the `dbt_project.yml` 16 | - [ ] Increase the upper bound of the dbt version in `run_test.sh` 17 | - [ ] Create a PR against the `master` branch to see if tests pass 18 | 19 | If test pass, this is _not_ a breaking change. You should: 20 | - [ ] Merge into `master` 21 | - [ ] Create a patch release 22 | 23 | If tests fail, this _is_ a breaking change. You'll need to create a minor release: 24 | - [ ] Change the PR base to be against the next `dev` branch. 25 | - [ ] Increase the lower bound to the current dbt minor version in both the `dbt_project.yml` and `run_test.sh` files 26 | - [ ] Fix any errors 27 | - [ ] Merge `dev` into `master` 28 | - [ ] Create a minor release 29 | - [ ] Once the release is available on hub, [create a new issue](https://github.com/fishtown-analytics/dbt-utils/issues/new/choose) using the "dbt-utils Minor Release Checklist" template 30 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | This is a: 2 | - [ ] bug fix PR with no breaking changes — please ensure the base branch is `master` 3 | - [ ] new functionality — please ensure the base branch is the latest `dev/` branch 4 | - [ ] a breaking change — please ensure the base branch is the latest `dev/` branch 5 | 6 | ## Description & motivation 7 | 10 | 11 | ## Checklist 12 | - [ ] I have verified that these changes work locally on the following warehouses (Note: it's okay if you do not have access to all warehouses, this helps us understand what has been covered) 13 | - [ ] BigQuery 14 | - [ ] Postgres 15 | - [ ] Redshift 16 | - [ ] Snowflake 17 | - [ ] I followed guidelines to ensure that my changes will work on "non-core" adapters by: 18 | - [ ] dispatching any new macro(s) so non-core adapters can also use them (e.g. [the `star()` source](https://github.com/fishtown-analytics/dbt-utils/blob/master/macros/sql/star.sql)) 19 | - [ ] using the `limit_zero()` macro in place of the literal string: `limit 0` 20 | - [ ] using `dbt_utils.type_*` macros instead of explicit datatypes (e.g. `dbt_utils.type_timestamp()` instead of `TIMESTAMP` 21 | - [ ] I have updated the README.md (if applicable) 22 | - [ ] I have added tests & descriptions to my models (and macros if applicable) 23 | - [ ] I have added an entry to CHANGELOG.md 24 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/cross_db_utils/last_day.sql: -------------------------------------------------------------------------------- 1 | /* 2 | This function has been tested with dateparts of month and quarters. Further 3 | testing is required to validate that it will work on other dateparts. 4 | */ 5 | 6 | {% macro last_day(date, datepart) %} 7 | {{ return(adapter.dispatch('last_day', 'dbt_utils') (date, datepart)) }} 8 | {% endmacro %} 9 | 10 | 11 | {%- macro default_last_day(date, datepart) -%} 12 | cast( 13 | {{dbt_utils.dateadd('day', '-1', 14 | dbt_utils.dateadd(datepart, '1', dbt_utils.date_trunc(datepart, date)) 15 | )}} 16 | as date) 17 | {%- endmacro -%} 18 | 19 | 20 | {% macro default__last_day(date, datepart) -%} 21 | {{dbt_utils.default_last_day(date, datepart)}} 22 | {%- endmacro %} 23 | 24 | 25 | {% macro postgres__last_day(date, datepart) -%} 26 | 27 | {%- if datepart == 'quarter' -%} 28 | -- postgres dateadd does not support quarter interval. 29 | cast( 30 | {{dbt_utils.dateadd('day', '-1', 31 | dbt_utils.dateadd('month', '3', dbt_utils.date_trunc(datepart, date)) 32 | )}} 33 | as date) 34 | {%- else -%} 35 | {{dbt_utils.default_last_day(date, datepart)}} 36 | {%- endif -%} 37 | 38 | {%- endmacro %} 39 | 40 | {# redshift should use default instead of postgres #} 41 | {% macro redshift__last_day(date, datepart) %} 42 | 43 | {{ return(dbt_utils.default__last_day(date, datepart)) }} 44 | 45 | {% endmacro %} 46 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/table_shape/expect_table_columns_to_match_set.sql: -------------------------------------------------------------------------------- 1 | {%- test expect_table_columns_to_match_set(model, column_list, transform="upper") -%} 2 | {%- if execute -%} 3 | {%- set column_list = column_list | map(transform) | list -%} 4 | {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%} 5 | {%- set matching_columns = dbt_expectations._list_intersect(column_list, relation_column_names) -%} 6 | with relation_columns as ( 7 | 8 | {% for col_name in relation_column_names %} 9 | select cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as relation_column 10 | {% if not loop.last %}union all{% endif %} 11 | {% endfor %} 12 | ), 13 | input_columns as ( 14 | 15 | {% for col_name in column_list %} 16 | select cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as input_column 17 | {% if not loop.last %}union all{% endif %} 18 | {% endfor %} 19 | ) 20 | select * 21 | from 22 | relation_columns r 23 | full outer join 24 | input_columns i on r.relation_column = i.input_column 25 | where 26 | -- catch any column in input list that is not in the list of table columns 27 | -- or any table column that is not in the input list 28 | r.relation_column is null or 29 | i.input_column is null 30 | 31 | {%- endif -%} 32 | {%- endtest -%} 33 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/sql/generate_series.sql: -------------------------------------------------------------------------------- 1 | {% macro get_powers_of_two(upper_bound) %} 2 | {{ return(adapter.dispatch('get_powers_of_two', 'dbt_utils')(upper_bound)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_powers_of_two(upper_bound) %} 6 | 7 | {% if upper_bound <= 0 %} 8 | {{ exceptions.raise_compiler_error("upper bound must be positive") }} 9 | {% endif %} 10 | 11 | {% for _ in range(1, 100) %} 12 | {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %} 13 | {% endfor %} 14 | 15 | {% endmacro %} 16 | 17 | 18 | {% macro generate_series(upper_bound) %} 19 | {{ return(adapter.dispatch('generate_series', 'dbt_utils')(upper_bound)) }} 20 | {% endmacro %} 21 | 22 | {% macro default__generate_series(upper_bound) %} 23 | 24 | {% set n = dbt_utils.get_powers_of_two(upper_bound) %} 25 | 26 | with p as ( 27 | select 0 as generated_number union all select 1 28 | ), unioned as ( 29 | 30 | select 31 | 32 | {% for i in range(n) %} 33 | p{{i}}.generated_number * power(2, {{i}}) 34 | {% if not loop.last %} + {% endif %} 35 | {% endfor %} 36 | + 1 37 | as generated_number 38 | 39 | from 40 | 41 | {% for i in range(n) %} 42 | p as p{{i}} 43 | {% if not loop.last %} cross join {% endif %} 44 | {% endfor %} 45 | 46 | ) 47 | 48 | select * 49 | from unioned 50 | where generated_number <= {{upper_bound}} 51 | order by generated_number 52 | 53 | {% endmacro %} 54 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_be_in_set.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_distinct_values_to_be_in_set(model, column_name, 2 | value_set, 3 | quote_values=False, 4 | row_condition=None 5 | ) %} 6 | 7 | with all_values as ( 8 | 9 | select distinct 10 | {{ column_name }} as value_field 11 | 12 | from {{ model }} 13 | {% if row_condition %} 14 | where {{ row_condition }} 15 | {% endif %} 16 | 17 | ), 18 | set_values as ( 19 | 20 | {% for value in value_set -%} 21 | select 22 | {% if quote_values -%} 23 | '{{ value }}' 24 | {%- else -%} 25 | {{ value }} 26 | {%- endif %} as value_field 27 | {% if not loop.last %}union all{% endif %} 28 | {% endfor %} 29 | 30 | ), 31 | unique_set_values as ( 32 | 33 | select distinct value_field 34 | from 35 | set_values 36 | 37 | ), 38 | validation_errors as ( 39 | -- values from the model that are not in the set 40 | select 41 | v.value_field 42 | from 43 | all_values v 44 | left join 45 | unique_set_values s on v.value_field = s.value_field 46 | where 47 | s.value_field is null 48 | 49 | ) 50 | 51 | select * 52 | from validation_errors 53 | 54 | {% endtest %} 55 | -------------------------------------------------------------------------------- /dbt_packages/dbt_date/macros/calendar_date/convert_timezone.sql: -------------------------------------------------------------------------------- 1 | {%- macro convert_timezone(column, target_tz=None, source_tz=None) -%} 2 | {%- set source_tz = "UTC" if not source_tz else source_tz -%} 3 | {%- set target_tz = var("dbt_date:time_zone") if not target_tz else target_tz -%} 4 | {{ adapter.dispatch('convert_timezone', 'dbt_date') (column, target_tz, source_tz) }} 5 | {%- endmacro -%} 6 | 7 | {% macro default__convert_timezone(column, target_tz, source_tz) -%} 8 | {%- if not source_tz -%} 9 | cast(convert_timezone('{{ target_tz }}', {{ column }}) as {{ dbt_utils.type_timestamp() }}) 10 | {%- else -%} 11 | cast(convert_timezone('{{ source_tz }}', '{{ target_tz }}', {{ column }}) as {{ dbt_utils.type_timestamp() }}) 12 | {%- endif -%} 13 | {%- endmacro -%} 14 | 15 | {%- macro bigquery__convert_timezone(column, target_tz, source_tz=None) -%} 16 | timestamp(datetime({{ column }}, '{{ target_tz}}')) 17 | {%- endmacro -%} 18 | 19 | {%- macro spark__convert_timezone(column, target_tz, source_tz) -%} 20 | from_utc_timestamp( 21 | to_utc_timestamp({{ column }}, '{{ source_tz }}'), 22 | '{{ target_tz }}' 23 | ) 24 | {%- endmacro -%} 25 | 26 | {% macro postgres__convert_timezone(column, target_tz, source_tz) -%} 27 | {%- if source_tz -%} 28 | cast({{ column }} at time zone '{{ source_tz }}' at time zone '{{ target_tz }}' as {{ dbt_utils.type_timestamp() }}) 29 | {%- else -%} 30 | cast({{ column }} at time zone '{{ target_tz }}' as {{ dbt_utils.type_timestamp() }}) 31 | {%- endif -%} 32 | {%- endmacro -%} 33 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_equal_set.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_distinct_values_to_equal_set(model, column_name, 2 | value_set, 3 | quote_values=True, 4 | row_condition=None 5 | ) %} 6 | 7 | with all_values as ( 8 | 9 | select distinct 10 | {{ column_name }} as column_value 11 | 12 | from {{ model }} 13 | {% if row_condition %} 14 | where {{ row_condition }} 15 | {% endif %} 16 | 17 | ), 18 | set_values as ( 19 | 20 | {% for value in value_set -%} 21 | select 22 | {% if quote_values -%} 23 | '{{ value }}' 24 | {%- else -%} 25 | {{ value }} 26 | {%- endif %} as value_field 27 | {% if not loop.last %}union all{% endif %} 28 | {% endfor %} 29 | 30 | ), 31 | unique_set_values as ( 32 | 33 | select distinct value_field 34 | from 35 | set_values 36 | 37 | ), 38 | validation_errors as ( 39 | 40 | select 41 | * 42 | from 43 | all_values v 44 | full outer join 45 | unique_set_values s on v.column_value = s.value_field 46 | where 47 | v.column_value is null or 48 | s.value_field is null 49 | 50 | ) 51 | 52 | select * 53 | from validation_errors 54 | 55 | {% endtest %} 56 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/aggregate_functions/expect_column_distinct_values_to_contain_set.sql: -------------------------------------------------------------------------------- 1 | {% test expect_column_distinct_values_to_contain_set(model, column_name, 2 | value_set, 3 | quote_values=True, 4 | row_condition=None 5 | ) %} 6 | 7 | with all_values as ( 8 | 9 | select distinct 10 | {{ column_name }} as value_field 11 | 12 | from {{ model }} 13 | {% if row_condition %} 14 | where {{ row_condition }} 15 | {% endif %} 16 | 17 | ), 18 | set_values as ( 19 | 20 | {% for value in value_set -%} 21 | select 22 | {% if quote_values -%} 23 | '{{ value }}' 24 | {%- else -%} 25 | {{ value }} 26 | {%- endif %} as value_field 27 | {% if not loop.last %}union all{% endif %} 28 | {% endfor %} 29 | 30 | ), 31 | unique_set_values as ( 32 | 33 | select distinct value_field 34 | from 35 | set_values 36 | 37 | ), 38 | validation_errors as ( 39 | -- values in set that are not in the list of values from the model 40 | select 41 | s.value_field 42 | from 43 | unique_set_values s 44 | left join 45 | all_values v on s.value_field = v.value_field 46 | where 47 | v.value_field is null 48 | 49 | ) 50 | 51 | select * 52 | from validation_errors 53 | 54 | {% endtest %} 55 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/table_shape/expect_table_row_count_to_equal.sql: -------------------------------------------------------------------------------- 1 | {%- test expect_table_row_count_to_equal(model, 2 | value, 3 | group_by=None, 4 | row_condition=None 5 | ) -%} 6 | {{ adapter.dispatch('test_expect_table_row_count_to_equal', 7 | 'dbt_expectations') (model, 8 | value, 9 | group_by, 10 | row_condition 11 | ) }} 12 | {% endtest %} 13 | 14 | 15 | 16 | {%- macro default__test_expect_table_row_count_to_equal(model, 17 | value, 18 | group_by, 19 | row_condition 20 | ) -%} 21 | {% set expression %} 22 | count(*) = {{ value }} 23 | {% endset %} 24 | {{ dbt_expectations.expression_is_true(model, 25 | expression=expression, 26 | group_by_columns=group_by, 27 | row_condition=row_condition) 28 | }} 29 | {%- endmacro -%} 30 | -------------------------------------------------------------------------------- /dbt_project.yml: -------------------------------------------------------------------------------- 1 | 2 | # Name your project! Project names should contain only lowercase characters 3 | # and underscores. A good package name should reflect your organization's 4 | # name or the intended use of these models 5 | name: 'demo_getdbt' 6 | version: '1.0.0' 7 | config-version: 2 8 | 9 | # This setting configures which "profile" dbt uses for this project. 10 | profile: 'demo_getdbt' 11 | 12 | # These configurations specify where dbt should look for different types of files. 13 | # The `model-paths` config, for example, states that models in this project can be 14 | # found in the "models/" directory. You probably won't need to change these! 15 | model-paths: ["models"] 16 | analysis-paths: ["analyses"] 17 | test-paths: ["tests"] 18 | seed-paths: ["seeds"] 19 | macro-paths: ["macros"] 20 | snapshot-paths: ["snapshots"] 21 | 22 | target-path: "target" # directory which will store compiled SQL files 23 | clean-targets: # directories to be removed by `dbt clean` 24 | - "target" 25 | - "dbt_packages" 26 | 27 | 28 | # Configuring models 29 | # Full documentation: https://docs.getdbt.com/docs/configuring-models 30 | 31 | # In this example config, we tell dbt to build all models in the example/ directory 32 | # as tables. These settings can be overridden in the individual model files 33 | # using the `{{ config(...) }}` macro. 34 | models: 35 | demo_getdbt: 36 | # Config indicated by + and applies to all files under models/example/ 37 | staging: 38 | +materialized: view 39 | datamart: 40 | +materialized: table 41 | 42 | vars: 43 | 'dbt_date:time_zone': 'America/Santiago' 44 | 45 | 46 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Report a bug or an issue you've found with this package 4 | title: '' 5 | labels: bug, triage 6 | assignees: '' 7 | 8 | --- 9 | 10 | ### Describe the bug 11 | 14 | 15 | ### Steps to reproduce 16 | 19 | 20 | ### Expected results 21 | 24 | 25 | ### Actual results 26 | 29 | 30 | ### Screenshots and log output 31 | 34 | 35 | ### System information 36 | **The contents of your `packages.yml` file:** 37 | 38 | **Which database are you using dbt with?** 39 | - [ ] postgres 40 | - [ ] redshift 41 | - [ ] bigquery 42 | - [ ] snowflake 43 | - [ ] other (specify: ____________) 44 | 45 | 46 | **The output of `dbt --version`:** 47 | ``` 48 | 49 | ``` 50 | 51 | 52 | ### Additional context 53 | 56 | 57 | ### Are you interested in contributing the fix? 58 | 61 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/table_shape/expect_table_columns_to_match_ordered_list.sql: -------------------------------------------------------------------------------- 1 | {%- test expect_table_columns_to_match_ordered_list(model, column_list, transform="upper") -%} 2 | {%- if execute -%} 3 | {%- set column_list = column_list | map(transform) | list -%} 4 | {%- set relation_column_names = dbt_expectations._get_column_list(model, transform) -%} 5 | {%- set matching_columns = dbt_expectations._list_intersect(column_list, relation_column_names) -%} 6 | with relation_columns as ( 7 | 8 | {% for col_name in relation_column_names %} 9 | select 10 | {{ loop.index }} as relation_column_idx, 11 | cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as relation_column 12 | {% if not loop.last %}union all{% endif %} 13 | {% endfor %} 14 | ), 15 | input_columns as ( 16 | 17 | {% for col_name in column_list %} 18 | select 19 | {{ loop.index }} as input_column_idx, 20 | cast('{{ col_name }}' as {{ dbt_utils.type_string() }}) as input_column 21 | {% if not loop.last %}union all{% endif %} 22 | {% endfor %} 23 | ) 24 | select * 25 | from 26 | relation_columns r 27 | full outer join 28 | input_columns i on r.relation_column = i.input_column and r.relation_column_idx = i.input_column_idx 29 | where 30 | -- catch any column in input list that is not in the sequence of table columns 31 | -- or any table column that is not in the input sequence 32 | r.relation_column is null or 33 | i.input_column is null 34 | 35 | {%- endif -%} 36 | {%- endtest -%} 37 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/macros/schema_tests/table_shape/expect_row_values_to_have_recent_data.sql: -------------------------------------------------------------------------------- 1 | {% test expect_row_values_to_have_recent_data(model, 2 | column_name, 3 | datepart, 4 | interval, 5 | row_condition=None) %} 6 | 7 | {{ adapter.dispatch('test_expect_row_values_to_have_recent_data', 'dbt_expectations') (model, 8 | column_name, 9 | datepart, 10 | interval, 11 | row_condition) }} 12 | 13 | {% endtest %} 14 | 15 | {% macro default__test_expect_row_values_to_have_recent_data(model, column_name, datepart, interval, row_condition) %} 16 | {%- set default_start_date = '1970-01-01' -%} 17 | with max_recency as ( 18 | 19 | select max({{ column_name }} ) as max_date 20 | from 21 | {{ model }} 22 | where 23 | {{ column_name }} <= {{ dbt_date.today() }} 24 | {% if row_condition %} 25 | and {{ row_condition }} 26 | {% endif %} 27 | ) 28 | select 29 | * 30 | from 31 | max_recency 32 | where 33 | -- if the row_condition excludes all row, we need to compare against a default date 34 | -- to avoid false negatives 35 | coalesce(max_date, '{{ default_start_date }}') 36 | < {{ dbt_utils.dateadd(datepart, interval * -1, dbt_date.now()) }} 37 | 38 | {% endmacro %} 39 | -------------------------------------------------------------------------------- /dbt_packages/dbt_utils/macros/sql/surrogate_key.sql: -------------------------------------------------------------------------------- 1 | {%- macro surrogate_key(field_list) -%} 2 | {# needed for safe_add to allow for non-keyword arguments see SO post #} 3 | {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} 4 | {% set frustrating_jinja_feature = varargs %} 5 | {{ return(adapter.dispatch('surrogate_key', 'dbt_utils')(field_list, *varargs)) }} 6 | {% endmacro %} 7 | 8 | {%- macro default__surrogate_key(field_list) -%} 9 | 10 | {%- if varargs|length >= 1 or field_list is string %} 11 | 12 | {%- set error_message = ' 13 | Warning: the `surrogate_key` macro now takes a single list argument instead of \ 14 | multiple string arguments. Support for multiple string arguments will be \ 15 | deprecated in a future release of dbt-utils. The {}.{} model triggered this warning. \ 16 | '.format(model.package_name, model.name) -%} 17 | 18 | {%- do exceptions.warn(error_message) -%} 19 | 20 | {# first argument is not included in varargs, so add first element to field_list_xf #} 21 | {%- set field_list_xf = [field_list] -%} 22 | 23 | {%- for field in varargs %} 24 | {%- set _ = field_list_xf.append(field) -%} 25 | {%- endfor -%} 26 | 27 | {%- else -%} 28 | 29 | {# if using list, just set field_list_xf as field_list #} 30 | {%- set field_list_xf = field_list -%} 31 | 32 | {%- endif -%} 33 | 34 | 35 | {%- set fields = [] -%} 36 | 37 | {%- for field in field_list_xf -%} 38 | 39 | {%- set _ = fields.append( 40 | "coalesce(cast(" ~ field ~ " as " ~ dbt_utils.type_string() ~ "), '')" 41 | ) -%} 42 | 43 | {%- if not loop.last %} 44 | {%- set _ = fields.append("'-'") -%} 45 | {%- endif -%} 46 | 47 | {%- endfor -%} 48 | 49 | {{dbt_utils.hash(dbt_utils.concat(fields))}} 50 | 51 | {%- endmacro -%} 52 | -------------------------------------------------------------------------------- /dbt_packages/dbt_expectations/integration_tests/ci/sample.profiles.yml: -------------------------------------------------------------------------------- 1 | 2 | # HEY! This file is used in the dbt-expectations integrations tests. 3 | # You should __NEVER__ check credentials into version control. Thanks for reading :) 4 | 5 | config: 6 | send_anonymous_usage_stats: False 7 | use_colors: True 8 | 9 | integration_tests: 10 | target: postgres 11 | outputs: 12 | postgres: 13 | type: postgres 14 | host: localhost 15 | user: "{{ env_var('CI_DBT_USER') }}" 16 | pass: "{{ env_var('CI_DBT_PASS') }}" 17 | port: "{{ env_var('CI_DBT_PORT') }}" 18 | dbname: "{{ env_var('CI_DBT_DBNAME') }}" 19 | schema: dbt_expectations_integration_tests 20 | threads: 1 21 | 22 | redshift: 23 | type: redshift 24 | host: "{{ env_var('CI_REDSHIFT_DBT_HOST') }}" 25 | user: "{{ env_var('CI_REDSHIFT_DBT_USER') }}" 26 | pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" 27 | dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" 28 | port: 5439 29 | schema: dbt_expectations_integration_tests 30 | threads: 1 31 | 32 | bigquery: 33 | type: bigquery 34 | method: service-account 35 | keyfile: "{{ env_var('GCLOUD_SERVICE_KEY_PATH') }}" 36 | project: 'dbt-integration-tests' 37 | schema: dbt_expectations_integration_tests 38 | threads: 1 39 | 40 | snowflake: 41 | type: snowflake 42 | account: "{{ env_var('CI_SNOWFLAKE_DBT_ACCOUNT') }}" 43 | user: "{{ env_var('CI_SNOWFLAKE_DBT_USER') }}" 44 | password: "{{ env_var('CI_SNOWFLAKE_DBT_PASS') }}" 45 | role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" 46 | database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" 47 | warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" 48 | schema: dbt_expectations_integration_tests 49 | threads: 1 50 | --------------------------------------------------------------------------------