├── .circleci └── config.yml ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── dbt_minor_release.md │ ├── feature_request.md │ └── utils_minor_release.md ├── pull_request_template.md └── workflows │ ├── ci.yml │ ├── create-table-of-contents.yml │ ├── stale.yml │ └── triage-labels.yml ├── .gitignore ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── RELEASE.md ├── dbt_project.yml ├── dev-requirements.txt ├── docker-compose.yml ├── docs └── decisions │ ├── README.md │ ├── adr-0000-documenting-architecture-decisions.md │ ├── adr-0001-decision-record-format.md │ └── adr-0002-cross-database-utils.md ├── integration_tests ├── .env │ ├── bigquery.env │ ├── postgres.env │ ├── redshift.env │ └── snowflake.env ├── .gitignore ├── README.md ├── data │ ├── .gitkeep │ ├── datetime │ │ └── data_date_spine.csv │ ├── etc │ │ └── data_people.csv │ ├── geo │ │ ├── data_haversine_km.csv │ │ └── data_haversine_mi.csv │ ├── schema_tests │ │ ├── data_cardinality_equality_a.csv │ │ ├── data_cardinality_equality_b.csv │ │ ├── data_not_null_proportion.csv │ │ ├── data_test_accepted_range.csv │ │ ├── data_test_at_least_one.csv │ │ ├── data_test_equal_rowcount.csv │ │ ├── data_test_equality_a.csv │ │ ├── data_test_equality_b.csv │ │ ├── data_test_equality_floats_a.csv │ │ ├── data_test_equality_floats_b.csv │ │ ├── data_test_equality_floats_columns_a.csv │ │ ├── data_test_equality_floats_columns_b.csv │ │ ├── data_test_expression_is_true.csv │ │ ├── data_test_fewer_rows_than_table_1.csv │ │ ├── data_test_fewer_rows_than_table_2.csv │ │ ├── data_test_mutually_exclusive_ranges_no_gaps.csv │ │ ├── data_test_mutually_exclusive_ranges_with_gaps.csv │ │ ├── data_test_mutually_exclusive_ranges_with_gaps_zero_length.csv │ │ ├── data_test_not_accepted_values.csv │ │ ├── data_test_not_constant.csv │ │ ├── data_test_relationships_where_table_1.csv │ │ ├── data_test_relationships_where_table_2.csv │ │ ├── data_test_sequential_timestamps.csv │ │ ├── data_test_sequential_values.csv │ │ ├── data_unique_combination_of_columns.csv │ │ └── schema.yml │ ├── sql │ │ ├── data_deduplicate.csv │ │ ├── data_deduplicate_expected.csv │ │ ├── data_events_20180101.csv │ │ ├── data_events_20180102.csv │ │ ├── data_events_20180103.csv │ │ ├── data_filtered_columns_in_relation.csv │ │ ├── data_filtered_columns_in_relation_expected.csv │ │ ├── data_generate_series.csv │ │ ├── data_generate_surrogate_key.csv │ │ ├── data_get_column_values.csv │ │ ├── data_get_column_values_dropped.csv │ │ ├── data_get_column_values_where.csv │ │ ├── data_get_column_values_where_expected.csv │ │ ├── data_get_query_results_as_dict.csv │ │ ├── data_get_single_value.csv │ │ ├── data_nullcheck_table.csv │ │ ├── data_pivot.csv │ │ ├── data_pivot_expected.csv │ │ ├── data_pivot_expected_apostrophe.csv │ │ ├── data_safe_add.csv │ │ ├── data_safe_divide.csv │ │ ├── data_safe_divide_denominator_expressions.csv │ │ ├── data_safe_divide_numerator_expressions.csv │ │ ├── data_safe_subtract.csv │ │ ├── data_star.csv │ │ ├── data_star_aggregate.csv │ │ ├── data_star_aggregate_expected.csv │ │ ├── data_star_expected.csv │ │ ├── data_star_prefix_suffix_expected.csv │ │ ├── data_star_quote_identifiers.csv │ │ ├── data_union_events_expected.csv │ │ ├── data_union_exclude_expected.csv │ │ ├── data_union_expected.csv │ │ ├── data_union_table_1.csv │ │ ├── data_union_table_2.csv │ │ ├── data_unpivot.csv │ │ ├── data_unpivot_bool.csv │ │ ├── data_unpivot_bool_expected.csv │ │ ├── data_unpivot_expected.csv │ │ ├── data_unpivot_original_api_expected.csv │ │ ├── data_unpivot_quote.csv │ │ ├── data_unpivot_quote_expected.csv │ │ └── data_width_bucket.csv │ └── web │ │ ├── data_url_host.csv │ │ ├── data_url_path.csv │ │ └── data_urls.csv ├── dbt_project.yml ├── macros │ ├── .gitkeep │ ├── assert_equal_values.sql │ ├── limit_zero.sql │ └── tests.sql ├── models │ ├── datetime │ │ ├── schema.yml │ │ └── test_date_spine.sql │ ├── generic_tests │ │ ├── equality_less_columns.sql │ │ ├── recency_time_excluded.sql │ │ ├── recency_time_included.sql │ │ ├── schema.yml │ │ ├── test_equal_column_subset.sql │ │ ├── test_equal_rowcount.sql │ │ └── test_fewer_rows_than.sql │ ├── geo │ │ ├── schema.yml │ │ ├── test_haversine_distance_km.sql │ │ └── test_haversine_distance_mi.sql │ ├── sql │ │ ├── schema.yml │ │ ├── test_deduplicate.sql │ │ ├── test_generate_series.sql │ │ ├── test_generate_surrogate_key.sql │ │ ├── test_get_column_values.sql │ │ ├── test_get_column_values_where.sql │ │ ├── test_get_filtered_columns_in_relation.sql │ │ ├── test_get_relations_by_pattern.sql │ │ ├── test_get_relations_by_prefix_and_union.sql │ │ ├── test_get_single_value.sql │ │ ├── test_get_single_value_default.sql │ │ ├── test_groupby.sql │ │ ├── test_not_empty_string_failing.sql │ │ ├── test_not_empty_string_passing.sql │ │ ├── test_nullcheck_table.sql │ │ ├── test_pivot.sql │ │ ├── test_pivot_apostrophe.sql │ │ ├── test_safe_add.sql │ │ ├── test_safe_divide.sql │ │ ├── test_safe_subtract.sql │ │ ├── test_star.sql │ │ ├── test_star_aggregate.sql │ │ ├── test_star_no_columns.sql │ │ ├── test_star_prefix_suffix.sql │ │ ├── test_star_quote_identifiers.sql │ │ ├── test_star_uppercase.sql │ │ ├── test_union.sql │ │ ├── test_union_base.sql │ │ ├── test_union_exclude_base_lowercase.sql │ │ ├── test_union_exclude_base_uppercase.sql │ │ ├── test_union_exclude_lowercase.sql │ │ ├── test_union_exclude_uppercase.sql │ │ ├── test_union_no_source_column.sql │ │ ├── test_union_where.sql │ │ ├── test_union_where_base.sql │ │ ├── test_unpivot.sql │ │ ├── test_unpivot_bool.sql │ │ ├── test_unpivot_quote.sql │ │ └── test_width_bucket.sql │ └── web │ │ ├── schema.yml │ │ ├── test_url_host.sql │ │ ├── test_url_path.sql │ │ └── test_urls.sql ├── package-lock.yml ├── packages.yml ├── profiles.yml └── tests │ ├── assert_get_query_results_as_dict_objects_equal.sql │ ├── generic │ └── expect_table_columns_to_match_set.sql │ ├── jinja_helpers │ ├── assert_pretty_output_msg_is_string.sql │ ├── assert_pretty_time_is_string.sql │ └── test_slugify.sql │ └── sql │ ├── test_get_column_values_use_default.sql │ └── test_get_single_value_multiple_rows.sql ├── macros ├── generic_tests │ ├── accepted_range.sql │ ├── at_least_one.sql │ ├── cardinality_equality.sql │ ├── equal_rowcount.sql │ ├── equality.sql │ ├── expression_is_true.sql │ ├── fewer_rows_than.sql │ ├── mutually_exclusive_ranges.sql │ ├── not_accepted_values.sql │ ├── not_constant.sql │ ├── not_empty_string.sql │ ├── not_null_proportion.sql │ ├── recency.sql │ ├── relationships_where.sql │ ├── sequential_values.sql │ └── unique_combination_of_columns.sql ├── jinja_helpers │ ├── _is_ephemeral.sql │ ├── _is_relation.sql │ ├── log_info.sql │ ├── pretty_log_format.sql │ ├── pretty_time.sql │ └── slugify.sql ├── sql │ ├── date_spine.sql │ ├── deduplicate.sql │ ├── generate_series.sql │ ├── generate_surrogate_key.sql │ ├── get_column_values.sql │ ├── get_filtered_columns_in_relation.sql │ ├── get_query_results_as_dict.sql │ ├── get_relations_by_pattern.sql │ ├── get_relations_by_prefix.sql │ ├── get_single_value.sql │ ├── get_table_types_sql.sql │ ├── get_tables_by_pattern_sql.sql │ ├── get_tables_by_prefix_sql.sql │ ├── groupby.sql │ ├── haversine_distance.sql │ ├── nullcheck.sql │ ├── nullcheck_table.sql │ ├── pivot.sql │ ├── safe_add.sql │ ├── safe_divide.sql │ ├── safe_subtract.sql │ ├── star.sql │ ├── surrogate_key.sql │ ├── union.sql │ ├── unpivot.sql │ └── width_bucket.sql └── web │ ├── get_url_host.sql │ ├── get_url_parameter.sql │ └── get_url_path.sql ├── pytest.ini ├── run_functional_test.sh ├── run_test.sh ├── supported_adapters.env └── tox.ini /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2.1 3 | 4 | jobs: 5 | 6 | integration-postgres: 7 | docker: 8 | - image: cimg/python:3.9 9 | - image: cimg/postgres:9.6 10 | environment: 11 | POSTGRES_USER: root 12 | environment: 13 | POSTGRES_HOST: localhost 14 | POSTGRES_USER: root 15 | DBT_ENV_SECRET_POSTGRES_PASS: '' 16 | POSTGRES_PORT: 5432 17 | POSTGRES_DATABASE: circle_test 18 | POSTGRES_SCHEMA: dbt_utils_integration_tests_postgres 19 | 20 | steps: 21 | - checkout 22 | - run: pip install --pre dbt-postgres -r dev-requirements.txt 23 | - run: 24 | name: "Run OG Tests - Postgres" 25 | command: ./run_test.sh postgres 26 | - store_artifacts: 27 | path: integration_tests/logs 28 | - store_artifacts: 29 | path: integration_tests/target 30 | 31 | # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass 32 | resource_class: large 33 | 34 | integration-redshift: 35 | docker: 36 | - image: cimg/python:3.9 37 | steps: 38 | - checkout 39 | - run: pip install --pre dbt-redshift -r dev-requirements.txt 40 | - run: 41 | name: "Run OG Tests - Redshift" 42 | command: ./run_test.sh redshift 43 | - store_artifacts: 44 | path: integration_tests/logs 45 | - store_artifacts: 46 | path: integration_tests/target 47 | # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass 48 | resource_class: large 49 | 50 | integration-snowflake: 51 | docker: 52 | - image: cimg/python:3.9 53 | steps: 54 | - checkout 55 | - run: pip install --pre dbt-snowflake -r dev-requirements.txt 56 | - run: 57 | name: "Run OG Tests - Snowflake" 58 | command: ./run_test.sh snowflake 59 | - store_artifacts: 60 | path: integration_tests/logs 61 | - store_artifacts: 62 | path: integration_tests/target 63 | # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass 64 | resource_class: large 65 | 66 | integration-bigquery: 67 | environment: 68 | BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json" 69 | docker: 70 | - image: cimg/python:3.9 71 | steps: 72 | - checkout 73 | - run: pip install --pre dbt-bigquery -r dev-requirements.txt 74 | - run: 75 | name: Setup Environment Variables 76 | command: echo 'export BIGQUERY_KEYFILE_JSON="$BIGQUERY_SERVICE_ACCOUNT_JSON"' >> "$BASH_ENV" 77 | - run: 78 | name: "Run OG Tests - BigQuery" 79 | command: ./run_test.sh bigquery 80 | - store_artifacts: 81 | path: integration_tests/logs 82 | - store_artifacts: 83 | path: integration_tests/target 84 | # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass 85 | resource_class: large 86 | 87 | workflows: 88 | version: 2 89 | test-all: 90 | jobs: 91 | - integration-postgres: 92 | context: profile-postgres 93 | - integration-redshift: 94 | context: profile-redshift 95 | requires: 96 | - integration-postgres 97 | - integration-snowflake: 98 | context: profile-snowflake 99 | requires: 100 | - integration-postgres 101 | - integration-bigquery: 102 | context: profile-bigquery 103 | requires: 104 | - integration-postgres 105 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @dbt-labs/dbt-package-owners 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Report a bug or an issue you've found with this package 4 | title: '' 5 | labels: bug, triage 6 | assignees: '' 7 | 8 | --- 9 | 10 | ### Describe the bug 11 | 14 | 15 | ### Steps to reproduce 16 | 19 | 20 | ### Expected results 21 | 24 | 25 | ### Actual results 26 | 29 | 30 | ### Screenshots and log output 31 | 34 | 35 | ### System information 36 | **The contents of your `packages.yml` file:** 37 | 38 | **Which database are you using dbt with?** 39 | - [ ] postgres 40 | - [ ] redshift 41 | - [ ] bigquery 42 | - [ ] snowflake 43 | - [ ] other (specify: ____________) 44 | 45 | 46 | **The output of `dbt --version`:** 47 | ``` 48 | 49 | ``` 50 | 51 | 52 | ### Additional context 53 | 56 | 57 | ### Are you interested in contributing the fix? 58 | 61 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/dbt_minor_release.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: dbt Minor Release Follow-Up 3 | about: A checklist of tasks to complete after a minor release is made to dbt 4 | title: 'dbt Minor Release Follow up for dbt v0.x.0' 5 | labels: 6 | assignees: '' 7 | --- 8 | 9 | 13 | 14 | First, check if this is a breaking change 15 | - [ ] Increase the upper bound of the `require-dbt-version` config in the `dbt_project.yml` 16 | - [ ] Increase the upper bound of the dbt version in `run_test.sh` 17 | - [ ] Create a PR against the `main` branch to see if tests pass 18 | 19 | If test pass, this is _not_ a breaking change. You should: 20 | - [ ] Merge into `main` 21 | - [ ] Create a patch release 22 | 23 | If tests fail, this _is_ a breaking change. You'll need to create a minor release: 24 | - [ ] Change the PR base to be against the next `dev` branch. 25 | - [ ] Increase the lower bound to the current dbt minor version in both the `dbt_project.yml` and `run_test.sh` files 26 | - [ ] Fix any errors 27 | - [ ] Merge `dev` into `main` 28 | - [ ] Create a minor release 29 | - [ ] Once the release is available on hub, [create a new issue](https://github.com/dbt-labs/dbt-utils/issues/new/choose) using the "dbt-utils Minor Release Checklist" template 30 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this package 4 | title: '' 5 | labels: enhancement, triage 6 | assignees: '' 7 | 8 | --- 9 | 10 | ### Describe the feature 11 | A clear and concise description of what you want to happen. 12 | 13 | ### Describe alternatives you've considered 14 | A clear and concise description of any alternative solutions or features you've considered. 15 | 16 | ### Additional context 17 | Is this feature database-specific? Which database(s) is/are relevant? Please include any other relevant context here. 18 | 19 | ### Who will this benefit? 20 | What kind of use case will this feature be useful for? Please be specific and provide examples, this will help us prioritize properly. 21 | 22 | ### Are you interested in contributing this feature? 23 | 26 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/utils_minor_release.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: dbt-utils Minor Release Follow-up 3 | about: A checklist of tasks to complete after making a dbt-utils minor release 4 | title: 'dbt Minor Release Follow up for dbt-utils v0.x.0' 5 | labels: 6 | assignees: '' 7 | --- 8 | 9 | 13 | 14 | ## Process for each dependent package 15 | First, check if this is a breaking change 16 | - [ ] Increase the upper bound of the `dbt-utils` `version:` config in the `packages.yml` of the dependent package. 17 | - [ ] Push to a new branch to see if tests pass, or test locally. 18 | 19 | If this is _not_ a breaking change: 20 | - [ ] Create a patch release 21 | 22 | If this _is_ a breaking change: 23 | - [ ] Fix any breaking changes 24 | - [ ] Increase the lower bound to the current dbt-utils minor version 25 | - [ ] Create a minor release for the package 26 | 27 | ## Checklist of dependent packages 28 | | Package | PR | Release | 29 | |------------------------------------------------------------------------------|--------|-------------| 30 | | [audit-helper](https://github.com/dbt-labs/dbt-audit-helper) | [PR]() | [Release]() | 31 | | [codegen](https://github.com/dbt-labs/dbt-codegen) | [PR]() | [Release]() | 32 | | [redshift](https://github.com/dbt-labs/redshift) | [PR]() | [Release]() | 33 | | [event-logging](https://github.com/dbt-labs/dbt-event-logging) | [PR]() | [Release]() | 34 | | [snowplow](https://github.com/dbt-labs/snowplow) | [PR]() | [Release]() | 35 | | [external-tables](https://github.com/dbt-labs/dbt-external-tables) | [PR]() | [Release]() | 36 | | [segment](https://github.com/dbt-labs/segment) | [PR]() | [Release]() | 37 | | [facebook-ads](https://github.com/dbt-labs/facebook-ads) | [PR]() | [Release]() | 38 | | [stitch-utils](https://github.com/dbt-labs/stitch-utils) | [PR]() | [Release]() | 39 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | resolves # 2 | 3 | ### Problem 4 | 5 | 9 | 10 | ### Solution 11 | 12 | 17 | 18 | ## Checklist 19 | - [ ] This code is associated with an [issue](https://github.com/dbt-labs/dbt-utils/issues) which has been triaged and [accepted for development](https://docs.getdbt.com/docs/contributing/oss-expectations#pull-requests). 20 | - [ ] I have read [the contributing guide](https://github.com/dbt-labs/dbt-utils/blob/main/CONTRIBUTING.md) and understand what's expected of me 21 | - [ ] I have run this code in development and it appears to resolve the stated issue 22 | - [ ] This PR includes tests, or tests are not required/relevant for this PR 23 | - [ ] I have updated the README.md (if applicable) 24 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # **what?** 2 | # Run tests for dbt-utils against supported adapters 3 | 4 | # **why?** 5 | # To ensure that dbt-utils works as expected with all supported adapters 6 | 7 | # **when?** 8 | # On every PR, and every push to main and when manually triggered 9 | 10 | name: Package Integration Tests 11 | 12 | on: 13 | push: 14 | branches: 15 | - main 16 | pull_request_target: 17 | workflow_dispatch: 18 | 19 | 20 | jobs: 21 | run-tests: 22 | uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1 23 | with: 24 | # no need to pass postgres vars in. We can just use the defaults in the local container 25 | # redshift 26 | REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }} 27 | REDSHIFT_USER: ${{ vars.REDSHIFT_USER }} 28 | REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }} 29 | REDSHIFT_SCHEMA: "dbt_utils_integration_tests_redshift_${{ github.run_number }}" 30 | REDSHIFT_PORT: 5439 31 | # bigquery 32 | BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }} 33 | BIGQUERY_SCHEMA: "dbt_utils_integration_tests_bigquery_${{ github.run_number }}" 34 | # snowflake 35 | SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} 36 | SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE }} 37 | SNOWFLAKE_DATABASE: ${{ vars.SNOWFLAKE_DATABASE }} 38 | SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE }} 39 | SNOWFLAKE_SCHEMA: "dbt_utils_integration_tests_snowflake_${{ github.run_number }}" 40 | secrets: 41 | DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.REDSHIFT_PASS }} 42 | BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }} 43 | SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} 44 | DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.SNOWFLAKE_PASS }} 45 | -------------------------------------------------------------------------------- /.github/workflows/create-table-of-contents.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: Update table of contents 4 | 5 | # Controls when the workflow will run 6 | 7 | # Never! 8 | on: [] 9 | 10 | # Disabled by Doug Beatty on 2024-04-25 to fix CI 11 | # https://github.com/dbt-labs/dbt-utils/issues/885 12 | # on: 13 | # push: 14 | # branches: [main] 15 | # paths: ['README.md'] 16 | 17 | jobs: 18 | build: 19 | runs-on: ubuntu-latest 20 | timeout-minutes: 5 21 | steps: 22 | - uses: actions/checkout@v3 23 | - run: | 24 | curl https://raw.githubusercontent.com/ekalinin/github-markdown-toc/master/gh-md-toc -o gh-md-toc 25 | chmod a+x gh-md-toc 26 | ./gh-md-toc --insert --no-backup README.md 27 | rm ./gh-md-toc 28 | - uses: stefanzweifel/git-auto-commit-action@v4 29 | with: 30 | commit_message: Auto update table of contents 31 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | # **what?** 2 | # For issues that have been open for awhile without activity, label 3 | # them as stale with a warning that they will be closed out. If 4 | # anyone comments to keep the issue open, it will automatically 5 | # remove the stale label and keep it open. 6 | 7 | # Stale label rules: 8 | # awaiting_response, more_information_needed -> 90 days 9 | # good_first_issue, help_wanted -> 360 days (a year) 10 | # tech_debt -> 720 (2 years) 11 | # all else defaults -> 180 days (6 months) 12 | 13 | # **why?** 14 | # To keep the repo in a clean state from issues that aren't relevant anymore 15 | 16 | # **when?** 17 | # Once a day 18 | 19 | name: "Close stale issues and PRs" 20 | on: 21 | schedule: 22 | - cron: "30 1 * * *" 23 | 24 | permissions: 25 | issues: write 26 | pull-requests: write 27 | 28 | jobs: 29 | stale: 30 | uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main 31 | -------------------------------------------------------------------------------- /.github/workflows/triage-labels.yml: -------------------------------------------------------------------------------- 1 | # **what?** 2 | # When we triage issues, we sometimes need more information from the issue creator. In 3 | # those cases we remove the `triage` label and add the `awaiting_response` label. Once we 4 | # receive a response in the form of a comment, we want the `awaiting_response` label removed 5 | # in favor of the `triage` label so we are aware that the issue needs action. 6 | 7 | # **why?** 8 | # To help with out team triage issue tracking 9 | 10 | # **when?** 11 | # This will run when a comment is added to an issue and that issue has the `awaiting_response` label. 12 | 13 | name: Update Triage Label 14 | 15 | on: issue_comment 16 | 17 | defaults: 18 | run: 19 | shell: bash 20 | 21 | permissions: 22 | issues: write 23 | 24 | jobs: 25 | triage_label: 26 | if: contains(github.event.issue.labels.*.name, 'awaiting_response') 27 | uses: dbt-labs/actions/.github/workflows/swap-labels.yml@main 28 | with: 29 | add_label: "triage" 30 | remove_label: "awaiting_response" 31 | secrets: inherit 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | target/ 3 | dbt_modules/ 4 | dbt_packages/ 5 | logs/ 6 | venv/ 7 | __pycache__ 8 | .tox/ 9 | /.pytest_cache/ 10 | 11 | 12 | # Ignore all directories that start with 'env-' and can have any name after 13 | env*/ 14 | 15 | # Do not ignore .env files in any directory and do not ignore .env directories 16 | !.env 17 | !*/.env/ 18 | 19 | # But explicitly ignore test.env files 20 | test.env 21 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to `dbt-utils` 2 | 3 | `dbt-utils` is open source software. It is what it is today because community members have opened issues, provided feedback, and [contributed to the knowledge loop](https://www.getdbt.com/dbt-labs/values/). Whether you are a seasoned open source contributor or a first-time committer, we welcome and encourage you to contribute code, documentation, ideas, or problem statements to this project. 4 | 5 | Remember: all PRs (apart from cosmetic fixes like typos) should be [associated with an issue](https://docs.getdbt.com/docs/contributing/oss-expectations#pull-requests). 6 | 7 | 1. [About this document](#about-this-document) 8 | 1. [Getting the code](#getting-the-code) 9 | 1. [Setting up an environment](#setting-up-an-environment) 10 | 1. [Implementation guidelines](#implementation-guidelines) 11 | 1. [Testing dbt-utils](#testing) 12 | 1. [Adding CHANGELOG Entry](#adding-changelog-entry) 13 | 1. [Submitting a Pull Request](#submitting-a-pull-request) 14 | 15 | Enable greater collaboration by selecting ["Allow edits from maintainers"](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/allowing-changes-to-a-pull-request-branch-created-from-a-fork#enabling-repository-maintainer-permissions-on-existing-pull-requests) which will allow commits on your PR branch. 16 | 17 | ## About this document 18 | 19 | There are many ways to contribute to the ongoing development of `dbt-utils`, such as by participating in discussions and issues. We encourage you to first read our higher-level document: ["Expectations for Open Source Contributors"](https://docs.getdbt.com/docs/contributing/oss-expectations). 20 | 21 | The rest of this document serves as a more granular guide for contributing code changes to `dbt-utils` (this repository). It is not intended as a guide for using `dbt-utils`, and some pieces assume a level of familiarity with Python development (virtualenvs, `pip`, etc). Specific code snippets in this guide assume you are using macOS or Linux and are comfortable with the command line. 22 | 23 | ### Notes 24 | 25 | - **CLA:** Please note that anyone contributing code to `dbt-utils` must sign the [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements). If you are unable to sign the CLA, the `dbt-utils` maintainers will unfortunately be unable to merge any of your Pull Requests. We welcome you to participate in discussions, open issues, and comment on existing ones. 26 | - **Branches:** All pull requests from community contributors should target the `main` branch (default). If the change is needed as a patch for a version of `dbt-utils` that has already been released (or is already a release candidate), a maintainer will backport the changes in your PR to the relevant branch. 27 | 28 | ## Getting the code 29 | 30 | ### Installing git 31 | 32 | You will need `git` in order to download and modify the `dbt-utils` source code. On macOS, the best way to download git is to just install [Xcode](https://developer.apple.com/support/xcode/). 33 | 34 | ### External contributors 35 | 36 | If you are not a member of the `dbt-labs` GitHub organization, you can contribute to `dbt-utils` by forking the `dbt-utils` repository. For a detailed overview on forking, check out the [GitHub docs on forking](https://help.github.com/en/articles/fork-a-repo). In short, you will need to: 37 | 38 | 1. Fork the `dbt-utils` repository 39 | 2. Clone your fork locally 40 | 3. Check out a new branch for your proposed changes 41 | 4. Push changes to your fork 42 | 5. Open a pull request against `dbt-labs/dbt-utils` from your forked repository 43 | 44 | ### dbt Labs contributors 45 | 46 | If you are a member of the `dbt-labs` GitHub organization, you will have push access to the `dbt-utils` repo. Rather than forking `dbt-utils` to make your changes, just clone the repository, check out a new branch, and push directly to that branch. 47 | 48 | ## Setting up an environment 49 | 50 | There are some tools that will be helpful to you in developing locally. While this is the list relevant for `dbt-utils` development, many of these tools are used commonly across open-source python projects. 51 | 52 | ### Tools 53 | 54 | These are the tools used in `dbt-utils` development and testing: 55 | - [`make`](https://users.cs.duke.edu/~ola/courses/programming/Makefiles/Makefiles.html) to run multiple setup or test steps in combination. Don't worry too much, nobody _really_ understands how `make` works, and our Makefile aims to be super simple. 56 | - [CircleCI](https://circleci.com/) for automating tests and checks, once a PR is pushed to the `dbt-utils` repository 57 | 58 | A deep understanding of these tools in not required to effectively contribute to `dbt-utils`, but we recommend checking out the attached documentation if you're interested in learning more about each one. 59 | 60 | ## Implementation guidelines 61 | 62 | Ensure that changes will work on "non-core" adapters by: 63 | - dispatching any new macro(s) so non-core adapters can also use them (e.g. [the `star()` source](https://github.com/dbt-labs/dbt-utils/blob/main/macros/sql/star.sql)) 64 | - using the `limit_zero()` macro in place of the literal string: `limit 0` 65 | - using [`type_*` macros](https://docs.getdbt.com/reference/dbt-jinja-functions/cross-database-macros#data-type-functions) instead of explicit datatypes (e.g. [`type_timestamp()`](https://docs.getdbt.com/reference/dbt-jinja-functions/cross-database-macros#type_timestamp) instead of `TIMESTAMP` 66 | 67 | ## Testing 68 | 69 | Once you're able to manually test that your code change is working as expected, it's important to run existing automated tests, as well as adding some new ones. These tests will ensure that: 70 | - Your code changes do not unexpectedly break other established functionality 71 | - Your code changes can handle all known edge cases 72 | - The functionality you're adding will _keep_ working in the future 73 | 74 | See here for details for running existing integration tests and adding new ones: 75 | - [integration_tests/README.md](integration_tests/README.md) 76 | 77 | ## Adding CHANGELOG Entry 78 | 79 | We use [automatically generated release notes](https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes) to generate `CHANGELOG` entries. **Note:** Do not edit the `CHANGELOG.md` directly. Your modifications will be lost. 80 | 81 | You don't need to worry about which `dbt-utils` version your change will go into. Just create the changelog entry at the top of CHANGELOG.md and open your PR against the `main` branch. All merged changes will be included in the next minor version of `dbt-utils`. The maintainers _may_ choose to "backport" specific changes in order to patch older minor versions. In that case, a maintainer will take care of that backport after merging your PR, before releasing the new version of `dbt-utils`. 82 | 83 | ## Submitting a Pull Request 84 | 85 | A `dbt-utils` maintainer will review your PR. They may suggest code revision for style or clarity, or request that you add unit or integration test(s). These are good things! We believe that, with a little bit of help, anyone can contribute high-quality code. 86 | 87 | Automated tests run via CircleCI. If you're a first-time contributor, all tests (including code checks and unit tests) will require a maintainer to approve. Changes in the `dbt-utils` repository trigger integration tests. 88 | 89 | Once all tests are passing and your PR has been approved, a `dbt-utils` maintainer will merge your changes into the active development branch. And that's it! Happy developing :tada: 90 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .DEFAULT_GOAL:=help 2 | 3 | .PHONY: test 4 | test: ## Run the integration tests. 5 | @\ 6 | tox -e dbt_integration_$(target) 7 | 8 | .PHONY: dev 9 | dev: ## Installs dbt-* packages in develop mode along with development dependencies. 10 | @\ 11 | echo "Install dbt-$(target)..."; \ 12 | pip install --upgrade pip setuptools; \ 13 | pip install --pre "dbt-$(target)" -r dev-requirements.txt; 14 | 15 | .PHONY: setup-db 16 | setup-db: ## Setup Postgres database with docker-compose for system testing. 17 | @\ 18 | docker-compose up --detach postgres 19 | 20 | .PHONY: help 21 | help: ## Show this help message. 22 | @echo 'usage: make [target]' 23 | @echo 24 | @echo 'targets:' 25 | @grep -E '^[8+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 26 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # dbt-utils releases 2 | 3 | ## When do we release? 4 | There's a few scenarios that might prompt a release: 5 | 6 | | Scenario | Release type | 7 | |--------------------------------------------|--------------| 8 | | Breaking changes to existing macros | major | 9 | | New functionality | minor | 10 | | Fixes to existing macros | patch | 11 | 12 | ## Release process 13 | 14 | 1. Begin a new release by clicking [here](https://github.com/dbt-labs/dbt-utils/releases/new) 15 | 1. Click "Choose a tag", then paste your version number (with no "v" in the name), then click "Create new tag: x.y.z. on publish" 16 | - The “Release title” will be identical to the tag name 17 | 1. Click the "Generate release notes" button 18 | 1. Copy and paste the generated release notes into [`CHANGELOG.md`](https://github.com/dbt-labs/dbt-utils/blob/main/CHANGELOG.md?plain=1), reformat to match previous entries, commit, and merge into the `main` branch ([example](https://github.com/dbt-labs/dbt-utils/pull/901)) 19 | 1. Click the "Publish release" button 20 | - This will automatically create an "Assets" section containing: 21 | - Source code (zip) 22 | - Source code (tar.gz) 23 | 24 | ## Post-release 25 | 26 | 1. Delete the automatic Zapier post ([example of one intentionally not deleted](https://getdbt.slack.com/archives/CU4MRJ7QB/p1646272037304639)) and replace it with a custom post in the `#package-ecosystem` channel in “The Community Slack” using the content from the tagged release notes (but replace GitHub handles with Slack handles) ([example](https://getdbt.slack.com/archives/CU4MRJ7QB/p1649372590957309)) 27 | -------------------------------------------------------------------------------- /dbt_project.yml: -------------------------------------------------------------------------------- 1 | name: 'dbt_utils' 2 | version: '0.1.0' 3 | 4 | require-dbt-version: [">=1.3.0", "<2.0.0"] 5 | 6 | config-version: 2 7 | 8 | target-path: "target" 9 | clean-targets: ["target", "dbt_modules", "dbt_packages"] 10 | macro-paths: ["macros"] 11 | log-path: "logs" 12 | -------------------------------------------------------------------------------- /dev-requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | pytest-dotenv 3 | dbt-core@git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core 4 | dbt-tests-adapter@git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter 5 | dbt-postgres@git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-postgres 6 | dbt-redshift@git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-redshift 7 | dbt-snowflake@git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-snowflake 8 | dbt-bigquery@git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-bigquery 9 | pytest-xdist 10 | tox>=3.13 11 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | services: 3 | postgres: 4 | image: cimg/postgres:9.6 5 | environment: 6 | - POSTGRES_USER=root 7 | ports: 8 | - "5432:5432" 9 | -------------------------------------------------------------------------------- /docs/decisions/README.md: -------------------------------------------------------------------------------- 1 | ## ADRs 2 | 3 | For any architectural/engineering decisions we make, we will create an [ADR (Architectural Decision Record)](https://cognitect.com/blog/2011/11/15/documenting-architecture-decisions) to keep track of what decision we made and why. This allows us to refer back to decisions in the future and see if the reasons we made a choice still holds true. This also allows for others to more easily understand the code. ADRs will follow this process (or its replacement): 4 | - [adr-0000-documenting-architecture-decisions.md](adr-0000-documenting-architecture-decisions.md) 5 | -------------------------------------------------------------------------------- /docs/decisions/adr-0000-documenting-architecture-decisions.md: -------------------------------------------------------------------------------- 1 | Source: https://www.cognitect.com/blog/2011/11/15/documenting-architecture-decisions 2 | 3 | # DOCUMENTING ARCHITECTURE DECISIONS 4 | Michael Nygard - November 15, 2011 5 | 6 | ## CONTEXT 7 | Architecture for agile projects has to be described and defined differently. Not all decisions will be made at once, nor will all of them be done when the project begins. 8 | 9 | Agile methods are not opposed to documentation, only to valueless documentation. Documents that assist the team itself can have value, but only if they are kept up to date. Large documents are never kept up to date. Small, modular documents have at least a chance at being updated. 10 | 11 | Nobody ever reads large documents, either. Most developers have been on at least one project where the specification document was larger (in bytes) than the total source code size. Those documents are too large to open, read, or update. Bite sized pieces are easier for for all stakeholders to consume. 12 | 13 | One of the hardest things to track during the life of a project is the motivation behind certain decisions. A new person coming on to a project may be perplexed, baffled, delighted, or infuriated by some past decision. Without understanding the rationale or consequences, this person has only two choices: 14 | 15 | 1. **Blindly accept the decision.** 16 | This response may be OK, if the decision is still valid. It may not be good, however, if the context has changed and the decision should really be revisited. If the project accumulates too many decisions accepted without understanding, then the development team becomes afraid to change anything and the project collapses under its own weight. 17 | 18 | 2. **Blindly change it.** 19 | Again, this may be OK if the decision needs to be reversed. On the other hand, changing the decision without understanding its motivation or consequences could mean damaging the project's overall value without realizing it. (E.g., the decision supported a non-functional requirement that hasn't been tested yet.) 20 | 21 | It's better to avoid either blind acceptance or blind reversal. 22 | 23 | ## DECISION 24 | We will keep a collection of records for "architecturally significant" decisions: those that affect the structure, non-functional characteristics, dependencies, interfaces, or construction techniques. 25 | 26 | An architecture decision record is a short text file in a format similar to an Alexandrian pattern. (Though the decisions themselves are not necessarily patterns, they share the characteristic balancing of forces.) Each record describes a set of forces and a single decision in response to those forces. Note that the decision is the central piece here, so specific forces may appear in multiple ADRs. 27 | 28 | We will keep ADRs in the project repository under doc/arch/adr-NNN.md 29 | 30 | We should use a lightweight text formatting language like Markdown or Textile. 31 | 32 | ADRs will be numbered sequentially and monotonically. Numbers will not be reused. 33 | 34 | If a decision is reversed, we will keep the old one around, but mark it as superseded. (It's still relevant to know that it was the decision, but is no longer the decision.) 35 | 36 | We will use a format with just a few parts, so each document is easy to digest. The format has just a few parts. 37 | 38 | **Title** These documents have names that are short noun phrases. For example, "ADR 1: Deployment on Ruby on Rails 3.0.10" or "ADR 9: LDAP for Multitenant Integration" 39 | 40 | **Context** This section describes the forces at play, including technological, political, social, and project local. These forces are probably in tension, and should be called out as such. The language in this section is value-neutral. It is simply describing facts. 41 | 42 | **Decision** This section describes our response to these forces. It is stated in full sentences, with active voice. "We will …" 43 | 44 | **Status** A decision may be "proposed" if the project stakeholders haven't agreed with it yet, or "accepted" once it is agreed. If a later ADR changes or reverses a decision, it may be marked as "deprecated" or "superseded" with a reference to its replacement. 45 | 46 | **Consequences** This section describes the resulting context, after applying the decision. All consequences should be listed here, not just the "positive" ones. A particular decision may have positive, negative, and neutral consequences, but all of them affect the team and project in the future. 47 | 48 | The whole document should be one or two pages long. We will write each ADR as if it is a conversation with a future developer. This requires good writing style, with full sentences organized into paragraphs. Bullets are acceptable only for visual style, not as an excuse for writing sentence fragments. (Bullets kill people, even PowerPoint bullets.) 49 | 50 | ## STATUS 51 | Superseded by [ADR-0001](adr-0001-decision-record-format.md). 52 | 53 | ## CONSEQUENCES 54 | One ADR describes one significant decision for a specific project. It should be something that has an effect on how the rest of the project will run. 55 | 56 | The consequences of one ADR are very likely to become the context for subsequent ADRs. This is also similar to Alexander's idea of a pattern language: the large-scale responses create spaces for the smaller scale to fit into. 57 | 58 | Developers and project stakeholders can see the ADRs, even as the team composition changes over time. 59 | 60 | The motivation behind previous decisions is visible for everyone, present and future. Nobody is left scratching their heads to understand, "What were they thinking?" and the time to change old decisions will be clear from changes in the project's context. 61 | -------------------------------------------------------------------------------- /docs/decisions/adr-0001-decision-record-format.md: -------------------------------------------------------------------------------- 1 | # FORMAT AND STRUCTURE OF DECISION RECORDS 2 | 3 | ## CONTEXT 4 | We previousy decicded to record any decisions made in this project using Nygard's architecture decision record (ADR) format. Should we continue with this format or adopt an alternative? 5 | 6 | There are multiple options for formatting: 7 | * [MADR 3.0.0-beta.2](https://github.com/adr/madr/blob/3.0.0-beta.2/template/adr-template.md) – Markdown Any Decision Records 8 | * [Michael Nygard's template](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions) – What we are using currently 9 | * [Sustainable Architectural Decisions](https://www.infoq.com/articles/sustainable-architectural-design-decisions) – The Y-Statements 10 | * Other templates listed at 11 | 12 | If we choose to adopt a new format, we'll need to also choose whether to re-format previous decisions. The two main options are: 13 | 1. Keep the original formatting 14 | 1. Re-format all previous records according to MADR 15 | 16 | Keeping the original formatting would have the benefit of not altering Nygard's original post, which was adopted as-is for its elegant self-describing nature. It would have the downside of inconsistent formatting though. 17 | 18 | Re-formatting would resolve consistency at the cost of altering Nygard's original work. 19 | 20 | ## DECISION 21 | Chosen option: "MADR 3.0.0-beta.2", because 22 | 23 | * MADR is a matured version of the original ADR proposal that represents the state-of-the-art for ADR. 24 | * MADR has ongoing development and is maintained similar to a software project. 25 | * MADR explicitly uses Markdown, which is easy to read and write. 26 | * MADR 3.0 (optionally) contains structured elements in a YAML block for machine-readability. 27 | 28 | * MADR allows for structured capturing of any decision. 29 | * The MADR project is active and continues to iterate with new versions. 30 | * The MADR project itself is maintained like sofware with specifications and new versions. 31 | 32 | Choosen option: "keep original formatting", because it feels special and deserves to be celebrated, even if there is slight inconsistency of formatting as a result. This decision is easily reversible in the future, if need be. 33 | 34 | ## STATUS 35 | Accepted. 36 | 37 | ## CONSEQUENCES 38 | New decisions will follow the MADR 3.0.0-beta.2 format, and we will update this decision and following decisions once MADR 3.0.0 is officially released. However, previous decisions may retain the original Nygard format. All decision records will be renamed according to MADR conventions including moving from `doc/arch` to `docs/decisions`. 39 | -------------------------------------------------------------------------------- /docs/decisions/adr-0002-cross-database-utils.md: -------------------------------------------------------------------------------- 1 | --- 2 | status: accepted 3 | date: 2022-03-01 4 | deciders: Joel Labes and Jeremy Cohen 5 | consulted: dbt community 6 | informed: dbt community 7 | --- 8 | # The future of `dbt_utils` - break it into more logical chunks 9 | 10 | ## Context and Problem Statement 11 | 12 | `dbt_utils` is the most-used package in the [dbt Hub]() by a wide margin and it is installed in 1/3 of weekly active projects (as-of early 2022). The functionality with this package can be categorized into different use cases (each having their own rate of iteration): 13 | - Cross-database macros serve as a foundation to enable compatibility in other packages 14 | - Macros to abstract away complex work 15 | - Useful tests which aren't built into dbt Core 16 | - A catchall for experiments 17 | 18 | The `dbt_utils` package is doing a lot, and it could be split up into more logical chunks. If we pull out each category into a stand-alone package, they can each do their own thing without interfering with one another. 19 | 20 | How would this affect users, package maintainers, and adapter maintainers? 21 | 22 | ## Considered Options 23 | 24 | For each category of functionality, there are four main options for its future home: 25 | * stay in `dbt_utils` 26 | * move to its own stand-alone package or another existing repository (e.g. [dbt-expectations](https://github.com/calogica/dbt-expectations) in the case of tests, and [dbt-labs-experimental-features](https://github.com/dbt-labs/dbt-labs-experimental-features) in the case of experiments) 27 | * move to definition in Core, implementation in adapters 28 | * complete abandonment / deprecation 29 | 30 | Since there are four categories and 4 possibilities for destinations, that gives 4^4 = 256 unique options. Rather than enumerate all of them, we'll restrict discussion to a shorter list: 31 | 32 | * Migrate cross-db functions from `dbt_utils` to definition in Core, implementation in adapters 33 | * Split `dbt_utils` into multiple stand-alone packages 34 | * Keep `dbt_utils` as-is 35 | 36 | ## Decision Outcome 37 | 38 | Chosen option: "Migrate cross-db functions from `dbt_utils` to definition in Core, implementation in adapters", because 39 | that was the consensus that emerged from the discussion in [dbt-utils #487](https://github.com/dbt-labs/dbt-utils/discussions/487). 40 | 41 | Passthroughs will be left behind for migrated macros (so that calls to `dbt_utils.hash` don't suddenly start failing). New cross-database macros can be added in minor and major releases for dbt Core (but not patch releases). End users will retain the ability to use `dispatch` to shim/extend packages to adapters that don't yet support a particular macro. 42 | 43 | Additional decisions: 44 | - Keep tests and non-cross-database macros together in `dbt_utils` 45 | - Move experiments to a separate repo (i.e., the `load_by_period` macro) 46 | 47 | ## Validation 48 | 49 | Each moved macro will be validated by leaving a definition in `dbt_utils` and dispatching it to `dbt-core`. Independent continuous integration (CI) testing will exist within `dbt-core`, adapters, and `dbt_utils` using the [new pytest framework](https://docs.getdbt.com/docs/contributing/testing-a-new-adapter). 50 | 51 | ## Pros and Cons of the Options 52 | 53 | ### Definition in Core, implementation in adapters 54 | 55 | * Good, because common, reusable functionality that differs across databases will work "out of the box" 56 | * Good, because functionality can subjected to more [rigorous testing](https://docs.getdbt.com/docs/contributing/testing-a-new-adapter) 57 | * Good, because we hope that many package vendors could drop their dependencies on `dbt_utils` altogether, which makes version resolution easier 58 | * Good, because it's more convenient to reference the macro as `dateadd` instead of `dbt_utils.dateadd` (and `dbt.dateadd` is preserved as an option for those that appreciate an explicit namespace) 59 | * Good, because overriding global macros is more simple than overriding package macros 60 | * Good, because changes to macros are more clearly tied to `dbt-core` versions, rather than needing to worry about breaking changes in the matrix of `dbt-core` + `dbt_utils` minor versions 61 | * Good, because it establishes a precedent and pathway for battle-testing and maturing functionality before being promoted to Core 62 | * Neutral, because new cross-database macros will need to wait for the next minor (or major version) of `dbt-core` -- patch versions aren't an option 63 | * End users can use `dispatch` or the macro can be added to a release of `dbt_utils` until it is promoted to `dbt-core` 64 | * Bad, because **higher barrier to contribution** 65 | * to contribute to `dbt_utils` today, you just need to be a fairly skilled user of dbt. Even the integration tests are "just" a dbt project. To contribute to `dbt-core` or adapter plugins, you need to also know enough to set up a local development environment, to feel comfortable writing/updating Pythonic integration tests. 66 | * Bad, because unknown **maturity** 67 | * adding these macros into `dbt-core` "locks" them in. Changes to any macros may result in uglier code due to our commitment to backwards compatibility (e.g. addition of new arguments) 68 | * Bad, because less **macro discoverability** 69 | * Arguably, the macros in `dbt-core` are less discoverable than the ones in `dbt_utils`. This can be mitigated somewhat via significant manual effort over at [docs.getdbt.com](https://docs.getdbt.com/) 70 | * Bad, because less opportunity to **teach users about macros/packages** 71 | * The fact that so many projects install `dbt_utils` feels like a good thing — in the process, users are prompted to learn about packages (an essential dbt feature), explore other available packages, and realize that anything written in `dbt_utils` is something they fully have the power to write themselves, in their own projects. (That's not the case for most code in `dbt-core` + adapter plugins). In particular, users can write their own generic tests. We want to empower users to realize that they can write their own and not feel constrained by what's available out of the box. 72 | 73 | ### Split `dbt_utils` into multiple stand-alone packages 74 | 75 | * Good, because all the tests could be in one package, which would make the purpose of each package more clear and logically separated. 76 | * Bad, because it is easier to install a single package and then discover more functionality within it. It is non-trivial to search the whole hub for more packages which is a higher barrier than looking within a single `dbt_utils` package curated by dbt Labs. 77 | 78 | ### Keep `dbt_utils` as-is 79 | 80 | * Good, because we wouldn't have to do anything. 81 | * Good, because the user only has to install one package and gets a ton of functionality. 82 | * Bad, because it feels like the `dbt_utils` package is trying to do too much. 83 | * Bad, because each category of macros can't target their own users and dictate their own rate of iteration. 84 | 85 | ## More Information 86 | 87 | The initial public discussion is in [dbt-utils #487](https://github.com/dbt-labs/dbt-utils/discussions/487), and [dbt-core #4813](https://github.com/dbt-labs/dbt-core/issues/4813) captures the main story. 88 | -------------------------------------------------------------------------------- /integration_tests/.env/bigquery.env: -------------------------------------------------------------------------------- 1 | BIGQUERY_KEYFILE_JSON= 2 | BIGQUERY_PROJECT= 3 | BIGQUERY_SCHEMA=dbt_utils_integration_tests_bigquery 4 | -------------------------------------------------------------------------------- /integration_tests/.env/postgres.env: -------------------------------------------------------------------------------- 1 | POSTGRES_HOST=localhost 2 | POSTGRES_USER=root 3 | DBT_ENV_SECRET_POSTGRES_PASS=password 4 | POSTGRES_PORT=5432 5 | POSTGRES_DATABASE=dbt_utils_test 6 | POSTGRES_SCHEMA=dbt_utils_integration_tests_postgres 7 | -------------------------------------------------------------------------------- /integration_tests/.env/redshift.env: -------------------------------------------------------------------------------- 1 | REDSHIFT_HOST= 2 | REDSHIFT_USER= 3 | DBT_ENV_SECRET_REDSHIFT_PASS= 4 | REDSHIFT_DATABASE= 5 | REDSHIFT_PORT= 6 | REDSHIFT_SCHEMA=dbt_utils_integration_tests_redshift 7 | -------------------------------------------------------------------------------- /integration_tests/.env/snowflake.env: -------------------------------------------------------------------------------- 1 | SNOWFLAKE_ACCOUNT= 2 | SNOWFLAKE_USER= 3 | DBT_ENV_SECRET_SNOWFLAKE_PASS= 4 | SNOWFLAKE_ROLE= 5 | SNOWFLAKE_DATABASE= 6 | SNOWFLAKE_WAREHOUSE= 7 | SNOWFLAKE_SCHEMA=dbt_utils_integration_tests_snowflake 8 | -------------------------------------------------------------------------------- /integration_tests/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | dbt_modules/ 3 | logs/ 4 | .env/ 5 | profiles.yml 6 | package-lock.yml 7 | -------------------------------------------------------------------------------- /integration_tests/README.md: -------------------------------------------------------------------------------- 1 | ### Overview 2 | 1. Prerequisites 3 | 1. Configure credentials 4 | 1. Setup Postgres (optional) 5 | 1. Setup virtual environment 6 | 1. Installation for development 7 | 1. Run the integration tests 8 | 1. Run tests 9 | 1. Creating a new integration test 10 | 11 | ### Prerequisites 12 | - python3 13 | - Docker 14 | 15 | ### Configure credentials 16 | Edit the env file for your TARGET in `integration_tests/.env/[TARGET].env`. These will be used for your profiles.yml. 17 | 18 | Load the environment variables: 19 | ```shell 20 | set -a; source integration_tests/.env/[TARGET].env; set +a 21 | ``` 22 | 23 | or more specific: 24 | ```shell 25 | set -a; source integration_tests/.env/postgres.env; set +a 26 | ``` 27 | 28 | #### Setup Postgres (optional) 29 | 30 | Docker and `docker-compose` are both used in testing. Specific instructions for your OS can be found [here](https://docs.docker.com/get-docker/). 31 | 32 | Postgres offers the easiest way to test most `dbt-utils` functionality today. Its tests are the fastest to run, and the easiest to set up. To run the Postgres integration tests, you'll have to do one extra step of setting up the test database: 33 | 34 | ```shell 35 | make setup-db 36 | ``` 37 | or, alternatively: 38 | ```shell 39 | docker-compose up --detach postgres 40 | ``` 41 | 42 | ### Setup virtual environment 43 | 44 | We strongly recommend using virtual environments when developing code in `dbt-utils`. We recommend creating this virtualenv 45 | in the root of the `dbt-utils` repository. To create a new virtualenv, run: 46 | ```shell 47 | python3 -m venv env 48 | source env/bin/activate 49 | ``` 50 | 51 | This will create and activate a new Python virtual environment. 52 | 53 | ### Installation for development 54 | 55 | First make sure that you set up your virtual environment as described above. Also ensure you have the latest version of pip installed with `pip install --upgrade pip`. Next, install `dbt-core` (and its dependencies) with: 56 | 57 | ```shell 58 | make dev target=[postgres|redshift|...] 59 | # or 60 | pip install --pre dbt-[postgres|redshift|...] -r dev-requirements.txt 61 | ``` 62 | 63 | or more specific: 64 | 65 | ```shell 66 | make dev target=postgres 67 | # or 68 | pip install --pre dbt-postgres -r dev-requirements.txt 69 | ``` 70 | 71 | ### Run the integration tests 72 | 73 | To run all the integration tests on your local machine like they will get run in the CI (using CircleCI): 74 | 75 | ```shell 76 | make test target=postgres 77 | ``` 78 | 79 | or, to run tests for a single model: 80 | ```shell 81 | make test target=[postgres|redshift|...] 82 | ``` 83 | 84 | or more specific: 85 | 86 | ```shell 87 | make test target=postgres 88 | ``` 89 | 90 | Where possible, targets are being run in docker containers (this works for Postgres or in the future Spark for example). For managed services like Snowflake, BigQuery and Redshift this is not possible, hence your own configuration for these services has to be provided in the appropriate env files in `integration_tests/.env/[TARGET].env` 91 | 92 | ### Creating a new integration test 93 | 94 | #### Add your integration test 95 | This directory contains an example dbt project which tests the macros in the `dbt-utils` package. An integration test typically involves making 1) a new seed file 2) a new model file 3) a generic test to assert anticipated behaviour. 96 | 97 | For an example integration tests, check out the tests for the `get_url_parameter` macro: 98 | 99 | 1. [Macro definition](https://github.com/dbt-labs/dbt-utils/blob/main/macros/web/get_url_parameter.sql) 100 | 2. [Seed file with fake data](https://github.com/dbt-labs/dbt-utils/blob/main/integration_tests/data/web/data_urls.csv) 101 | 3. [Model to test the macro](https://github.com/dbt-labs/dbt-utils/blob/main/integration_tests/models/web/test_urls.sql) 102 | 4. [A generic test to assert the macro works as expected](https://github.com/dbt-labs/dbt-utils/blob/main/integration_tests/models/web/schema.yml) 103 | 104 | Once you've added all of these files, you should be able to run: 105 | 106 | Assuming you are in the `integration_tests` folder, 107 | ```shell 108 | dbt deps --target {your_target} 109 | dbt seed --target {your_target} 110 | dbt run --target {your_target} --model {your_model_name} 111 | dbt test --target {your_target} --model {your_model_name} 112 | ``` 113 | 114 | Alternatively: 115 | ```shell 116 | dbt deps --target {your_target} 117 | dbt build --target {your_target} --select +{your_model_name} 118 | ``` 119 | 120 | If the tests all pass, then you're good to go! All tests will be run automatically when you create a PR against this repo. -------------------------------------------------------------------------------- /integration_tests/data/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbt-labs/dbt-utils/22855099135cd4de445bed45601f11cb1caabb24/integration_tests/data/.gitkeep -------------------------------------------------------------------------------- /integration_tests/data/datetime/data_date_spine.csv: -------------------------------------------------------------------------------- 1 | date_day 2 | 2018-01-01 3 | 2018-01-02 4 | 2018-01-03 5 | 2018-01-04 6 | 2018-01-05 7 | 2018-01-06 8 | 2018-01-07 9 | 2018-01-08 10 | 2018-01-09 -------------------------------------------------------------------------------- /integration_tests/data/etc/data_people.csv: -------------------------------------------------------------------------------- 1 | id,first_name,last_name,email,ip_address,created_at,is_active 2 | 1,Dame,Cluley,dcluley0@nih.gov,155.86.204.241,2017-02-07 09:48:26,false 3 | 2,Guy,Wittering,gwittering1@reddit.com,221.174.176.36,2017-08-08 00:37:53,false 4 | 3,Klement,Bucke,kbucke2@dedecms.com,167.94.85.199,2016-09-05 23:43:19,true 5 | 4,Roselia,Dallander,rdallander3@adobe.com,135.10.21.248,2016-08-11 00:00:11,false 6 | 5,Arly,Terzza,aterzza4@va.gov,219.66.192.10,2017-03-23 22:11:42,true 7 | 6,Arron,Siehard,asiehard5@ibm.com,116.211.108.88,2017-07-07 23:11:50,true 8 | 7,Debera,Petrazzi,dpetrazzi6@addthis.com,18.167.49.108,2017-11-12 04:34:50,false 9 | 8,Timi,Agget,tagget7@home.pl,170.171.78.217,2016-03-14 02:04:33,true 10 | 9,Ines,Brixey,ibrixey8@biblegateway.com,251.141.4.42,2017-10-01 16:41:21,false 11 | 10,Karlen,Eggleton,keggleton9@amazon.co.jp,100.179.149.224,2016-04-15 10:05:00,true 12 | 11,Hamish,Winfield,hwinfielda@squarespace.com,5.34.205.16,2017-12-29 22:44:52,true 13 | 12,Stanton,Tiron,stironb@rambler.ru,171.5.190.125,2017-01-20 23:31:15,true 14 | 13,Tyne,Elner,telnerc@jiathis.com,165.155.112.184,2017-06-12 23:42:54,false 15 | 14,Lita,Kitley,lkitleyd@gmpg.org,138.131.8.94,2018-01-25 15:03:51,false 16 | 15,Alan,Morsley,amorsleye@dell.com,5.81.121.91,2016-03-18 19:37:49,true 17 | 16,Erinn,Stokoe,estokoef@walmart.com,244.57.254.248,2017-02-23 22:51:09,true 18 | 17,Dela,Oxley,doxleyg@state.gov,163.86.24.94,2017-04-12 20:19:20,true 19 | 18,Daryle,Reeve,dreeveh@1und1.de,175.30.172.20,2017-07-09 20:46:10,false 20 | 19,Micah,Smitham,msmithami@techcrunch.com,164.75.157.186,2016-02-25 16:17:57,true 21 | 20,Bernice,Van der Velde,bvanderveldej@i2i.jp,141.99.132.98,2017-07-28 23:31:24,false 22 | 21,Odo,Janacek,ojanacekk@redcross.org,50.195.72.49,2017-05-01 05:59:30,false 23 | 22,Lyndsey,Exter,lexterl@scribd.com,244.5.43.160,2017-02-13 11:32:04,false 24 | 23,Correy,Brash,cbrashm@loc.gov,233.67.52.95,2018-02-27 05:26:29,false 25 | 24,Lyle,Josilevich,ljosilevichn@rambler.ru,99.16.127.176,2016-08-06 03:37:03,false 26 | 25,Skip,Castiello,scastielloo@rambler.ru,118.174.3.50,2016-06-07 23:32:19,true 27 | 26,Philbert,Daltry,pdaltryp@tamu.edu,181.93.127.23,2016-08-16 12:52:52,true 28 | 27,Addie,Sikora,asikoraq@theatlantic.com,120.33.67.44,2016-09-01 12:45:37,true 29 | 28,Sibyl,Songist,ssongistr@noaa.gov,151.85.172.142,2016-02-11 01:14:50,false 30 | 29,Eyde,Dankersley,edankersleys@illinois.edu,147.170.154.132,2017-08-09 18:14:00,false 31 | 30,Dion,Pessler,dpesslert@reverbnation.com,51.92.202.203,2017-01-30 02:05:47,true 32 | 31,Rodd,Huntly,rhuntlyu@google.ru,82.198.158.0,2016-04-22 06:44:15,false 33 | 32,Inness,Cartmer,icartmerv@tripod.com,44.147.127.200,2017-03-11 12:03:56,false 34 | 33,Blakeley,Figgins,bfigginsw@ebay.co.uk,116.54.91.30,2016-05-28 14:25:49,true 35 | 34,Yancey,Leeburne,yleeburnex@people.com.cn,8.44.104.205,2016-08-09 03:15:02,false 36 | 35,Gustavus,Kemp,gkempy@sourceforge.net,101.126.34.176,2018-02-02 12:15:57,true 37 | 36,Annabela,Ardron,aardronz@slideshare.net,135.255.20.212,2017-10-29 03:13:03,true 38 | 37,Allister,Janota,ajanota10@yahoo.com,41.139.90.112,2016-09-19 04:21:50,true 39 | 38,Yoko,McBryde,ymcbryde11@weather.com,124.17.222.132,2016-08-21 14:32:04,false 40 | 39,Aprilette,Colebeck,acolebeck12@elegantthemes.com,14.62.14.45,2017-04-04 04:47:31,true 41 | 40,Oralia,Marklew,omarklew13@cnet.com,108.161.10.231,2017-12-29 23:15:15,true 42 | 41,Vi,Bryde,vbryde14@harvard.edu,20.91.132.215,2017-12-01 21:02:36,false 43 | 42,Koren,Emmanueli,kemmanueli15@fotki.com,151.86.146.63,2016-11-10 22:36:05,true 44 | 43,Corrie,Pendry,cpendry16@technorati.com,78.110.104.252,2017-11-22 07:57:23,true 45 | 44,Berton,Jakovijevic,bjakovijevic17@themeforest.net,243.201.191.244,2017-12-22 20:30:37,false 46 | 45,Ahmad,Lawerence,alawerence18@bluehost.com,234.146.69.92,2017-07-07 17:37:17,true 47 | 46,Walther,Mardee,wmardee19@sciencedirect.com,86.10.226.173,2016-06-23 09:20:51,false 48 | 47,Raynor,Reignolds,rreignolds1a@github.com,192.159.109.53,2016-04-19 13:32:00,false 49 | 48,Dom,Brodhead,dbrodhead1b@ed.gov,13.193.83.80,2016-09-24 03:16:43,false 50 | 49,Patton,Marrett,pmarrett1c@sourceforge.net,73.142.143.198,2016-06-02 19:20:48,true 51 | 50,Murielle,Reina,mreina1d@washington.edu,88.67.241.169,2017-10-01 01:56:52,true 52 | 51,Markos,Zylberdik,mzylberdik1e@ask.com,169.62.233.37,2017-03-23 19:40:19,true 53 | 52,Dorisa,Gosalvez,dgosalvez1f@mit.edu,10.111.156.111,2016-02-24 12:37:30,true 54 | 53,Amata,Moar,amoar1g@tinypic.com,214.241.229.183,2016-05-22 05:04:06,true 55 | 54,Graehme,Finnigan,gfinnigan1h@trellian.com,229.14.230.4,2016-12-27 00:49:18,true 56 | 55,Tanya,Sheers,tsheers1i@house.gov,43.212.37.134,2018-02-04 05:17:30,true 57 | 56,Germain,Beavers,gbeavers1j@hexun.com,91.219.240.74,2017-01-26 23:03:39,false 58 | 57,Emmye,Cerie,ecerie1k@independent.co.uk,58.183.233.79,2017-04-30 14:13:31,true 59 | 58,Reese,Glaisner,rglaisner1l@dropbox.com,63.181.9.68,2016-07-29 05:49:41,true 60 | 59,Christie,Phlippsen,cphlippsen1m@ucoz.ru,236.91.248.168,2017-07-07 12:37:10,false 61 | 60,Anthia,Tolland,atolland1n@hibu.com,124.60.13.101,2016-02-06 14:38:37,true 62 | 61,Annamarie,Pipworth,apipworth1o@ftc.gov,53.219.191.107,2017-06-13 08:29:04,true 63 | 62,Price,O'Gready,pogready1p@theatlantic.com,131.188.180.57,2016-09-28 08:44:38,false 64 | 63,Sergei,Cicero,scicero1q@telegraph.co.uk,100.97.16.84,2017-10-02 15:58:45,false 65 | 64,Dolorita,Lilion,dlilion1r@vimeo.com,150.43.252.51,2017-09-06 12:39:46,true 66 | 65,Perrine,Peetermann,ppeetermann1s@fema.gov,93.27.202.229,2017-07-08 08:49:37,false 67 | 66,Frieda,Gemelli,fgemelli1t@altervista.org,20.21.177.102,2016-04-18 05:58:59,false 68 | 67,Webster,Tully,wtully1u@nba.com,61.55.62.136,2016-02-18 11:01:23,true 69 | 68,Clara,Dadd,cdadd1v@rakuten.co.jp,67.84.203.36,2017-06-10 22:20:50,false 70 | 69,Gardener,Clarkin,gclarkin1w@bbc.co.uk,211.175.17.92,2017-11-27 23:33:42,true 71 | 70,Doll,Celez,dcelez1x@imdb.com,65.124.34.165,2017-01-03 06:40:44,true 72 | 71,Willy,Remnant,wremnant1y@nasa.gov,183.190.219.35,2017-05-27 11:05:47,false 73 | 72,Felicle,Scoterbosh,fscoterbosh1z@macromedia.com,12.103.23.230,2017-05-04 05:22:27,true 74 | 73,Egan,Ryland,eryland20@t.co,227.35.15.147,2016-06-09 11:50:39,true 75 | 74,Donny,Clissold,dclissold21@yellowpages.com,210.51.117.212,2016-03-08 22:48:18,true 76 | 75,Gwyneth,Brash,gbrash22@vistaprint.com,30.243.157.153,2016-01-23 17:11:17,true 77 | 76,Mervin,Woolis,mwoolis23@elegantthemes.com,52.171.162.135,2017-06-17 15:36:58,false 78 | 77,Alicea,Mewton,amewton24@com.com,236.53.172.152,2017-12-21 10:35:45,true 79 | 78,Whittaker,Biaggiotti,wbiaggiotti25@patch.com,85.219.77.207,2017-12-27 09:25:13,true 80 | 79,Teddie,Matevushev,tmatevushev26@nsw.gov.au,121.24.14.214,2017-12-09 16:30:35,false 81 | 80,Mal,Mc Gee,mmcgee27@eventbrite.com,85.138.92.81,2016-01-14 03:02:43,true 82 | 81,Teressa,Lourenco,tlourenco28@zdnet.com,33.2.78.199,2016-03-17 02:29:47,false 83 | 82,Willabella,Danelutti,wdanelutti29@ted.com,221.78.224.255,2016-03-06 14:34:53,true 84 | 83,Samaria,Hessle,shessle2a@surveymonkey.com,216.8.59.131,2017-03-30 11:02:45,true 85 | 84,Ruperto,Staite,rstaite2b@wikispaces.com,79.47.189.125,2017-08-23 22:09:19,true 86 | 85,Ashlee,Scotsbrook,ascotsbrook2c@trellian.com,91.104.127.195,2017-10-02 15:01:49,false 87 | 86,Godfry,Lawson,glawson2d@seesaa.net,241.54.44.84,2016-04-03 04:42:19,false 88 | 87,Rose,Lathleiffure,rlathleiffure2e@instagram.com,21.172.211.218,2016-05-11 04:37:44,true 89 | 88,Ricky,Helwig,rhelwig2f@sciencedirect.com,130.213.100.214,2017-12-02 11:58:19,true 90 | 89,Hersh,Castleman,hcastleman2g@mediafire.com,196.170.63.20,2016-11-06 15:18:34,false 91 | 90,Upton,Midghall,umidghall2h@wordpress.org,29.108.156.94,2017-03-24 03:48:22,true 92 | 91,Devi,Lowmass,dlowmass2i@parallels.com,243.189.157.74,2016-07-31 13:35:43,true 93 | 92,Cherry,Goldstein,cgoldstein2j@delicious.com,21.78.25.159,2016-06-02 22:19:13,true 94 | 93,Alfy,Yakubovics,ayakubovics2k@bigcartel.com,29.28.179.184,2016-10-13 08:03:28,true 95 | 94,Ambrosi,Martinyuk,amartinyuk2l@163.com,1.42.244.146,2016-01-24 23:02:40,false 96 | 95,Daniel,Duly,dduly2m@engadget.com,74.32.138.66,2017-03-26 09:02:19,false 97 | 96,Hazlett,Oakton,hoakton2n@booking.com,248.196.158.127,2016-11-01 10:55:45,true 98 | 97,Vivienne,Millery,vmillery2o@nyu.edu,99.21.99.255,2016-04-19 15:25:08,true 99 | 98,Glynda,Kupper,gkupper2p@yahoo.co.jp,89.13.224.249,2016-04-05 07:01:28,false 100 | 99,Pavlov,MacDwyer,pmacdwyer2q@netvibes.com,147.162.14.191,2018-02-10 06:57:45,true 101 | 100,Fonzie,Filip,ffilip2r@tripadvisor.com,244.178.118.180,2016-11-18 00:09:42,false 102 | -------------------------------------------------------------------------------- /integration_tests/data/geo/data_haversine_km.csv: -------------------------------------------------------------------------------- 1 | lat_1,lon_1,lat_2,lon_2,output 2 | 48.864716,2.349014,52.379189,4.899431,430 3 | -------------------------------------------------------------------------------- /integration_tests/data/geo/data_haversine_mi.csv: -------------------------------------------------------------------------------- 1 | lat_1,lon_1,lat_2,lon_2,output 2 | 48.864716,2.349014,52.379189,4.899431,267 3 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_cardinality_equality_a.csv: -------------------------------------------------------------------------------- 1 | same_name 2 | 1 3 | 2 4 | 3 -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_cardinality_equality_b.csv: -------------------------------------------------------------------------------- 1 | same_name,different_name 2 | 1,2 3 | 2,3 4 | 3,1 -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_not_null_proportion.csv: -------------------------------------------------------------------------------- 1 | point_5,point_9 2 | 1,1 3 | ,2 4 | ,3 5 | 4,4 6 | 5,5 7 | 6,6 8 | ,7 9 | ,8 10 | , 11 | 10,10 -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_accepted_range.csv: -------------------------------------------------------------------------------- 1 | id 2 | -1 3 | 11 -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_at_least_one.csv: -------------------------------------------------------------------------------- 1 | field,value 2 | a,1 3 | b, 4 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_equal_rowcount.csv: -------------------------------------------------------------------------------- 1 | field 2 | 1 3 | 1 4 | 2 5 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_equality_a.csv: -------------------------------------------------------------------------------- 1 | col_a,col_b,col_c 2 | 1,1,3 3 | 1,2,1 4 | 2,3,3 5 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_equality_b.csv: -------------------------------------------------------------------------------- 1 | col_a,col_b,col_c 2 | 1,1,2 3 | 1,2,2 4 | 2,3,2 5 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_equality_floats_a.csv: -------------------------------------------------------------------------------- 1 | id,float_number 2 | 05ac09c4-f947-45a8-8c14-88f430f8b294,62.3888186 3 | cfae9054-940b-42a1-84d4-052daae6194f,81.2511656 4 | 6029501d-c274-49f2-a69d-4c75a3d9931d,23.3959675 5 | c653e520-df81-4a5f-b44b-bb1b4c1b7846,72.2100841 6 | 59caed0d-53d6-473c-a88c-3726c7693f05,68.6029434 7 | b441f6a0-ce7f-4ad9-b96b-b41d73a94ae7,72.7861425 8 | 26491840-bfd4-4496-9ca9-ad9220a2de47,35.3662223 9 | b4f233ce-a494-4bb6-9cf2-73bb6854e58a,89.1524680 10 | 11c979b7-2661-4375-8143-7c9b54b90627,19.5755431 11 | a8057f73-312e-48e6-b344-f4a510a2c4a8,22.9237047 12 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_equality_floats_b.csv: -------------------------------------------------------------------------------- 1 | id,float_number 2 | 05ac09c4-f947-45a8-8c14-88f430f8b294,62.3888187 3 | cfae9054-940b-42a1-84d4-052daae6194f,81.2511657 4 | 6029501d-c274-49f2-a69d-4c75a3d9931d,23.3959676 5 | c653e520-df81-4a5f-b44b-bb1b4c1b7846,72.2100842 6 | 59caed0d-53d6-473c-a88c-3726c7693f05,68.6029435 7 | b441f6a0-ce7f-4ad9-b96b-b41d73a94ae7,72.7861426 8 | 26491840-bfd4-4496-9ca9-ad9220a2de47,35.3662224 9 | b4f233ce-a494-4bb6-9cf2-73bb6854e58a,89.1524681 10 | 11c979b7-2661-4375-8143-7c9b54b90627,19.5755432 11 | a8057f73-312e-48e6-b344-f4a510a2c4a8,22.9237048 12 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_equality_floats_columns_a.csv: -------------------------------------------------------------------------------- 1 | id,float_number,to_ignore 2 | 05ac09c4-f947-45a8-8c14-88f430f8b294,62.3888186,a 3 | cfae9054-940b-42a1-84d4-052daae6194f,81.2511656,a 4 | 6029501d-c274-49f2-a69d-4c75a3d9931d,23.3959675,a 5 | c653e520-df81-4a5f-b44b-bb1b4c1b7846,72.2100841,a 6 | 59caed0d-53d6-473c-a88c-3726c7693f05,68.6029434,a 7 | b441f6a0-ce7f-4ad9-b96b-b41d73a94ae7,72.7861425,a 8 | 26491840-bfd4-4496-9ca9-ad9220a2de47,35.3662223,a 9 | b4f233ce-a494-4bb6-9cf2-73bb6854e58a,89.1524680,a 10 | 11c979b7-2661-4375-8143-7c9b54b90627,19.5755431,a 11 | a8057f73-312e-48e6-b344-f4a510a2c4a8,22.9237047,a 12 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_equality_floats_columns_b.csv: -------------------------------------------------------------------------------- 1 | id,float_number,to_ignore 2 | 05ac09c4-f947-45a8-8c14-88f430f8b294,62.3888186,b 3 | cfae9054-940b-42a1-84d4-052daae6194f,81.2511656,b 4 | 6029501d-c274-49f2-a69d-4c75a3d9931d,23.3959675,b 5 | c653e520-df81-4a5f-b44b-bb1b4c1b7846,72.2100841,b 6 | 59caed0d-53d6-473c-a88c-3726c7693f05,68.6029434,b 7 | b441f6a0-ce7f-4ad9-b96b-b41d73a94ae7,72.7861425,b 8 | 26491840-bfd4-4496-9ca9-ad9220a2de47,35.3662223,b 9 | b4f233ce-a494-4bb6-9cf2-73bb6854e58a,89.1524680,b 10 | 11c979b7-2661-4375-8143-7c9b54b90627,19.5755431,b 11 | a8057f73-312e-48e6-b344-f4a510a2c4a8,22.9237047,b 12 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_expression_is_true.csv: -------------------------------------------------------------------------------- 1 | col_a,col_b 2 | 0,1 3 | 1,0 4 | 0.5,0.5 5 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_fewer_rows_than_table_1.csv: -------------------------------------------------------------------------------- 1 | col_a,field 2 | 1,1 3 | 1,2 4 | 1,3 5 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_fewer_rows_than_table_2.csv: -------------------------------------------------------------------------------- 1 | col_a,field 2 | 1,1 3 | 1,2 4 | 1,3 5 | 1,4 6 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_mutually_exclusive_ranges_no_gaps.csv: -------------------------------------------------------------------------------- 1 | lower_bound,upper_bound 2 | 0,1 3 | 1,2 4 | 2,4 5 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_mutually_exclusive_ranges_with_gaps.csv: -------------------------------------------------------------------------------- 1 | subscription_id,valid_from,valid_to 2 | 1,2019-01-01,2019-02-01 3 | 1,2019-03-03,2019-04-01 4 | 2,2019-05-06,2019-07-02 5 | 2,2019-07-03, 6 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_mutually_exclusive_ranges_with_gaps_zero_length.csv: -------------------------------------------------------------------------------- 1 | subscription_id,valid_from,valid_to 2 | 3,2020-05-06,2020-05-07 3 | 3,2020-05-08,2020-05-10 4 | 3,2020-05-08,2020-05-08 5 | 3,2020-05-12,2020-05-15 6 | 4,2020-06-06,2020-06-07 7 | 4,2020-06-08,2020-06-08 8 | 4,2020-06-09,2020-06-10 9 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_not_accepted_values.csv: -------------------------------------------------------------------------------- 1 | id,city 2 | 1,Barcelona 3 | 2,London 4 | 3,Paris 5 | 4,New York 6 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_not_constant.csv: -------------------------------------------------------------------------------- 1 | col_a,field 2 | 1,1 3 | 1,1 4 | 1,2 5 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_relationships_where_table_1.csv: -------------------------------------------------------------------------------- 1 | id 2 | 1 3 | 2 4 | 3 5 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_relationships_where_table_2.csv: -------------------------------------------------------------------------------- 1 | id 2 | 1 3 | 2 4 | 4 5 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_sequential_timestamps.csv: -------------------------------------------------------------------------------- 1 | my_timestamp 2 | 2021-01-01 00:00 3 | 2021-01-01 01:00 4 | 2021-01-01 02:00 5 | 2021-01-01 03:00 6 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_test_sequential_values.csv: -------------------------------------------------------------------------------- 1 | col_a,my_even_sequence 2 | 1,2 3 | 1,4 4 | 1,6 5 | 2,8 6 | 2,10 7 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/data_unique_combination_of_columns.csv: -------------------------------------------------------------------------------- 1 | month,product,revenue 2 | 2019-01-01,jaffle,500 3 | 2019-01-01,lamington,100 4 | 2019-01-01,pavlova,600 5 | 2019-02-01,jaffle,300 6 | 2019-02-01,lamington,300 7 | 2019-02-01,pavlova,400 8 | -------------------------------------------------------------------------------- /integration_tests/data/schema_tests/schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | seeds: 4 | - name: data_test_sequential_values 5 | columns: 6 | - name: my_even_sequence 7 | data_tests: 8 | - dbt_utils.sequential_values: 9 | interval: 2 10 | - dbt_utils.sequential_values: 11 | interval: 2 12 | group_by_columns: ['col_a'] 13 | 14 | 15 | - name: data_test_sequential_timestamps 16 | columns: 17 | - name: my_timestamp 18 | data_tests: 19 | - dbt_utils.sequential_values: 20 | interval: 1 21 | datepart: 'hour' 22 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_deduplicate.csv: -------------------------------------------------------------------------------- 1 | user_id,event,version 2 | 1,play,1 3 | 1,play,2 4 | 2,pause,1 5 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_deduplicate_expected.csv: -------------------------------------------------------------------------------- 1 | user_id,event,version 2 | 1,play,2 3 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_events_20180101.csv: -------------------------------------------------------------------------------- 1 | user_id,event 2 | 1,play 3 | 2,pause 4 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_events_20180102.csv: -------------------------------------------------------------------------------- 1 | user_id,event 2 | 3,play 3 | 4,pause 4 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_events_20180103.csv: -------------------------------------------------------------------------------- 1 | user_id,event 2 | 5,play 3 | 6,pause 4 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_filtered_columns_in_relation.csv: -------------------------------------------------------------------------------- 1 | field_1,field_2,field_3 2 | a,b,c 3 | d,e,f 4 | g,h,i -------------------------------------------------------------------------------- /integration_tests/data/sql/data_filtered_columns_in_relation_expected.csv: -------------------------------------------------------------------------------- 1 | field_2,field_3 2 | h,i -------------------------------------------------------------------------------- /integration_tests/data/sql/data_generate_series.csv: -------------------------------------------------------------------------------- 1 | generated_number 2 | 1 3 | 2 4 | 3 5 | 4 6 | 5 7 | 6 8 | 7 9 | 8 10 | 9 11 | 10 12 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_generate_surrogate_key.csv: -------------------------------------------------------------------------------- 1 | column_1,column_2,column_3,expected_column_1_only,expected_all_columns 2 | a,b,c,0cc175b9c0f1b6a831c399e269772661,7b193b3d33184464106f41ddf733783b 3 | a,,c,0cc175b9c0f1b6a831c399e269772661,4f32a73dc87b7bbb7a654d8898d58c7e 4 | ,,c,f14cc5cdce0420f4a5a6b6d9d7b85f39,d9c538b129f1a3ad6ecfe55345c32a05 5 | ,,,f14cc5cdce0420f4a5a6b6d9d7b85f39,2fa5491950d66d153d23cfbcfea4e164 6 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_get_column_values.csv: -------------------------------------------------------------------------------- 1 | field 2 | a 3 | b 4 | c 5 | d 6 | e 7 | f 8 | g 9 | g 10 | g 11 | g 12 | g 13 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_get_column_values_dropped.csv: -------------------------------------------------------------------------------- 1 | field 2 | a 3 | b 4 | c 5 | d 6 | e 7 | f 8 | g 9 | g 10 | g 11 | g 12 | g 13 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_get_column_values_where.csv: -------------------------------------------------------------------------------- 1 | field,condition 2 | a,left 3 | b,right 4 | c,left 5 | d,right 6 | e,left 7 | f,right 8 | g,left 9 | g,right 10 | g,left 11 | g,right 12 | g,left -------------------------------------------------------------------------------- /integration_tests/data/sql/data_get_column_values_where_expected.csv: -------------------------------------------------------------------------------- 1 | field 2 | a 3 | c 4 | e 5 | g -------------------------------------------------------------------------------- /integration_tests/data/sql/data_get_query_results_as_dict.csv: -------------------------------------------------------------------------------- 1 | col_1,col_2,col_3 2 | 1,a,True 3 | 2,b,False 4 | 3,c, 5 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_get_single_value.csv: -------------------------------------------------------------------------------- 1 | date_value,float_value,int_value,string_value 2 | 2017-01-01 00:00:00,3.3,19,string_a -------------------------------------------------------------------------------- /integration_tests/data/sql/data_nullcheck_table.csv: -------------------------------------------------------------------------------- 1 | field_1,field_2,field_3 2 | a,'',1 3 | '',b,2 4 | '','',3 5 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_pivot.csv: -------------------------------------------------------------------------------- 1 | size,color 2 | S,red 3 | S,blue 4 | S,blue's 5 | M,red -------------------------------------------------------------------------------- /integration_tests/data/sql/data_pivot_expected.csv: -------------------------------------------------------------------------------- 1 | size,red,blue 2 | S,1,1 3 | M,1,0 -------------------------------------------------------------------------------- /integration_tests/data/sql/data_pivot_expected_apostrophe.csv: -------------------------------------------------------------------------------- 1 | size,red,blue,blues 2 | S,1,1,1 3 | M,1,0,0 -------------------------------------------------------------------------------- /integration_tests/data/sql/data_safe_add.csv: -------------------------------------------------------------------------------- 1 | field_1,field_2,field_3,expected 2 | 1,2,3,6 3 | 1,,3,4 4 | ,,2,2 5 | ,,,0 6 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_safe_divide.csv: -------------------------------------------------------------------------------- 1 | numerator,denominator,output 2 | 6,0, 3 | 10,5,2 4 | ,, 5 | ,0, 6 | 17,, 7 | 0,, 8 | ,9, 9 | 0,5,0 -------------------------------------------------------------------------------- /integration_tests/data/sql/data_safe_divide_denominator_expressions.csv: -------------------------------------------------------------------------------- 1 | numerator,denominator_1,denominator_2,output 2 | ,0,4, 3 | 6,3,2,1 4 | 0,2,6,0 5 | 0,,8, 6 | 5,,2, 7 | 4,0,4, -------------------------------------------------------------------------------- /integration_tests/data/sql/data_safe_divide_numerator_expressions.csv: -------------------------------------------------------------------------------- 1 | numerator_1,numerator_2,denominator,output 2 | 0,5,9,0 3 | 2,3,0, 4 | 0,0,0, 5 | 3,4,, 6 | ,6,14, 7 | 2,5,2,5 -------------------------------------------------------------------------------- /integration_tests/data/sql/data_safe_subtract.csv: -------------------------------------------------------------------------------- 1 | field_1,field_2,field_3,expected 2 | 3,2,1,0 3 | 4,,3,1 4 | ,,2,-2 5 | ,,,0 6 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_star.csv: -------------------------------------------------------------------------------- 1 | field_1,field_2,field_3 2 | a,b,c 3 | d,e,f 4 | g,h,i 5 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_star_aggregate.csv: -------------------------------------------------------------------------------- 1 | group_field_1,group_field_2,value_field 2 | a,b,1 3 | a,b,2 4 | c,d,3 5 | c,e,4 -------------------------------------------------------------------------------- /integration_tests/data/sql/data_star_aggregate_expected.csv: -------------------------------------------------------------------------------- 1 | group_field_1,group_field_2,value_field_sum 2 | a,b,3 3 | c,d,3 4 | c,e,4 -------------------------------------------------------------------------------- /integration_tests/data/sql/data_star_expected.csv: -------------------------------------------------------------------------------- 1 | field_1,field_2 2 | a,b 3 | d,e 4 | g,h 5 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_star_prefix_suffix_expected.csv: -------------------------------------------------------------------------------- 1 | prefix_field_1_suffix,prefix_field_2_suffix,prefix_field_3_suffix 2 | a,b,c 3 | d,e,f 4 | g,h,i 5 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_star_quote_identifiers.csv: -------------------------------------------------------------------------------- 1 | column_one 2 | a 3 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_union_events_expected.csv: -------------------------------------------------------------------------------- 1 | user_id,event 2 | 1,play 3 | 2,pause 4 | 3,play 5 | 4,pause 6 | 5,play 7 | 6,pause 8 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_union_exclude_expected.csv: -------------------------------------------------------------------------------- 1 | id,favorite_color,favorite_number 2 | 1,,pi 3 | 2,,e 4 | 3,,4 5 | 1,"green",7 6 | 2,"pink",13 7 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_union_expected.csv: -------------------------------------------------------------------------------- 1 | id,name,favorite_color,favorite_number 2 | 1,"drew",,pi 3 | 2,"bob",,e 4 | 3,"alice",,4 5 | 1,,"green",7 6 | 2,,"pink",13 7 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_union_table_1.csv: -------------------------------------------------------------------------------- 1 | id,name,favorite_number 2 | 1,drew,pi 3 | 2,bob,e 4 | 3,alice,4 5 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_union_table_2.csv: -------------------------------------------------------------------------------- 1 | id,favorite_color,favorite_number 2 | 1,green,7 3 | 2,pink,13 4 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_unpivot.csv: -------------------------------------------------------------------------------- 1 | customer_id,created_at,status,segment,name 2 | 123,2017-01-01,active,tier 1,name 1 3 | 234,2017-02-01,active,tier 3,name 3 4 | 567,2017-03-01,churned,tier 2,name 2 5 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_unpivot_bool.csv: -------------------------------------------------------------------------------- 1 | customer_id,created_at,status,segment,is_updated 2 | 123,2017-01-01,active,tier 1,TRUE 3 | 234,2017-02-01,active,tier 3,FALSE 4 | 567,2017-03-01,churned,tier 2, 5 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_unpivot_bool_expected.csv: -------------------------------------------------------------------------------- 1 | customer_id,created_at,prop,val 2 | 123,2017-01-01,segment,tier 1 3 | 123,2017-01-01,status,active 4 | 123,2017-01-01,is_updated,true 5 | 234,2017-02-01,segment,tier 3 6 | 234,2017-02-01,status,active 7 | 234,2017-02-01,is_updated,false 8 | 567,2017-03-01,status,churned 9 | 567,2017-03-01,is_updated, 10 | 567,2017-03-01,segment,tier 2 11 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_unpivot_expected.csv: -------------------------------------------------------------------------------- 1 | customer_id,created_at,prop,val 2 | 123,"2017-01-01","segment","tier 1" 3 | 123,"2017-01-01","status","active" 4 | 234,"2017-02-01","segment","tier 3" 5 | 234,"2017-02-01","status","active" 6 | 567,"2017-03-01","status","churned" 7 | 567,"2017-03-01","segment","tier 2" 8 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_unpivot_original_api_expected.csv: -------------------------------------------------------------------------------- 1 | customer_id,created_at,field_name,value 2 | 123,2017-01-01,status,active 3 | 123,2017-01-01,segment,tier 1 4 | 234,2017-02-01,status,active 5 | 234,2017-02-01,segment,tier 3 6 | 567,2017-03-01,status,churned 7 | 567,2017-03-01,segment,tier 2 8 | 123,2017-01-01,name,name 1 9 | 234,2017-02-01,name,name 3 10 | 567,2017-03-01,name,name 2 -------------------------------------------------------------------------------- /integration_tests/data/sql/data_unpivot_quote.csv: -------------------------------------------------------------------------------- 1 | Customer_Id,Created_At,sTaTuS,SEGMENT,Name 2 | 123,2017-01-01,active,tier 1,name 1 3 | 234,2017-02-01,active,tier 3,name 3 4 | 567,2017-03-01,churned,tier 2,name 2 5 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_unpivot_quote_expected.csv: -------------------------------------------------------------------------------- 1 | Customer_Id,Created_At,Prop,Val 2 | 123,"2017-01-01","SEGMENT","tier 1" 3 | 123,"2017-01-01","sTaTuS","active" 4 | 234,"2017-02-01","SEGMENT","tier 3" 5 | 234,"2017-02-01","sTaTuS","active" 6 | 567,"2017-03-01","sTaTuS","churned" 7 | 567,"2017-03-01","SEGMENT","tier 2" 8 | -------------------------------------------------------------------------------- /integration_tests/data/sql/data_width_bucket.csv: -------------------------------------------------------------------------------- 1 | date_col,amount,num_buckets,min_value,max_value,bucket 2 | 2012-08-01,190000.00,4,200000.0,600000.0,0 3 | 2013-08-01,290000.00,4,200000.0,600000.0,1 4 | 2014-02-01,320000.00,4,200000.0,600000.0,2 5 | 2015-04-01,399999.99,4,200000.0,600000.0,2 6 | 2016-04-01,400000.00,4,200000.0,600000.0,3 7 | 2017-04-01,470000.00,4,200000.0,600000.0,3 8 | 2018-04-01,510000.00,4,200000.0,600000.0,4 9 | 2019-04-01,610000.00,4,200000.0,600000.0,5 10 | -------------------------------------------------------------------------------- /integration_tests/data/web/data_url_host.csv: -------------------------------------------------------------------------------- 1 | original_url,parsed_url 2 | www.google.co.uk?utm_source=google&utm_medium=cpc&utm_campaign=spring-summer,www.google.co.uk 3 | http://witanddelight.com/2018/01/tips-tricks-how-run-half-marathon-first-time/,witanddelight.com 4 | https://www.nytimes.com/2018/01/01/blog,www.nytimes.com 5 | android-app://m.facebook.com/,m.facebook.com 6 | docs.nytimes.com/2021/01/01/index.js?utm_source=google,docs.nytimes.com 7 | https://m.facebook.com/,m.facebook.com -------------------------------------------------------------------------------- /integration_tests/data/web/data_url_path.csv: -------------------------------------------------------------------------------- 1 | original_url,parsed_path 2 | www.google.co.uk?utm_source=google&utm_medium=cpc&utm_campaign=spring-summer, 3 | http://witanddelight.com/2018/01/tips-tricks-how-run-half-marathon-first-time/,2018/01/tips-tricks-how-run-half-marathon-first-time/ 4 | https://www.nytimes.com/2018/01/01/blog,2018/01/01/blog 5 | http://witanddelight.com/2018/01/tips-tricks-how-run-half-marathon-first-time/?utm_source=google&utm_medium=cpc&utm_campaign=spring-summer,2018/01/tips-tricks-how-run-half-marathon-first-time/ -------------------------------------------------------------------------------- /integration_tests/data/web/data_urls.csv: -------------------------------------------------------------------------------- 1 | url,medium,source 2 | http://drewbanin.com/milky?utm_medium=organic,organic, 3 | http://drewbanin.com/milky?utm_medium=organic&utm_source=github,organic,github 4 | -------------------------------------------------------------------------------- /integration_tests/dbt_project.yml: -------------------------------------------------------------------------------- 1 | 2 | name: 'dbt_utils_integration_tests' 3 | version: '1.0' 4 | 5 | profile: 'integration_tests' 6 | 7 | # require-dbt-version: inherit this from dbt-utils 8 | 9 | config-version: 2 10 | 11 | model-paths: ["models"] 12 | analysis-paths: ["analysis"] 13 | test-paths: ["tests"] 14 | seed-paths: ["data"] 15 | macro-paths: ["macros"] 16 | 17 | target-path: "target" # directory which will store compiled SQL files 18 | clean-targets: # directories to be removed by `dbt clean` 19 | - "target" 20 | - "dbt_modules" 21 | - "dbt_packages" 22 | 23 | flags: 24 | send_anonymous_usage_stats: False 25 | use_colors: True 26 | 27 | dispatch: 28 | - macro_namespace: 'dbt_utils' 29 | search_order: ['dbt_utils_integration_tests', 'dbt_utils'] 30 | 31 | seeds: 32 | 33 | +quote_columns: false 34 | dbt_utils_integration_tests: 35 | 36 | sql: 37 | data_events_20180103: 38 | +schema: events 39 | 40 | data_get_column_values_dropped: 41 | # this.incorporate() to hardcode the node's type as otherwise dbt doesn't know it yet 42 | +post-hook: "{% do adapter.drop_relation(this.incorporate(type='table')) %}" 43 | 44 | data_get_single_value: 45 | +column_types: 46 | date_value: timestamp 47 | float_value: float 48 | int_value: integer 49 | 50 | data_width_bucket: 51 | +column_types: 52 | num_buckets: integer 53 | min_value: float 54 | max_value: float 55 | 56 | data_unpivot_quote: 57 | +quote_columns: true 58 | 59 | data_unpivot_quote_expected: 60 | +quote_columns: true 61 | 62 | schema_tests: 63 | data_test_sequential_timestamps: 64 | +column_types: 65 | my_timestamp: timestamp 66 | 67 | data_test_equality_floats_a: 68 | +column_types: 69 | float_number: float 70 | 71 | data_test_equality_floats_columns_a: 72 | +column_types: 73 | float_number: float 74 | 75 | data_test_equality_floats_b: 76 | +column_types: 77 | float_number: float 78 | 79 | data_test_equality_floats_columns_b: 80 | +column_types: 81 | float_number: float 82 | -------------------------------------------------------------------------------- /integration_tests/macros/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbt-labs/dbt-utils/22855099135cd4de445bed45601f11cb1caabb24/integration_tests/macros/.gitkeep -------------------------------------------------------------------------------- /integration_tests/macros/assert_equal_values.sql: -------------------------------------------------------------------------------- 1 | {% macro assert_equal_values(actual_object, expected_object) %} 2 | {% if not execute %} 3 | 4 | {# pass #} 5 | 6 | {% elif actual_object != expected_object %} 7 | 8 | {% set msg %} 9 | Expected did not match actual 10 | 11 | ----------- 12 | Actual: 13 | ----------- 14 | --->{{ actual_object }}<--- 15 | 16 | ----------- 17 | Expected: 18 | ----------- 19 | --->{{ expected_object }}<--- 20 | 21 | {% endset %} 22 | 23 | {{ log(msg, info=True) }} 24 | 25 | select 'fail' 26 | 27 | {% else %} 28 | 29 | select 'ok' {{ limit_zero() }} 30 | 31 | {% endif %} 32 | {% endmacro %} -------------------------------------------------------------------------------- /integration_tests/macros/limit_zero.sql: -------------------------------------------------------------------------------- 1 | {% macro my_custom_macro() %} 2 | whatever 3 | {% endmacro %} 4 | 5 | {% macro limit_zero() %} 6 | {{ return(adapter.dispatch('limit_zero', 'dbt_utils')()) }} 7 | {% endmacro %} 8 | 9 | {% macro default__limit_zero() %} 10 | {{ return('limit 0') }} 11 | {% endmacro %} -------------------------------------------------------------------------------- /integration_tests/macros/tests.sql: -------------------------------------------------------------------------------- 1 | 2 | {% test assert_equal(model, actual, expected) %} 3 | select * from {{ model }} where {{ actual }} != {{ expected }} 4 | 5 | {% endtest %} 6 | 7 | 8 | {% test not_empty_string(model, column_name) %} 9 | 10 | select * from {{ model }} where {{ column_name }} = '' 11 | 12 | {% endtest %} 13 | -------------------------------------------------------------------------------- /integration_tests/models/datetime/schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: test_date_spine 5 | data_tests: 6 | - dbt_utils.equality: 7 | compare_model: ref('data_date_spine') 8 | -------------------------------------------------------------------------------- /integration_tests/models/datetime/test_date_spine.sql: -------------------------------------------------------------------------------- 1 | 2 | -- snowflake doesn't like this as a view because the `generate_series` 3 | -- call creates a CTE called `unioned`, as does the `equality` generic test. 4 | -- Ideally, Snowflake would be smart enough to know that these CTE names are 5 | -- different, as they live in different relations. TODO: use a less common cte name 6 | 7 | {{ config(materialized='table') }} 8 | 9 | with date_spine as ( 10 | 11 | {% if target.type == 'postgres' %} 12 | {{ dbt_utils.date_spine("day", "'2018-01-01'::date", "'2018-01-10'::date") }} 13 | 14 | {% elif target.type == 'bigquery' %} 15 | select cast(date_day as date) as date_day 16 | from ({{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }}) 17 | 18 | {% else %} 19 | {{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }} 20 | {% endif %} 21 | 22 | ) 23 | 24 | select date_day 25 | from date_spine 26 | 27 | -------------------------------------------------------------------------------- /integration_tests/models/generic_tests/equality_less_columns.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | 3 | select * from {{ ref('data_test_equality_b') }} 4 | 5 | ) 6 | 7 | select 8 | col_a, col_b 9 | from data 10 | -------------------------------------------------------------------------------- /integration_tests/models/generic_tests/recency_time_excluded.sql: -------------------------------------------------------------------------------- 1 | with yesterday_time as ( 2 | select 3 | 1 as col1, 4 | 2 as col2, 5 | {{ dbt.dateadd('day', -1, dbt.current_timestamp()) }} as created_at 6 | ) 7 | 8 | select 9 | col1, 10 | col2, 11 | {{ dbt.date_trunc('day', 'created_at') }} as created_at 12 | from yesterday_time -------------------------------------------------------------------------------- /integration_tests/models/generic_tests/recency_time_included.sql: -------------------------------------------------------------------------------- 1 | select 2 | 1 as col1, 3 | 2 as col2, 4 | cast({{ dbt.dateadd('hour', -23, dbt.current_timestamp()) }} as {{ dbt.type_timestamp() }}) as created_at 5 | -------------------------------------------------------------------------------- /integration_tests/models/generic_tests/test_equal_column_subset.sql: -------------------------------------------------------------------------------- 1 | {{ config(materialized='ephemeral') }} 2 | 3 | select 4 | 5 | first_name, 6 | last_name, 7 | email 8 | 9 | from {{ ref('data_people') }} 10 | -------------------------------------------------------------------------------- /integration_tests/models/generic_tests/test_equal_rowcount.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | 3 | select * from {{ ref('data_test_equal_rowcount') }} 4 | 5 | ) 6 | 7 | select 8 | field 9 | from data -------------------------------------------------------------------------------- /integration_tests/models/generic_tests/test_fewer_rows_than.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | 3 | select * from {{ ref('data_test_fewer_rows_than_table_1') }} 4 | 5 | ) 6 | 7 | select 8 | col_a, field 9 | from data -------------------------------------------------------------------------------- /integration_tests/models/geo/schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: test_haversine_distance_km 5 | data_tests: 6 | - assert_equal: 7 | actual: actual 8 | expected: expected 9 | - name: test_haversine_distance_mi 10 | data_tests: 11 | - assert_equal: 12 | actual: actual 13 | expected: expected 14 | -------------------------------------------------------------------------------- /integration_tests/models/geo/test_haversine_distance_km.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | select * from {{ ref('data_haversine_km') }} 3 | ), 4 | final as ( 5 | select 6 | output as expected, 7 | cast( 8 | {{ 9 | dbt_utils.haversine_distance( 10 | lat1='lat_1', 11 | lon1='lon_1', 12 | lat2='lat_2', 13 | lon2='lon_2', 14 | unit='km' 15 | ) 16 | }} as {{ type_numeric() }} 17 | ) as actual 18 | from data 19 | ) 20 | select 21 | expected, 22 | round(actual,0) as actual 23 | from final 24 | -------------------------------------------------------------------------------- /integration_tests/models/geo/test_haversine_distance_mi.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | select * from {{ ref('data_haversine_mi') }} 3 | ), 4 | final as ( 5 | select 6 | output as expected, 7 | cast( 8 | {{ 9 | dbt_utils.haversine_distance( 10 | lat1='lat_1', 11 | lon1='lon_1', 12 | lat2='lat_2', 13 | lon2='lon_2', 14 | unit='mi' 15 | ) 16 | }} as {{ type_numeric() }} 17 | ) as actual 18 | from data 19 | 20 | union all 21 | 22 | select 23 | output as expected, 24 | cast( 25 | {{ 26 | dbt_utils.haversine_distance( 27 | lat1='lat_1', 28 | lon1='lon_1', 29 | lat2='lat_2', 30 | lon2='lon_2', 31 | ) 32 | }} as {{ type_numeric() }} 33 | ) as actual 34 | from data 35 | ) 36 | select 37 | expected, 38 | round(actual,0) as actual 39 | from final 40 | -------------------------------------------------------------------------------- /integration_tests/models/sql/schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: test_get_single_value 5 | data_tests: 6 | - assert_equal: 7 | actual: date_actual 8 | expected: date_expected 9 | - assert_equal: 10 | actual: float_actual 11 | expected: float_expected 12 | - assert_equal: 13 | actual: int_actual 14 | expected: int_expected 15 | - assert_equal: 16 | actual: string_actual 17 | expected: string_expected 18 | 19 | - name: test_get_single_value_default 20 | data_tests: 21 | - assert_equal: 22 | actual: date_actual 23 | expected: date_expected 24 | - assert_equal: 25 | actual: float_actual 26 | expected: float_expected 27 | - assert_equal: 28 | actual: int_actual 29 | expected: int_expected 30 | - assert_equal: 31 | actual: string_actual 32 | expected: string_expected 33 | 34 | - name: test_generate_series 35 | data_tests: 36 | - dbt_utils.equality: 37 | compare_model: ref('data_generate_series') 38 | 39 | - name: test_get_column_values 40 | columns: 41 | - name: count_a 42 | data_tests: 43 | - accepted_values: 44 | values: 45 | - '1' 46 | 47 | - name: count_b 48 | data_tests: 49 | - accepted_values: 50 | values: 51 | - '1' 52 | 53 | - name: count_c 54 | data_tests: 55 | - accepted_values: 56 | values: 57 | - '1' 58 | 59 | - name: count_d 60 | data_tests: 61 | - accepted_values: 62 | values: 63 | - '1' 64 | 65 | - name: count_e 66 | data_tests: 67 | - accepted_values: 68 | values: 69 | - '1' 70 | 71 | - name: count_f 72 | data_tests: 73 | - accepted_values: 74 | values: 75 | - '1' 76 | 77 | - name: count_g 78 | data_tests: 79 | - accepted_values: 80 | values: 81 | - '5' 82 | 83 | - name: test_get_column_values_where 84 | data_tests: 85 | - dbt_utils.equality: 86 | compare_model: ref('data_get_column_values_where_expected') 87 | 88 | - name: test_get_filtered_columns_in_relation 89 | data_tests: 90 | - dbt_utils.equality: 91 | compare_model: ref('data_filtered_columns_in_relation_expected') 92 | 93 | - name: test_get_relations_by_prefix_and_union 94 | columns: 95 | - name: event 96 | data_tests: 97 | - not_null 98 | - name: user_id 99 | data_tests: 100 | - dbt_utils.at_least_one 101 | - not_null 102 | - unique 103 | 104 | - name: test_nullcheck_table 105 | columns: 106 | - name: field_1 107 | data_tests: 108 | - not_empty_string 109 | 110 | - name: field_2 111 | data_tests: 112 | - not_empty_string 113 | 114 | - name: field_3 115 | data_tests: 116 | - not_empty_string 117 | 118 | - name: test_safe_add 119 | data_tests: 120 | - assert_equal: 121 | actual: actual 122 | expected: expected 123 | 124 | - name: test_safe_subtract 125 | data_tests: 126 | - assert_equal: 127 | actual: actual 128 | expected: expected 129 | 130 | - name: test_safe_divide 131 | data_tests: 132 | - assert_equal: 133 | actual: actual 134 | expected: expected 135 | 136 | - name: test_pivot 137 | data_tests: 138 | - dbt_utils.equality: 139 | compare_model: ref('data_pivot_expected') 140 | 141 | - name: test_pivot_apostrophe 142 | data_tests: 143 | - dbt_utils.equality: 144 | compare_model: ref('data_pivot_expected_apostrophe') 145 | 146 | - name: test_unpivot_original_api 147 | data_tests: 148 | - dbt_utils.equality: 149 | compare_model: ref('data_unpivot_original_api_expected') 150 | 151 | - name: test_unpivot 152 | data_tests: 153 | - dbt_utils.equality: 154 | compare_model: ref('data_unpivot_expected') 155 | 156 | - name: test_unpivot_bool 157 | data_tests: 158 | - dbt_utils.equality: 159 | compare_model: ref('data_unpivot_bool_expected') 160 | 161 | - name: test_unpivot_quote 162 | data_tests: 163 | - dbt_utils.equality: 164 | compare_model: ref('data_unpivot_quote_expected') 165 | 166 | - name: test_star 167 | data_tests: 168 | - dbt_utils.equality: 169 | compare_model: ref('data_star_expected') 170 | 171 | - name: test_star_quote_identifiers 172 | data_tests: 173 | - assert_equal: 174 | actual: actual 175 | expected: expected 176 | 177 | - name: test_star_prefix_suffix 178 | data_tests: 179 | - dbt_utils.equality: 180 | compare_model: ref('data_star_prefix_suffix_expected') 181 | 182 | - name: test_star_aggregate 183 | data_tests: 184 | - dbt_utils.equality: 185 | compare_model: ref('data_star_aggregate_expected') 186 | 187 | - name: test_star_uppercase 188 | data_tests: 189 | - dbt_utils.equality: 190 | compare_model: ref('data_star_expected') 191 | 192 | - name: test_star_no_columns 193 | columns: 194 | - name: canary_column #If the no-columns state isn't hit, this table won't be queryable because there will be a missing comma 195 | data_tests: 196 | - not_null 197 | 198 | - name: test_generate_surrogate_key 199 | data_tests: 200 | - assert_equal: 201 | actual: actual_column_1_only 202 | expected: expected_column_1_only 203 | - assert_equal: 204 | actual: actual_all_columns_list 205 | expected: expected_all_columns 206 | 207 | - name: test_union 208 | data_tests: 209 | - dbt_utils.equality: 210 | compare_model: ref('data_union_expected') 211 | 212 | - name: test_union_where 213 | columns: 214 | - name: id 215 | data_tests: 216 | - dbt_utils.expression_is_true: 217 | expression: "= 1" 218 | - name: favorite_number 219 | data_tests: 220 | - dbt_utils.not_constant 221 | 222 | - name: test_union_no_source_column 223 | data_tests: 224 | - expect_table_columns_to_match_set: 225 | column_list: ["id", "name", "favorite_color", "favorite_number"] 226 | 227 | - name: test_union_exclude_lowercase 228 | data_tests: 229 | - dbt_utils.equality: 230 | compare_model: ref('data_union_exclude_expected') 231 | 232 | - name: test_union_exclude_uppercase 233 | data_tests: 234 | - dbt_utils.equality: 235 | compare_model: ref('data_union_exclude_expected') 236 | 237 | - name: test_get_relations_by_pattern 238 | data_tests: 239 | - dbt_utils.equality: 240 | compare_model: ref('data_union_events_expected') 241 | 242 | - name: test_deduplicate 243 | data_tests: 244 | - dbt_utils.equality: 245 | compare_model: ref('data_deduplicate_expected') 246 | 247 | - name: test_not_empty_string_failing 248 | columns: 249 | - name: string_trim_whitespace_true 250 | data_tests: 251 | - dbt_utils.not_empty_string: 252 | config: 253 | severity: error 254 | error_if: "<1" 255 | warn_if: "<0" 256 | 257 | - name: test_not_empty_string_passing 258 | columns: 259 | - name: string_trim_whitespace_true 260 | data_tests: 261 | - dbt_utils.not_empty_string 262 | - name: string_trim_whitespace_false 263 | data_tests: 264 | - dbt_utils.not_empty_string: 265 | trim_whitespace: false 266 | 267 | - name: test_width_bucket 268 | data_tests: 269 | - assert_equal: 270 | actual: actual 271 | expected: expected 272 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_deduplicate.sql: -------------------------------------------------------------------------------- 1 | with 2 | 3 | source as ( 4 | select * 5 | from {{ ref('data_deduplicate') }} 6 | where user_id = 1 7 | ), 8 | 9 | deduped as ( 10 | 11 | {{ 12 | dbt_utils.deduplicate( 13 | 'source', 14 | partition_by='user_id', 15 | order_by='version desc', 16 | ) | indent 17 | }} 18 | 19 | ) 20 | 21 | select * from deduped 22 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_generate_series.sql: -------------------------------------------------------------------------------- 1 | 2 | -- snowflake doesn't like this as a view because the `generate_series` 3 | -- call creates a CTE called `unioned`, as does the `equality` generic test. 4 | -- Ideally, Snowflake would be smart enough to know that these CTE names are 5 | -- different, as they live in different relations. TODO: use a less common cte name 6 | 7 | {{ config(materialized='table') }} 8 | 9 | with data as ( 10 | 11 | {{ dbt_utils.generate_series(10) }} 12 | 13 | ) 14 | 15 | select generated_number from data 16 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_generate_surrogate_key.sql: -------------------------------------------------------------------------------- 1 | 2 | with data as ( 3 | 4 | select * from {{ ref('data_generate_surrogate_key') }} 5 | 6 | ) 7 | 8 | select 9 | {{ dbt_utils.generate_surrogate_key(['column_1']) }} as actual_column_1_only, 10 | expected_column_1_only, 11 | {{ dbt_utils.generate_surrogate_key(['column_1', 'column_2', 'column_3']) }} as actual_all_columns_list, 12 | expected_all_columns 13 | 14 | from data 15 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_get_column_values.sql: -------------------------------------------------------------------------------- 1 | 2 | {% set column_values = dbt_utils.get_column_values(ref('data_get_column_values'), 'field', default=[], order_by="field") %} 3 | 4 | 5 | {% if target.type == 'snowflake' %} 6 | 7 | select 8 | {% for val in column_values -%} 9 | 10 | sum(case when field = '{{ val }}' then 1 else 0 end) as count_{{ val }} 11 | {%- if not loop.last %},{% endif -%} 12 | 13 | {%- endfor %} 14 | 15 | from {{ ref('data_get_column_values') }} 16 | 17 | {% else %} 18 | 19 | select 20 | {% for val in column_values -%} 21 | 22 | {{ safe_cast("sum(case when field = '" ~ val ~ "' then 1 else 0 end)", type_string()) }} as count_{{ val }} 23 | {%- if not loop.last %},{% endif -%} 24 | 25 | {%- endfor %} 26 | 27 | from {{ ref('data_get_column_values') }} 28 | 29 | {% endif %} 30 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_get_column_values_where.sql: -------------------------------------------------------------------------------- 1 | {% set column_values = dbt_utils.get_column_values(ref('data_get_column_values_where'), 'field', where="condition = 'left'") %} 2 | 3 | -- Create a relation using the values 4 | {% for val in column_values -%} 5 | select {{ string_literal(val) }} as field {% if not loop.last %}union all{% endif %} 6 | {% endfor %} -------------------------------------------------------------------------------- /integration_tests/models/sql/test_get_filtered_columns_in_relation.sql: -------------------------------------------------------------------------------- 1 | {% set exclude_field = 'field_1' %} 2 | {% set column_names = dbt_utils.get_filtered_columns_in_relation(from= ref('data_filtered_columns_in_relation'), except=[exclude_field]) %} 3 | 4 | with data as ( 5 | 6 | select 7 | 8 | {% for column_name in column_names %} 9 | max({{ column_name }}) as {{ column_name }} {% if not loop.last %},{% endif %} 10 | {% endfor %} 11 | 12 | from {{ ref('data_filtered_columns_in_relation') }} 13 | 14 | ) 15 | 16 | select * from data 17 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_get_relations_by_pattern.sql: -------------------------------------------------------------------------------- 1 | {{ config(materialized = 'table') }} 2 | 3 | -- depends_on: {{ ref('data_events_20180101') }}, {{ ref('data_events_20180102') }}, {{ ref('data_events_20180103') }} 4 | 5 | {% set relations = dbt_utils.get_relations_by_pattern(target.schema ~ '%', 'data_events_%') %} 6 | 7 | with unioned as ( 8 | 9 | {{ dbt_utils.union_relations(relations) }} 10 | 11 | ) 12 | 13 | select 14 | 15 | user_id, 16 | event 17 | 18 | from unioned 19 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql: -------------------------------------------------------------------------------- 1 | {{ config(materialized = 'table') }} 2 | 3 | -- depends_on: {{ ref('data_events_20180101') }}, {{ ref('data_events_20180102') }}, {{ ref('data_events_20180103') }} 4 | 5 | {% set relations = dbt_utils.get_relations_by_prefix(target.schema, 'data_events_') %} 6 | {{ dbt_utils.union_relations(relations) }} 7 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_get_single_value.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Dear future reader, 3 | Before you go restructuring the delicate web of casts and quotes below, a warning: 4 | I once thought as you are thinking. Proceed with caution. 5 | #} 6 | 7 | {% set date_statement %} 8 | select date_value from {{ ref('data_get_single_value') }} 9 | {% endset %} 10 | 11 | {% set float_statement %} 12 | select float_value from {{ ref('data_get_single_value') }} 13 | {% endset %} 14 | 15 | {% set int_statement %} 16 | select int_value from {{ ref('data_get_single_value') }} 17 | {% endset %} 18 | 19 | {% set string_statement %} 20 | select string_value from {{ ref('data_get_single_value') }} 21 | {% endset %} 22 | 23 | with default_data as ( 24 | 25 | select 26 | cast(date_value as {{ dbt.type_timestamp() }}) as date_expected, 27 | cast({{ dbt.string_literal(dbt_utils.get_single_value(date_statement)) }} as {{ dbt.type_timestamp() }}) as date_actual, 28 | 29 | float_value as float_expected, 30 | {{ dbt_utils.get_single_value(float_statement) }} as float_actual, 31 | 32 | int_value as int_expected, 33 | {{ dbt_utils.get_single_value(int_statement) }} as int_actual, 34 | 35 | string_value as string_expected, 36 | cast({{ dbt.string_literal(dbt_utils.get_single_value(string_statement)) }} as {{ dbt.type_string() }}) as string_actual 37 | 38 | from {{ ref('data_get_single_value') }} 39 | ) 40 | 41 | select * 42 | from default_data -------------------------------------------------------------------------------- /integration_tests/models/sql/test_get_single_value_default.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Dear future reader, 3 | Before you go restructuring the delicate web of casts and quotes below, a warning: 4 | I once thought as you are thinking. Proceed with caution. 5 | #} 6 | 7 | {% set false_statement = 'select 1 as id ' ~ limit_zero() %} 8 | 9 | with default_data as ( 10 | 11 | select 12 | cast({{ dbt.string_literal('2022-01-01') }} as {{ dbt.type_timestamp() }}) as date_expected, 13 | cast({{ dbt.string_literal(dbt_utils.get_single_value(false_statement, '2022-01-01')) }} as {{ dbt.type_timestamp() }}) as date_actual, 14 | 15 | 1.23456 as float_expected, 16 | {{ dbt_utils.get_single_value(false_statement, 1.23456) }} as float_actual, 17 | 18 | 123456 as int_expected, 19 | {{ dbt_utils.get_single_value(false_statement, 123456) }} as int_actual, 20 | 21 | cast({{ dbt.string_literal('fallback') }} as {{ dbt.type_string() }}) as string_expected, 22 | cast({{ dbt.string_literal(dbt_utils.get_single_value(false_statement, 'fallback')) }} as {{ dbt.type_string() }}) as string_actual 23 | 24 | from {{ ref('data_get_single_value') }} 25 | ) 26 | 27 | select * 28 | from default_data -------------------------------------------------------------------------------- /integration_tests/models/sql/test_groupby.sql: -------------------------------------------------------------------------------- 1 | with test_data as ( 2 | 3 | select 4 | 5 | {{ safe_cast("'a'", type_string() )}} as column_1, 6 | {{ safe_cast("'b'", type_string() )}} as column_2 7 | 8 | ), 9 | 10 | grouped as ( 11 | 12 | select 13 | *, 14 | count(*) as total 15 | 16 | from test_data 17 | {{ dbt_utils.group_by(2) }} 18 | 19 | ) 20 | 21 | select * from grouped 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_not_empty_string_failing.sql: -------------------------------------------------------------------------------- 1 | -- dbt seed casts '' as NULL, so we need to select empty strings to enable testing 2 | 3 | with blank_data as ( 4 | 5 | select 6 | 1 as id, 7 | 'not an empty string' as string_trim_whitespace_true 8 | 9 | union all 10 | 11 | select 12 | 2 as id, 13 | 'also not an empty string' as string_trim_whitespace_true 14 | 15 | union all 16 | 17 | select 18 | 3 as id, 19 | 'string with trailing whitespace ' as string_trim_whitespace_true 20 | 21 | union all 22 | 23 | select 24 | 4 as id, 25 | ' ' as string_trim_whitespace_true 26 | 27 | union all 28 | 29 | select 30 | 5 as id, 31 | '' as string_trim_whitespace_true 32 | 33 | union all 34 | 35 | select 36 | 6 as id, 37 | null as string_trim_whitespace_true 38 | 39 | ) 40 | 41 | select * from blank_data -------------------------------------------------------------------------------- /integration_tests/models/sql/test_not_empty_string_passing.sql: -------------------------------------------------------------------------------- 1 | -- dbt seed casts '' as NULL, so we need to select empty strings to enable testing 2 | 3 | with blank_data as ( 4 | 5 | select 6 | 1 as id, 7 | 'not an empty string' as string_trim_whitespace_true, 8 | 'not an empty string' as string_trim_whitespace_false 9 | 10 | union all 11 | 12 | select 13 | 2 as id, 14 | 'also not an empty string' as string_trim_whitespace_true, 15 | 'also not an empty string' as string_trim_whitespace_false 16 | 17 | union all 18 | 19 | select 20 | 3 as id, 21 | 'string with trailing whitespace ' as string_trim_whitespace_true, 22 | ' ' as string_trim_whitespace_false -- This will cause a failure when trim_whitespace = true 23 | 24 | union all 25 | 26 | select 27 | 6 as id, 28 | null as string_trim_whitespace_true, 29 | null as string_trim_whitespace_false 30 | 31 | ) 32 | 33 | select * from blank_data -------------------------------------------------------------------------------- /integration_tests/models/sql/test_nullcheck_table.sql: -------------------------------------------------------------------------------- 1 | {{ config( materialized = "table" ) }} 2 | 3 | -- TO DO: remove if-statement 4 | 5 | {% set tbl = ref('data_nullcheck_table') %} 6 | 7 | 8 | with nulled as ( 9 | 10 | {{ dbt_utils.nullcheck_table(tbl) }} 11 | 12 | ) 13 | 14 | {% if target.type == 'snowflake' %} 15 | 16 | select 17 | field_1::varchar as field_1, 18 | field_2::varchar as field_2, 19 | field_3::varchar as field_3 20 | 21 | from nulled 22 | 23 | {% else %} 24 | 25 | select 26 | 27 | {{ safe_cast('field_1', 28 | type_string() 29 | )}} as field_1, 30 | 31 | {{ safe_cast('field_2', 32 | type_string() 33 | )}} as field_2, 34 | 35 | {{ safe_cast('field_3', 36 | type_string() 37 | )}} as field_3 38 | 39 | from nulled 40 | 41 | {% endif %} 42 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_pivot.sql: -------------------------------------------------------------------------------- 1 | 2 | -- TODO: How do we make this work nicely on Snowflake too? 3 | 4 | {% if target.type == 'snowflake' %} 5 | {% set column_values = ['RED', 'BLUE'] %} 6 | {% set cmp = 'ilike' %} 7 | {% else %} 8 | {% set column_values = ['red', 'blue'] %} 9 | {% set cmp = '=' %} 10 | {% endif %} 11 | 12 | select 13 | size, 14 | {{ dbt_utils.pivot('color', column_values, cmp=cmp) }} 15 | 16 | from {{ ref('data_pivot') }} 17 | group by size 18 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_pivot_apostrophe.sql: -------------------------------------------------------------------------------- 1 | 2 | -- TODO: How do we make this work nicely on Snowflake too? 3 | 4 | {% if target.type == 'snowflake' %} 5 | {% set column_values = ['RED', 'BLUE', "BLUE'S"] %} 6 | {% set cmp = 'ilike' %} 7 | {% else %} 8 | {% set column_values = ['red', 'blue', "blue's"] %} 9 | {% set cmp = '=' %} 10 | {% endif %} 11 | 12 | select 13 | size, 14 | {{ dbt_utils.pivot('color', column_values, cmp=cmp, quote_identifiers=False) }} 15 | 16 | from {{ ref('data_pivot') }} 17 | group by size 18 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_safe_add.sql: -------------------------------------------------------------------------------- 1 | 2 | with data as ( 3 | 4 | select * from {{ ref('data_safe_add') }} 5 | 6 | ) 7 | 8 | select 9 | {{ dbt_utils.safe_add(['field_1', 'field_2', 'field_3']) }} as actual, 10 | expected 11 | 12 | from data 13 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_safe_divide.sql: -------------------------------------------------------------------------------- 1 | 2 | with data_safe_divide as ( 3 | 4 | select * from {{ ref('data_safe_divide') }} 5 | 6 | ), 7 | 8 | data_safe_divide_numerator_expressions as ( 9 | 10 | select * from {{ ref('data_safe_divide_numerator_expressions') }} 11 | ), 12 | 13 | data_safe_divide_denominator_expressions as ( 14 | 15 | select * from {{ ref('data_safe_divide_denominator_expressions') }} 16 | ) 17 | 18 | select 19 | {{ dbt_utils.safe_divide('numerator', 'denominator') }} as actual, 20 | output as expected 21 | 22 | from data_safe_divide 23 | 24 | union all 25 | 26 | select 27 | {{ dbt_utils.safe_divide('numerator_1 * numerator_2', 'denominator') }} as actual, 28 | output as expected 29 | 30 | from data_safe_divide_numerator_expressions 31 | 32 | union all 33 | 34 | select 35 | {{ dbt_utils.safe_divide('numerator', 'denominator_1 * denominator_2') }} as actual, 36 | output as expected 37 | 38 | from data_safe_divide_denominator_expressions -------------------------------------------------------------------------------- /integration_tests/models/sql/test_safe_subtract.sql: -------------------------------------------------------------------------------- 1 | 2 | with data as ( 3 | 4 | select * from {{ ref('data_safe_subtract') }} 5 | 6 | ) 7 | 8 | select 9 | {{ dbt_utils.safe_subtract(['field_1', 'field_2', 'field_3']) }} as actual, 10 | expected 11 | 12 | from data 13 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_star.sql: -------------------------------------------------------------------------------- 1 | {% set exclude_field = 'field_3' %} 2 | 3 | 4 | with data as ( 5 | 6 | select 7 | {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} 8 | 9 | from {{ ref('data_star') }} 10 | 11 | ) 12 | 13 | select * from data 14 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_star_aggregate.sql: -------------------------------------------------------------------------------- 1 | /*This test checks that column aliases aren't applied unless there's a prefix/suffix necessary, to ensure that GROUP BYs keep working*/ 2 | 3 | {% set selected_columns = dbt_utils.star(from=ref('data_star_aggregate'), except=['value_field']) %} 4 | 5 | with data as ( 6 | 7 | select 8 | {{ selected_columns }}, 9 | sum(value_field) as value_field_sum 10 | 11 | from {{ ref('data_star_aggregate') }} 12 | group by {{ selected_columns }} 13 | 14 | ) 15 | 16 | select * from data 17 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_star_no_columns.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | 3 | select 4 | {{ dbt_utils.star(from=ref('data_star'), except=['field_1', 'field_2', 'field_3']) }} 5 | -- if star() returns `*` or a list of columns, this query will fail because there's no comma between the columns 6 | 1 as canary_column 7 | from {{ ref('data_star') }} 8 | 9 | ) 10 | 11 | select * from data 12 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_star_prefix_suffix.sql: -------------------------------------------------------------------------------- 1 | {% set prefix_with = 'prefix_' if target.type != 'snowflake' else 'PREFIX_' %} 2 | {% set suffix_with = '_suffix' if target.type != 'snowflake' else '_SUFFIX' %} 3 | 4 | with data as ( 5 | 6 | select 7 | {{ dbt_utils.star(from=ref('data_star'), prefix=prefix_with, suffix=suffix_with) }} 8 | 9 | from {{ ref('data_star') }} 10 | 11 | ) 12 | 13 | select * from data -------------------------------------------------------------------------------- /integration_tests/models/sql/test_star_quote_identifiers.sql: -------------------------------------------------------------------------------- 1 | select 2 | {{ dbt.string_literal(adapter.quote("column_one")) | lower }} as expected, 3 | {{ dbt.string_literal(dbt_utils.star(from=ref('data_star_quote_identifiers'), quote_identifiers=True)) | trim | lower }} as actual 4 | 5 | union all 6 | 7 | select 8 | {{ dbt.string_literal("column_one") | lower }} as expected, 9 | {{ dbt.string_literal(dbt_utils.star(from=ref('data_star_quote_identifiers'), quote_identifiers=False)) | trim | lower }} as actual -------------------------------------------------------------------------------- /integration_tests/models/sql/test_star_uppercase.sql: -------------------------------------------------------------------------------- 1 | {% set exclude_field = 'FIELD_3' %} 2 | 3 | 4 | with data as ( 5 | 6 | select 7 | {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} 8 | 9 | from {{ ref('data_star') }} 10 | 11 | ) 12 | 13 | select * from data 14 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_union.sql: -------------------------------------------------------------------------------- 1 | 2 | select 3 | id, 4 | name, 5 | favorite_color, 6 | favorite_number 7 | 8 | from {{ ref('test_union_base') }} 9 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_union_base.sql: -------------------------------------------------------------------------------- 1 | 2 | {{ dbt_utils.union_relations([ 3 | ref('data_union_table_1'), 4 | ref('data_union_table_2')] 5 | ) }} 6 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_union_exclude_base_lowercase.sql: -------------------------------------------------------------------------------- 1 | 2 | {{ dbt_utils.union_relations( 3 | relations=[ 4 | ref('data_union_table_1'), 5 | ref('data_union_table_2'), 6 | ], 7 | exclude=['name'] 8 | ) }} 9 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_union_exclude_base_uppercase.sql: -------------------------------------------------------------------------------- 1 | 2 | {{ dbt_utils.union_relations( 3 | relations=[ 4 | ref('data_union_table_1'), 5 | ref('data_union_table_2'), 6 | ], 7 | exclude=['NAME'] 8 | ) }} 9 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_union_exclude_lowercase.sql: -------------------------------------------------------------------------------- 1 | select 2 | {{ dbt_utils.star(ref("test_union_exclude_base_lowercase"), except=["_dbt_source_relation"]) }} 3 | 4 | from {{ ref("test_union_exclude_base_lowercase") }} 5 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_union_exclude_uppercase.sql: -------------------------------------------------------------------------------- 1 | select 2 | {{ dbt_utils.star(ref("test_union_exclude_base_uppercase"), except=["_DBT_SOURCE_RELATION"]) }} 3 | 4 | from {{ ref("test_union_exclude_base_uppercase") }} 5 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_union_no_source_column.sql: -------------------------------------------------------------------------------- 1 | {{ dbt_utils.union_relations([ 2 | ref('data_union_table_1'), 3 | ref('data_union_table_2') 4 | ], 5 | source_column_name = none 6 | ) }} -------------------------------------------------------------------------------- /integration_tests/models/sql/test_union_where.sql: -------------------------------------------------------------------------------- 1 | select 2 | id, 3 | favorite_number 4 | from 5 | {{ ref('test_union_where_base') }} 6 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_union_where_base.sql: -------------------------------------------------------------------------------- 1 | {{ dbt_utils.union_relations( 2 | [ref('data_union_table_1'), ref('data_union_table_2')], 3 | where="id = 1" 4 | ) }} 5 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_unpivot.sql: -------------------------------------------------------------------------------- 1 | 2 | -- snowflake messes with these tests pretty badly since the 3 | -- output of the macro considers the casing of the source 4 | -- table columns. Using some hacks here to get this to work, 5 | -- but we should consider lowercasing the unpivot macro output 6 | -- at some point in the future for consistency 7 | 8 | {% if target.name == 'snowflake' %} 9 | {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} 10 | {% else %} 11 | {% set exclude = ['customer_id', 'created_at'] %} 12 | {% endif %} 13 | 14 | 15 | select 16 | customer_id, 17 | created_at, 18 | case 19 | when '{{ target.name }}' = 'snowflake' then lower(prop) 20 | else prop 21 | end as prop, 22 | val 23 | 24 | from ( 25 | {{ dbt_utils.unpivot( 26 | relation=ref('data_unpivot'), 27 | cast_to=type_string(), 28 | exclude=exclude, 29 | remove=['name'], 30 | field_name='prop', 31 | value_name='val' 32 | ) }} 33 | ) as sbq 34 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_unpivot_bool.sql: -------------------------------------------------------------------------------- 1 | 2 | -- snowflake messes with these tests pretty badly since the 3 | -- output of the macro considers the casing of the source 4 | -- table columns. Using some hacks here to get this to work, 5 | -- but we should consider lowercasing the unpivot macro output 6 | -- at some point in the future for consistency 7 | 8 | {% if target.name == 'snowflake' %} 9 | {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} 10 | {% else %} 11 | {% set exclude = ['customer_id', 'created_at'] %} 12 | {% endif %} 13 | 14 | 15 | select 16 | customer_id, 17 | created_at, 18 | case 19 | when '{{ target.name }}' = 'snowflake' then lower(prop) 20 | else prop 21 | end as prop, 22 | val 23 | 24 | from ( 25 | {{ dbt_utils.unpivot( 26 | relation=ref('data_unpivot_bool'), 27 | cast_to=type_string(), 28 | exclude=exclude, 29 | field_name='prop', 30 | value_name='val' 31 | ) }} 32 | ) as sbq 33 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_unpivot_quote.sql: -------------------------------------------------------------------------------- 1 | 2 | {{ dbt_utils.unpivot( 3 | relation=ref('data_unpivot_quote'), 4 | cast_to=type_string(), 5 | exclude=['Customer_Id', 'Created_At'], 6 | remove=['Name'], 7 | field_name='Prop', 8 | value_name='Val', 9 | quote_identifiers=True, 10 | ) }} 11 | -------------------------------------------------------------------------------- /integration_tests/models/sql/test_width_bucket.sql: -------------------------------------------------------------------------------- 1 | 2 | with data as ( 3 | 4 | select * from {{ ref('data_width_bucket') }} 5 | 6 | ) 7 | 8 | select 9 | {{ dbt_utils.width_bucket('amount', 'min_value', 'max_value', 'num_buckets') }} as actual, 10 | bucket as expected 11 | 12 | from data 13 | -------------------------------------------------------------------------------- /integration_tests/models/web/schema.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: test_urls 5 | data_tests: 6 | - assert_equal: 7 | actual: actual 8 | expected: expected 9 | 10 | - name: test_url_host 11 | data_tests: 12 | - assert_equal: 13 | actual: actual 14 | expected: expected 15 | 16 | - name: test_url_path 17 | data_tests: 18 | - assert_equal: 19 | actual: actual 20 | expected: expected -------------------------------------------------------------------------------- /integration_tests/models/web/test_url_host.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | 3 | select * from {{ref('data_url_host')}} 4 | 5 | ) 6 | 7 | select 8 | 9 | {{ dbt_utils.get_url_host('original_url') }} as actual, 10 | parsed_url as expected 11 | 12 | from data -------------------------------------------------------------------------------- /integration_tests/models/web/test_url_path.sql: -------------------------------------------------------------------------------- 1 | with data as ( 2 | 3 | select * from {{ref('data_url_path')}} 4 | 5 | ) 6 | 7 | select 8 | 9 | coalesce({{ dbt_utils.get_url_path('original_url') }}, '') as actual, 10 | coalesce(parsed_path, '') as expected 11 | 12 | from data -------------------------------------------------------------------------------- /integration_tests/models/web/test_urls.sql: -------------------------------------------------------------------------------- 1 | 2 | with data as ( 3 | 4 | select * from {{ ref('data_urls') }} 5 | 6 | ) 7 | 8 | select 9 | {{ dbt_utils.get_url_parameter('url', 'utm_medium') }} as actual, 10 | medium as expected 11 | 12 | from data 13 | 14 | union all 15 | 16 | select 17 | {{ dbt_utils.get_url_parameter('url', 'utm_source') }} as actual, 18 | source as expected 19 | 20 | from data 21 | -------------------------------------------------------------------------------- /integration_tests/package-lock.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - local: ../ 3 | sha1_hash: de2deba3d66ce03d8c02949013650cc9b94f6030 4 | -------------------------------------------------------------------------------- /integration_tests/packages.yml: -------------------------------------------------------------------------------- 1 | 2 | packages: 3 | - local: ../ 4 | -------------------------------------------------------------------------------- /integration_tests/profiles.yml: -------------------------------------------------------------------------------- 1 | 2 | # HEY! This file is used in the dbt-utils integrations tests with CircleCI. 3 | # You should __NEVER__ check credentials into version control. Thanks for reading :) 4 | 5 | 6 | integration_tests: 7 | target: postgres 8 | outputs: 9 | postgres: 10 | type: "postgres" 11 | host: "{{ env_var('POSTGRES_HOST') }}" 12 | user: "{{ env_var('POSTGRES_USER') }}" 13 | pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}" 14 | port: "{{ env_var('POSTGRES_PORT') | as_number }}" 15 | dbname: "{{ env_var('POSTGRES_DATABASE') }}" 16 | schema: "{{ env_var('POSTGRES_SCHEMA') }}" 17 | threads: 5 18 | 19 | redshift: 20 | type: "redshift" 21 | host: "{{ env_var('REDSHIFT_HOST') }}" 22 | user: "{{ env_var('REDSHIFT_USER') }}" 23 | pass: "{{ env_var('DBT_ENV_SECRET_REDSHIFT_PASS') }}" 24 | dbname: "{{ env_var('REDSHIFT_DATABASE') }}" 25 | port: "{{ env_var('REDSHIFT_PORT') | as_number }}" 26 | schema: "{{ env_var('REDSHIFT_SCHEMA') }}" 27 | threads: 5 28 | 29 | bigquery: 30 | type: "bigquery" 31 | method: "service-account-json" 32 | project: "{{ env_var('BIGQUERY_PROJECT') }}" 33 | dataset: "{{ env_var('BIGQUERY_SCHEMA') }}" 34 | threads: 10 35 | keyfile_json: 36 | "{{ env_var('BIGQUERY_KEYFILE_JSON') | as_native }}" 37 | job_retries: 3 38 | 39 | snowflake: 40 | type: "snowflake" 41 | account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" 42 | user: "{{ env_var('SNOWFLAKE_USER') }}" 43 | password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_PASS') }}" 44 | role: "{{ env_var('SNOWFLAKE_ROLE') }}" 45 | database: "{{ env_var('SNOWFLAKE_DATABASE') }}" 46 | warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE') }}" 47 | schema: "{{ env_var('SNOWFLAKE_SCHEMA') }}" 48 | threads: 10 49 | -------------------------------------------------------------------------------- /integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql: -------------------------------------------------------------------------------- 1 | -- depends_on: {{ ref('data_get_query_results_as_dict') }} 2 | 3 | {% set expected_dictionary={ 4 | 'col_1': [1, 2, 3], 5 | 'col_2': ['a', 'b', 'c'], 6 | 'col_3': [True, False, none] 7 | } %} 8 | 9 | {#- Handle snowflake casing silliness -#} 10 | {% if target.type == 'snowflake' %} 11 | {% set expected_dictionary={ 12 | 'COL_1': [1, 2, 3], 13 | 'COL_2': ['a', 'b', 'c'], 14 | 'COL_3': [True, False, none] 15 | } %} 16 | {% endif %} 17 | 18 | 19 | {% set actual_dictionary=dbt_utils.get_query_results_as_dict( 20 | "select * from " ~ ref('data_get_query_results_as_dict') ~ " order by 1" 21 | ) %} 22 | {#- 23 | For reasons that remain unclear, Jinja won't return True for actual_dictionary == expected_dictionary. 24 | Instead, we'll manually check that the values of these dictionaries are equivalent. 25 | -#} 26 | 27 | {% set ns = namespace( 28 | pass=True, 29 | err_msg = "" 30 | ) %} 31 | {% if execute %} 32 | {#- Check that the dictionaries have the same keys -#} 33 | {% set expected_keys=expected_dictionary.keys() | list | sort %} 34 | {% set actual_keys=actual_dictionary.keys() | list | sort %} 35 | 36 | {% if expected_keys != actual_keys %} 37 | {% set ns.pass=False %} 38 | {% set ns.err_msg %} 39 | The two dictionaries have different keys: 40 | expected_dictionary has keys: {{ expected_keys }} 41 | actual_dictionary has keys: {{ actual_keys }} 42 | {% endset %} 43 | 44 | {% else %} 45 | 46 | {% for key, value in expected_dictionary.items() %} 47 | {% set expected_length=expected_dictionary[key] | length %} 48 | {% set actual_length=actual_dictionary[key] | length %} 49 | 50 | {% if expected_length != actual_length %} 51 | {% set ns.pass=False %} 52 | {% set ns.err_msg %} 53 | The {{ key }} column has different lengths: 54 | expected_dictionary[{{ key }}] has length {{ expected_length }} 55 | actual_dictionary[{{ key }}] has length {{ actual_length }} 56 | {% endset %} 57 | 58 | {% else %} 59 | 60 | {% for i in range(value | length) %} 61 | {% set expected_value=expected_dictionary[key][i] %} 62 | {% set actual_value=actual_dictionary[key][i] %} 63 | {% if expected_value != actual_value %} 64 | {% set ns.pass=False %} 65 | {% set ns.err_msg %} 66 | The {{ key }} column has differing values: 67 | expected_dictionary[{{ key }}][{{ i }}] == {{ expected_value }} 68 | actual_dictionary[{{ key }}][{{ i }}] == {{ actual_value }} 69 | {% endset %} 70 | 71 | {% endif %} 72 | {% endfor %} 73 | {% endif %} 74 | 75 | {% endfor %} 76 | 77 | {% endif %} 78 | 79 | {{ log(ns.err_msg, info=True) }} 80 | select 1 as col_name {% if ns.pass %} {{ limit_zero() }} {% endif %} 81 | {% endif %} 82 | -------------------------------------------------------------------------------- /integration_tests/tests/generic/expect_table_columns_to_match_set.sql: -------------------------------------------------------------------------------- 1 | {# 2 | This macro is copied and slightly edited from the dbt_expectations package. 3 | At the time of this addition, dbt_expectations couldn't be added because 4 | integration_tests is installing dbt_utils from local without a hard-coded 5 | path. dbt is not able to resolve duplicate dependencies of dbt_utils 6 | due to this. 7 | #} 8 | 9 | {%- test expect_table_columns_to_match_set(model, column_list, transform="upper") -%} 10 | {%- if execute -%} 11 | {%- set column_list = column_list | map(transform) | list -%} 12 | 13 | {# Replaces dbt_expectations._get_column_list() #} 14 | {%- set relation_column_names = adapter.get_columns_in_relation(model) 15 | | map(attribute="name") 16 | | map(transform) 17 | | list 18 | -%} 19 | 20 | {# Replaces dbt_expectations._list_intersect() #} 21 | {%- set matching_columns = [] -%} 22 | {%- for itm in column_list -%} 23 | {%- if itm in relation_column_names -%} 24 | {%- do matching_columns.append(itm) -%} 25 | {%- endif -%} 26 | {%- endfor -%} 27 | 28 | with relation_columns as ( 29 | 30 | {% for col_name in relation_column_names %} 31 | select cast('{{ col_name }}' as {{ type_string() }}) as relation_column 32 | {% if not loop.last %}union all{% endif %} 33 | {% endfor %} 34 | ), 35 | input_columns as ( 36 | 37 | {% for col_name in column_list %} 38 | select cast('{{ col_name }}' as {{ type_string() }}) as input_column 39 | {% if not loop.last %}union all{% endif %} 40 | {% endfor %} 41 | ) 42 | select * 43 | from 44 | relation_columns r 45 | full outer join 46 | input_columns i on r.relation_column = i.input_column 47 | where 48 | -- catch any column in input list that is not in the list of table columns 49 | -- or any table column that is not in the input list 50 | r.relation_column is null or 51 | i.input_column is null 52 | 53 | {%- endif -%} 54 | {%- endtest -%} 55 | -------------------------------------------------------------------------------- /integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql: -------------------------------------------------------------------------------- 1 | {% if dbt_utils.pretty_log_format() is string %} 2 | {# Return 0 rows for the test to pass #} 3 | select 1 as col_name {{ limit_zero() }} 4 | {% else %} 5 | {# Return >0 rows for the test to fail #} 6 | select 1 7 | {% endif %} 8 | -------------------------------------------------------------------------------- /integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql: -------------------------------------------------------------------------------- 1 | {% if dbt_utils.pretty_time() is string %} 2 | {# Return 0 rows for the test to pass #} 3 | select 1 as col_name {{ limit_zero() }} 4 | {% else %} 5 | {# Return >0 rows for the test to fail #} 6 | select 1 7 | {% endif %} 8 | -------------------------------------------------------------------------------- /integration_tests/tests/jinja_helpers/test_slugify.sql: -------------------------------------------------------------------------------- 1 | with comparisons as ( 2 | select '{{ dbt_utils.slugify("") }}' as output, '' as expected 3 | union all 4 | select '{{ dbt_utils.slugify(None) }}' as output, '' as expected 5 | union all 6 | select '{{ dbt_utils.slugify("!Hell0 world-hi") }}' as output, 'hell0_world_hi' as expected 7 | union all 8 | select '{{ dbt_utils.slugify("0Hell0 world-hi") }}' as output, '_0hell0_world_hi' as expected 9 | ) 10 | 11 | select * 12 | from comparisons 13 | where output != expected 14 | -------------------------------------------------------------------------------- /integration_tests/tests/sql/test_get_column_values_use_default.sql: -------------------------------------------------------------------------------- 1 | 2 | {# 3 | This keeps succeeding locally and failing in CI. Disabling it to get everything else out, but it should still be tested. 4 | https://github.com/dbt-labs/dbt-utils/issues/788 5 | #} 6 | 7 | {{ config(enabled = false)}} 8 | 9 | {% set column_values = dbt_utils.get_column_values(ref('data_get_column_values_dropped'), 'field', default=['y', 'z'], order_by="field") %} 10 | 11 | with expected as ( 12 | select {{ safe_cast("'y'", type_string()) }} as expected_column_value union all 13 | select {{ safe_cast("'z'", type_string()) }} as expected_column_value 14 | ), 15 | 16 | actual as ( 17 | 18 | {% for val in column_values %} 19 | select {{ safe_cast("'" ~ val ~ "'", type_string()) }} as actual_column_value 20 | {% if not loop.last %} 21 | union all 22 | {% endif %} 23 | {% endfor %} 24 | ), 25 | 26 | failures as ( 27 | select * from actual 28 | where actual.actual_column_value not in ( 29 | select expected.expected_column_value from expected 30 | ) 31 | ) 32 | 33 | select * from failures 34 | -------------------------------------------------------------------------------- /integration_tests/tests/sql/test_get_single_value_multiple_rows.sql: -------------------------------------------------------------------------------- 1 | {% set query %} 2 | with input as ( 3 | select 1 as id, 4 as di 4 | union all 5 | select 2 as id, 5 as di 6 | union all 7 | select 3 as id, 6 as di 8 | ) 9 | {% endset %} 10 | 11 | with comparisons as ( 12 | select {{ dbt_utils.get_single_value(query ~ " select min(id) from input") }} as output, 1 as expected 13 | union all 14 | select {{ dbt_utils.get_single_value(query ~ " select max(di) from input") }} as output, 6 as expected 15 | ) 16 | select * 17 | from comparisons 18 | where output != expected -------------------------------------------------------------------------------- /macros/generic_tests/accepted_range.sql: -------------------------------------------------------------------------------- 1 | {% test accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %} 2 | {{ return(adapter.dispatch('test_accepted_range', 'dbt_utils')(model, column_name, min_value, max_value, inclusive)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %} 6 | 7 | with meet_condition as( 8 | select * 9 | from {{ model }} 10 | ), 11 | 12 | validation_errors as ( 13 | select * 14 | from meet_condition 15 | where 16 | -- never true, defaults to an empty result set. Exists to ensure any combo of the `or` clauses below succeeds 17 | 1 = 2 18 | 19 | {%- if min_value is not none %} 20 | -- records with a value >= min_value are permitted. The `not` flips this to find records that don't meet the rule. 21 | or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} 22 | {%- endif %} 23 | 24 | {%- if max_value is not none %} 25 | -- records with a value <= max_value are permitted. The `not` flips this to find records that don't meet the rule. 26 | or not {{ column_name }} < {{- "=" if inclusive }} {{ max_value }} 27 | {%- endif %} 28 | ) 29 | 30 | select * 31 | from validation_errors 32 | 33 | {% endmacro %} 34 | -------------------------------------------------------------------------------- /macros/generic_tests/at_least_one.sql: -------------------------------------------------------------------------------- 1 | {% test at_least_one(model, column_name, group_by_columns = []) %} 2 | {{ return(adapter.dispatch('test_at_least_one', 'dbt_utils')(model, column_name, group_by_columns)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_at_least_one(model, column_name, group_by_columns) %} 6 | 7 | {% set pruned_cols = [column_name] %} 8 | 9 | {% if group_by_columns|length() > 0 %} 10 | 11 | {% set select_gb_cols = group_by_columns|join(' ,') + ', ' %} 12 | {% set groupby_gb_cols = 'group by ' + group_by_columns|join(',') %} 13 | {% set pruned_cols = group_by_columns %} 14 | 15 | {% if column_name not in pruned_cols %} 16 | {% do pruned_cols.append(column_name) %} 17 | {% endif %} 18 | 19 | {% endif %} 20 | 21 | {% set select_pruned_cols = pruned_cols|join(' ,') %} 22 | 23 | select * 24 | from ( 25 | with pruned_rows as ( 26 | select 27 | {{ select_pruned_cols }} 28 | from {{ model }} 29 | {% if group_by_columns|length() == 0 %} 30 | where {{ column_name }} is not null 31 | limit 1 32 | {% endif %} 33 | ) 34 | select 35 | {# In TSQL, subquery aggregate columns need aliases #} 36 | {# thus: a filler col name, 'filler_column' #} 37 | {{select_gb_cols}} 38 | count({{ column_name }}) as filler_column 39 | 40 | from pruned_rows 41 | 42 | {{groupby_gb_cols}} 43 | 44 | having count({{ column_name }}) = 0 45 | 46 | ) validation_errors 47 | 48 | {% endmacro %} 49 | -------------------------------------------------------------------------------- /macros/generic_tests/cardinality_equality.sql: -------------------------------------------------------------------------------- 1 | {% test cardinality_equality(model, column_name, to, field) %} 2 | {{ return(adapter.dispatch('test_cardinality_equality', 'dbt_utils')(model, column_name, to, field)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_cardinality_equality(model, column_name, to, field) %} 6 | 7 | {# T-SQL does not let you use numbers as aliases for columns #} 8 | {# Thus, no "GROUP BY 1" #} 9 | 10 | with table_a as ( 11 | select 12 | {{ column_name }}, 13 | count(*) as num_rows 14 | from {{ model }} 15 | group by {{ column_name }} 16 | ), 17 | 18 | table_b as ( 19 | select 20 | {{ field }}, 21 | count(*) as num_rows 22 | from {{ to }} 23 | group by {{ field }} 24 | ), 25 | 26 | except_a as ( 27 | select * 28 | from table_a 29 | {{ dbt.except() }} 30 | select * 31 | from table_b 32 | ), 33 | 34 | except_b as ( 35 | select * 36 | from table_b 37 | {{ dbt.except() }} 38 | select * 39 | from table_a 40 | ), 41 | 42 | unioned as ( 43 | select * 44 | from except_a 45 | union all 46 | select * 47 | from except_b 48 | ) 49 | 50 | select * 51 | from unioned 52 | 53 | {% endmacro %} 54 | -------------------------------------------------------------------------------- /macros/generic_tests/equal_rowcount.sql: -------------------------------------------------------------------------------- 1 | {% test equal_rowcount(model, compare_model, group_by_columns = []) %} 2 | {{ return(adapter.dispatch('test_equal_rowcount', 'dbt_utils')(model, compare_model, group_by_columns)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_equal_rowcount(model, compare_model, group_by_columns) %} 6 | 7 | {#-- Needs to be set at parse time, before we return '' below --#} 8 | {{ config(fail_calc = 'sum(coalesce(diff_count, 0))') }} 9 | 10 | {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} 11 | {%- if not execute -%} 12 | {{ return('') }} 13 | {% endif %} 14 | 15 | {% if group_by_columns|length() > 0 %} 16 | {% set select_gb_cols = group_by_columns|join(', ') + ', ' %} 17 | {% set join_gb_cols %} 18 | {% for c in group_by_columns %} 19 | and a.{{c}} = b.{{c}} 20 | {% endfor %} 21 | {% endset %} 22 | {% set groupby_gb_cols = 'group by ' + group_by_columns|join(',') %} 23 | {% endif %} 24 | 25 | {#-- We must add a fake join key in case additional grouping variables are not provided --#} 26 | {#-- Redshift does not allow for dynamically created join conditions (e.g. full join on 1 = 1 --#} 27 | {#-- The same logic is used in fewer_rows_than. In case of changes, maintain consistent logic --#} 28 | {% set group_by_columns = ['id_dbtutils_test_equal_rowcount'] + group_by_columns %} 29 | {% set groupby_gb_cols = 'group by ' + group_by_columns|join(',') %} 30 | 31 | with a as ( 32 | 33 | select 34 | {{select_gb_cols}} 35 | 1 as id_dbtutils_test_equal_rowcount, 36 | count(*) as count_a 37 | from {{ model }} 38 | {{groupby_gb_cols}} 39 | 40 | 41 | ), 42 | b as ( 43 | 44 | select 45 | {{select_gb_cols}} 46 | 1 as id_dbtutils_test_equal_rowcount, 47 | count(*) as count_b 48 | from {{ compare_model }} 49 | {{groupby_gb_cols}} 50 | 51 | ), 52 | final as ( 53 | 54 | select 55 | 56 | {% for c in group_by_columns -%} 57 | a.{{c}} as {{c}}_a, 58 | b.{{c}} as {{c}}_b, 59 | {% endfor %} 60 | 61 | count_a, 62 | count_b, 63 | abs(count_a - count_b) as diff_count 64 | 65 | from a 66 | full join b 67 | on 68 | a.id_dbtutils_test_equal_rowcount = b.id_dbtutils_test_equal_rowcount 69 | {{join_gb_cols}} 70 | 71 | 72 | ) 73 | 74 | select * from final 75 | 76 | {% endmacro %} 77 | -------------------------------------------------------------------------------- /macros/generic_tests/equality.sql: -------------------------------------------------------------------------------- 1 | {% test equality(model, compare_model, compare_columns=None, exclude_columns=None, precision = None) %} 2 | {{ return(adapter.dispatch('test_equality', 'dbt_utils')(model, compare_model, compare_columns, exclude_columns, precision)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_equality(model, compare_model, compare_columns=None, exclude_columns=None, precision = None) %} 6 | 7 | {%- if compare_columns and exclude_columns -%} 8 | {{ exceptions.raise_compiler_error("Both a compare and an ignore list were provided to the `equality` macro. Only one is allowed") }} 9 | {%- endif -%} 10 | 11 | {% set set_diff %} 12 | count(*) + coalesce(abs( 13 | sum(case when which_diff = 'a_minus_b' then 1 else 0 end) - 14 | sum(case when which_diff = 'b_minus_a' then 1 else 0 end) 15 | ), 0) 16 | {% endset %} 17 | 18 | {#-- Needs to be set at parse time, before we return '' below --#} 19 | {{ config(fail_calc = set_diff) }} 20 | 21 | {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} 22 | {%- if not execute -%} 23 | {{ return('') }} 24 | {% endif %} 25 | 26 | 27 | 28 | -- setup 29 | {%- do dbt_utils._is_relation(model, 'test_equality') -%} 30 | 31 | {# Ensure there are no extra columns in the compare_model vs model #} 32 | {%- if not compare_columns -%} 33 | {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} 34 | {%- do dbt_utils._is_ephemeral(compare_model, 'test_equality') -%} 35 | 36 | {%- set model_columns = adapter.get_columns_in_relation(model) -%} 37 | {%- set compare_model_columns = adapter.get_columns_in_relation(compare_model) -%} 38 | 39 | 40 | {%- if exclude_columns -%} 41 | {#-- Lower case ignore columns for easier comparison --#} 42 | {%- set exclude_columns = exclude_columns | map("lower") | list %} 43 | 44 | {# Filter out the excluded columns #} 45 | {%- set include_columns = [] %} 46 | {%- set include_model_columns = [] %} 47 | {%- for column in model_columns -%} 48 | {%- if column.name | lower not in exclude_columns -%} 49 | {% do include_columns.append(column) %} 50 | {%- endif %} 51 | {%- endfor %} 52 | {%- for column in compare_model_columns -%} 53 | {%- if column.name | lower not in exclude_columns -%} 54 | {% do include_model_columns.append(column) %} 55 | {%- endif %} 56 | {%- endfor %} 57 | 58 | {%- set compare_columns_set = set(include_columns | map(attribute='quoted') | map("lower")) %} 59 | {%- set compare_model_columns_set = set(include_model_columns | map(attribute='quoted') | map("lower")) %} 60 | {%- else -%} 61 | {%- set compare_columns_set = set(model_columns | map(attribute='quoted') | map("lower")) %} 62 | {%- set compare_model_columns_set = set(compare_model_columns | map(attribute='quoted') | map("lower")) %} 63 | {%- endif -%} 64 | 65 | {% if compare_columns_set != compare_model_columns_set %} 66 | {{ exceptions.raise_compiler_error(compare_model ~" has less columns than " ~ model ~ ", please ensure they have the same columns or use the `compare_columns` or `exclude_columns` arguments to subset them.") }} 67 | {% endif %} 68 | 69 | 70 | {% endif %} 71 | 72 | {%- if not precision -%} 73 | {%- if not compare_columns -%} 74 | {# 75 | You cannot get the columns in an ephemeral model (due to not existing in the information schema), 76 | so if the user does not provide an explicit list of columns we must error in the case it is ephemeral 77 | #} 78 | {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} 79 | {%- set compare_columns = adapter.get_columns_in_relation(model)-%} 80 | 81 | {%- if exclude_columns -%} 82 | {#-- Lower case ignore columns for easier comparison --#} 83 | {%- set exclude_columns = exclude_columns | map("lower") | list %} 84 | 85 | {# Filter out the excluded columns #} 86 | {%- set include_columns = [] %} 87 | {%- for column in compare_columns -%} 88 | {%- if column.name | lower not in exclude_columns -%} 89 | {% do include_columns.append(column) %} 90 | {%- endif %} 91 | {%- endfor %} 92 | 93 | {%- set compare_columns = include_columns | map(attribute='quoted') %} 94 | {%- else -%} {# Compare columns provided #} 95 | {%- set compare_columns = compare_columns | map(attribute='quoted') %} 96 | {%- endif -%} 97 | {%- endif -%} 98 | 99 | {% set compare_cols_csv = compare_columns | join(', ') %} 100 | 101 | {% else %} {# Precision required #} 102 | {#- 103 | If rounding is required, we need to get the types, so it cannot be ephemeral even if they provide column names 104 | -#} 105 | {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} 106 | {%- set columns = adapter.get_columns_in_relation(model) -%} 107 | 108 | {% set columns_list = [] %} 109 | {%- for col in columns -%} 110 | {%- if ( 111 | (col.name|lower in compare_columns|map('lower') or not compare_columns) and 112 | (col.name|lower not in exclude_columns|map('lower') or not exclude_columns) 113 | ) -%} 114 | {# Databricks double type is not picked up by any number type checks in dbt #} 115 | {%- if col.is_float() or col.is_numeric() or col.data_type == 'double' -%} 116 | {# Cast is required due to postgres not having round for a double precision number #} 117 | {%- do columns_list.append('round(cast(' ~ col.quoted ~ ' as ' ~ dbt.type_numeric() ~ '),' ~ precision ~ ') as ' ~ col.quoted) -%} 118 | {%- else -%} {# Non-numeric type #} 119 | {%- do columns_list.append(col.quoted) -%} 120 | {%- endif -%} 121 | {% endif %} 122 | {%- endfor -%} 123 | 124 | {% set compare_cols_csv = columns_list | join(', ') %} 125 | 126 | {% endif %} 127 | 128 | with a as ( 129 | 130 | select * from {{ model }} 131 | 132 | ), 133 | 134 | b as ( 135 | 136 | select * from {{ compare_model }} 137 | 138 | ), 139 | 140 | a_minus_b as ( 141 | 142 | select {{compare_cols_csv}} from a 143 | {{ dbt.except() }} 144 | select {{compare_cols_csv}} from b 145 | 146 | ), 147 | 148 | b_minus_a as ( 149 | 150 | select {{compare_cols_csv}} from b 151 | {{ dbt.except() }} 152 | select {{compare_cols_csv}} from a 153 | 154 | ), 155 | 156 | unioned as ( 157 | 158 | select 'a_minus_b' as which_diff, a_minus_b.* from a_minus_b 159 | union all 160 | select 'b_minus_a' as which_diff, b_minus_a.* from b_minus_a 161 | 162 | ) 163 | 164 | select * from unioned 165 | 166 | {% endmacro %} 167 | -------------------------------------------------------------------------------- /macros/generic_tests/expression_is_true.sql: -------------------------------------------------------------------------------- 1 | {% test expression_is_true(model, expression, column_name=None) %} 2 | {{ return(adapter.dispatch('test_expression_is_true', 'dbt_utils')(model, expression, column_name)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_expression_is_true(model, expression, column_name) %} 6 | 7 | {% set column_list = '*' if should_store_failures() else "1" %} 8 | 9 | select 10 | {{ column_list }} 11 | from {{ model }} 12 | {% if column_name is none %} 13 | where not({{ expression }}) 14 | {%- else %} 15 | where not({{ column_name }} {{ expression }}) 16 | {%- endif %} 17 | 18 | {% endmacro %} 19 | -------------------------------------------------------------------------------- /macros/generic_tests/fewer_rows_than.sql: -------------------------------------------------------------------------------- 1 | {% test fewer_rows_than(model, compare_model, group_by_columns = []) %} 2 | {{ return(adapter.dispatch('test_fewer_rows_than', 'dbt_utils')(model, compare_model, group_by_columns)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_fewer_rows_than(model, compare_model, group_by_columns) %} 6 | 7 | {{ config(fail_calc = 'sum(coalesce(row_count_delta, 0))') }} 8 | 9 | {% if group_by_columns|length() > 0 %} 10 | {% set select_gb_cols = group_by_columns|join(' ,') + ', ' %} 11 | {% set join_gb_cols %} 12 | {% for c in group_by_columns %} 13 | and a.{{c}} = b.{{c}} 14 | {% endfor %} 15 | {% endset %} 16 | {% set groupby_gb_cols = 'group by ' + group_by_columns|join(',') %} 17 | {% endif %} 18 | 19 | {#-- We must add a fake join key in case additional grouping variables are not provided --#} 20 | {#-- Redshift does not allow for dynamically created join conditions (e.g. full join on 1 = 1 --#} 21 | {#-- The same logic is used in equal_rowcount. In case of changes, maintain consistent logic --#} 22 | {% set group_by_columns = ['id_dbtutils_test_fewer_rows_than'] + group_by_columns %} 23 | {% set groupby_gb_cols = 'group by ' + group_by_columns|join(',') %} 24 | 25 | 26 | with a as ( 27 | 28 | select 29 | {{select_gb_cols}} 30 | 1 as id_dbtutils_test_fewer_rows_than, 31 | count(*) as count_our_model 32 | from {{ model }} 33 | {{ groupby_gb_cols }} 34 | 35 | ), 36 | b as ( 37 | 38 | select 39 | {{select_gb_cols}} 40 | 1 as id_dbtutils_test_fewer_rows_than, 41 | count(*) as count_comparison_model 42 | from {{ compare_model }} 43 | {{ groupby_gb_cols }} 44 | 45 | ), 46 | counts as ( 47 | 48 | select 49 | 50 | {% for c in group_by_columns -%} 51 | a.{{c}} as {{c}}_a, 52 | b.{{c}} as {{c}}_b, 53 | {% endfor %} 54 | 55 | count_our_model, 56 | count_comparison_model 57 | from a 58 | full join b on 59 | a.id_dbtutils_test_fewer_rows_than = b.id_dbtutils_test_fewer_rows_than 60 | {{ join_gb_cols }} 61 | 62 | ), 63 | final as ( 64 | 65 | select *, 66 | case 67 | -- fail the test if we have more rows than the reference model and return the row count delta 68 | when count_our_model > count_comparison_model then (count_our_model - count_comparison_model) 69 | -- fail the test if they are the same number 70 | when count_our_model = count_comparison_model then 1 71 | -- pass the test if the delta is positive (i.e. return the number 0) 72 | else 0 73 | end as row_count_delta 74 | from counts 75 | 76 | ) 77 | 78 | select * from final 79 | 80 | {% endmacro %} 81 | -------------------------------------------------------------------------------- /macros/generic_tests/mutually_exclusive_ranges.sql: -------------------------------------------------------------------------------- 1 | {% test mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %} 2 | {{ return(adapter.dispatch('test_mutually_exclusive_ranges', 'dbt_utils')(model, lower_bound_column, upper_bound_column, partition_by, gaps, zero_length_range_allowed)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %} 6 | {% if gaps == 'not_allowed' %} 7 | {% set allow_gaps_operator='=' %} 8 | {% set allow_gaps_operator_in_words='equal_to' %} 9 | {% elif gaps == 'allowed' %} 10 | {% set allow_gaps_operator='<=' %} 11 | {% set allow_gaps_operator_in_words='less_than_or_equal_to' %} 12 | {% elif gaps == 'required' %} 13 | {% set allow_gaps_operator='<' %} 14 | {% set allow_gaps_operator_in_words='less_than' %} 15 | {% else %} 16 | {{ exceptions.raise_compiler_error( 17 | "`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '" ~ gaps ~"'.'" 18 | ) }} 19 | {% endif %} 20 | {% if not zero_length_range_allowed %} 21 | {% set allow_zero_length_operator='<' %} 22 | {% set allow_zero_length_operator_in_words='less_than' %} 23 | {% elif zero_length_range_allowed %} 24 | {% set allow_zero_length_operator='<=' %} 25 | {% set allow_zero_length_operator_in_words='less_than_or_equal_to' %} 26 | {% else %} 27 | {{ exceptions.raise_compiler_error( 28 | "`zero_length_range_allowed` argument for mutually_exclusive_ranges test must be one of [true, false] Got: '" ~ zero_length_range_allowed ~"'.'" 29 | ) }} 30 | {% endif %} 31 | 32 | {% set partition_clause="partition by " ~ partition_by if partition_by else '' %} 33 | 34 | with window_functions as ( 35 | 36 | select 37 | {% if partition_by %} 38 | {{ partition_by }} as partition_by_col, 39 | {% endif %} 40 | {{ lower_bound_column }} as lower_bound, 41 | {{ upper_bound_column }} as upper_bound, 42 | 43 | lead({{ lower_bound_column }}) over ( 44 | {{ partition_clause }} 45 | order by {{ lower_bound_column }}, {{ upper_bound_column }} 46 | ) as next_lower_bound, 47 | 48 | row_number() over ( 49 | {{ partition_clause }} 50 | order by {{ lower_bound_column }} desc, {{ upper_bound_column }} desc 51 | ) = 1 as is_last_record 52 | 53 | from {{ model }} 54 | 55 | ), 56 | 57 | calc as ( 58 | -- We want to return records where one of our assumptions fails, so we'll use 59 | -- the `not` function with `and` statements so we can write our assumptions more cleanly 60 | select 61 | *, 62 | 63 | -- For each record: lower_bound should be < upper_bound. 64 | -- Coalesce it to return an error on the null case (implicit assumption 65 | -- these columns are not_null) 66 | coalesce( 67 | lower_bound {{ allow_zero_length_operator }} upper_bound, 68 | false 69 | ) as lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound, 70 | 71 | -- For each record: upper_bound {{ allow_gaps_operator }} the next lower_bound. 72 | -- Coalesce it to handle null cases for the last record. 73 | coalesce( 74 | upper_bound {{ allow_gaps_operator }} next_lower_bound, 75 | is_last_record, 76 | false 77 | ) as upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound 78 | 79 | from window_functions 80 | 81 | ), 82 | 83 | validation_errors as ( 84 | 85 | select 86 | * 87 | from calc 88 | 89 | where not( 90 | -- THE FOLLOWING SHOULD BE TRUE -- 91 | lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound 92 | and upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound 93 | ) 94 | ) 95 | 96 | select * from validation_errors 97 | {% endmacro %} 98 | -------------------------------------------------------------------------------- /macros/generic_tests/not_accepted_values.sql: -------------------------------------------------------------------------------- 1 | {% test not_accepted_values(model, column_name, values, quote=True) %} 2 | {{ return(adapter.dispatch('test_not_accepted_values', 'dbt_utils')(model, column_name, values, quote)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_not_accepted_values(model, column_name, values, quote=True) %} 6 | with all_values as ( 7 | 8 | select distinct 9 | {{ column_name }} as value_field 10 | 11 | from {{ model }} 12 | 13 | ), 14 | 15 | validation_errors as ( 16 | 17 | select 18 | value_field 19 | 20 | from all_values 21 | where value_field in ( 22 | {% for value in values -%} 23 | {% if quote -%} 24 | '{{ value }}' 25 | {%- else -%} 26 | {{ value }} 27 | {%- endif -%} 28 | {%- if not loop.last -%},{%- endif %} 29 | {%- endfor %} 30 | ) 31 | 32 | ) 33 | 34 | select * 35 | from validation_errors 36 | 37 | {% endmacro %} 38 | -------------------------------------------------------------------------------- /macros/generic_tests/not_constant.sql: -------------------------------------------------------------------------------- 1 | 2 | {% test not_constant(model, column_name, group_by_columns = []) %} 3 | {{ return(adapter.dispatch('test_not_constant', 'dbt_utils')(model, column_name, group_by_columns)) }} 4 | {% endtest %} 5 | 6 | {% macro default__test_not_constant(model, column_name, group_by_columns) %} 7 | 8 | {% if group_by_columns|length() > 0 %} 9 | {% set select_gb_cols = group_by_columns|join(' ,') + ', ' %} 10 | {% set groupby_gb_cols = 'group by ' + group_by_columns|join(',') %} 11 | {% endif %} 12 | 13 | 14 | select 15 | {# In TSQL, subquery aggregate columns need aliases #} 16 | {# thus: a filler col name, 'filler_column' #} 17 | {{select_gb_cols}} 18 | count(distinct {{ column_name }}) as filler_column 19 | 20 | from {{ model }} 21 | 22 | {{groupby_gb_cols}} 23 | 24 | having count(distinct {{ column_name }}) = 1 25 | 26 | 27 | {% endmacro %} 28 | -------------------------------------------------------------------------------- /macros/generic_tests/not_empty_string.sql: -------------------------------------------------------------------------------- 1 | {% test not_empty_string(model, column_name, trim_whitespace=true) %} 2 | 3 | {{ return(adapter.dispatch('test_not_empty_string', 'dbt_utils')(model, column_name, trim_whitespace)) }} 4 | 5 | {% endtest %} 6 | 7 | {% macro default__test_not_empty_string(model, column_name, trim_whitespace=true) %} 8 | 9 | with 10 | 11 | all_values as ( 12 | 13 | select 14 | 15 | 16 | {% if trim_whitespace == true -%} 17 | 18 | trim({{ column_name }}) as {{ column_name }} 19 | 20 | {%- else -%} 21 | 22 | {{ column_name }} 23 | 24 | {%- endif %} 25 | 26 | from {{ model }} 27 | 28 | ), 29 | 30 | errors as ( 31 | 32 | select * from all_values 33 | where {{ column_name }} = '' 34 | 35 | ) 36 | 37 | select * from errors 38 | 39 | {% endmacro %} -------------------------------------------------------------------------------- /macros/generic_tests/not_null_proportion.sql: -------------------------------------------------------------------------------- 1 | {% macro test_not_null_proportion(model, group_by_columns = []) %} 2 | {{ return(adapter.dispatch('test_not_null_proportion', 'dbt_utils')(model, group_by_columns, **kwargs)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__test_not_null_proportion(model, group_by_columns) %} 6 | 7 | {% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} 8 | {% set at_least = kwargs.get('at_least', kwargs.get('arg')) %} 9 | {% set at_most = kwargs.get('at_most', kwargs.get('arg', 1)) %} 10 | 11 | {% if group_by_columns|length() > 0 %} 12 | {% set select_gb_cols = group_by_columns|join(' ,') + ', ' %} 13 | {% set groupby_gb_cols = 'group by ' + group_by_columns|join(',') %} 14 | {% endif %} 15 | 16 | with validation as ( 17 | select 18 | {{select_gb_cols}} 19 | sum(case when {{ column_name }} is null then 0 else 1 end) / cast(count(*) as {{ dbt.type_numeric() }}) as not_null_proportion 20 | from {{ model }} 21 | {{groupby_gb_cols}} 22 | ), 23 | validation_errors as ( 24 | select 25 | {{select_gb_cols}} 26 | not_null_proportion 27 | from validation 28 | where not_null_proportion < {{ at_least }} or not_null_proportion > {{ at_most }} 29 | ) 30 | select 31 | * 32 | from validation_errors 33 | 34 | {% endmacro %} 35 | -------------------------------------------------------------------------------- /macros/generic_tests/recency.sql: -------------------------------------------------------------------------------- 1 | {% test recency(model, field, datepart, interval, ignore_time_component=False, group_by_columns = []) %} 2 | {{ return(adapter.dispatch('test_recency', 'dbt_utils')(model, field, datepart, interval, ignore_time_component, group_by_columns)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_recency(model, field, datepart, interval, ignore_time_component, group_by_columns) %} 6 | 7 | {% set threshold = 'cast(' ~ dbt.dateadd(datepart, interval * -1, dbt.current_timestamp()) ~ ' as ' ~ ('date' if ignore_time_component else dbt.type_timestamp()) ~ ')' %} 8 | 9 | {% if group_by_columns|length() > 0 %} 10 | {% set select_gb_cols = group_by_columns|join(' ,') + ', ' %} 11 | {% set groupby_gb_cols = 'group by ' + group_by_columns|join(',') %} 12 | {% endif %} 13 | 14 | 15 | with recency as ( 16 | 17 | select 18 | 19 | {{ select_gb_cols }} 20 | {% if ignore_time_component %} 21 | cast(max({{ field }}) as date) as most_recent 22 | {%- else %} 23 | max({{ field }}) as most_recent 24 | {%- endif %} 25 | 26 | from {{ model }} 27 | 28 | {{ groupby_gb_cols }} 29 | 30 | ) 31 | 32 | select 33 | 34 | {{ select_gb_cols }} 35 | most_recent, 36 | {{ threshold }} as threshold 37 | 38 | from recency 39 | where most_recent < {{ threshold }} 40 | 41 | {% endmacro %} 42 | -------------------------------------------------------------------------------- /macros/generic_tests/relationships_where.sql: -------------------------------------------------------------------------------- 1 | {% test relationships_where(model, column_name, to, field, from_condition="1=1", to_condition="1=1") %} 2 | {{ return(adapter.dispatch('test_relationships_where', 'dbt_utils')(model, column_name, to, field, from_condition, to_condition)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_relationships_where(model, column_name, to, field, from_condition="1=1", to_condition="1=1") %} 6 | 7 | {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} 8 | {# ref https://stackoverflow.com/a/7170753/3842610 #} 9 | 10 | with left_table as ( 11 | 12 | select 13 | {{column_name}} as id 14 | 15 | from {{model}} 16 | 17 | where {{column_name}} is not null 18 | and {{from_condition}} 19 | 20 | ), 21 | 22 | right_table as ( 23 | 24 | select 25 | {{field}} as id 26 | 27 | from {{to}} 28 | 29 | where {{field}} is not null 30 | and {{to_condition}} 31 | 32 | ), 33 | 34 | exceptions as ( 35 | 36 | select 37 | left_table.id, 38 | right_table.id as right_id 39 | 40 | from left_table 41 | 42 | left join right_table 43 | on left_table.id = right_table.id 44 | 45 | where right_table.id is null 46 | 47 | ) 48 | 49 | select * from exceptions 50 | 51 | {% endmacro %} 52 | -------------------------------------------------------------------------------- /macros/generic_tests/sequential_values.sql: -------------------------------------------------------------------------------- 1 | {% test sequential_values(model, column_name, interval=1, datepart=None, group_by_columns = []) %} 2 | 3 | {{ return(adapter.dispatch('test_sequential_values', 'dbt_utils')(model, column_name, interval, datepart, group_by_columns)) }} 4 | 5 | {% endtest %} 6 | 7 | {% macro default__test_sequential_values(model, column_name, interval=1, datepart=None, group_by_columns = []) %} 8 | 9 | {% set previous_column_name = "previous_" ~ dbt_utils.slugify(column_name) %} 10 | 11 | {% if group_by_columns|length() > 0 %} 12 | {% set select_gb_cols = group_by_columns|join(',') + ', ' %} 13 | {% set partition_gb_cols = 'partition by ' + group_by_columns|join(',') %} 14 | {% endif %} 15 | 16 | with windowed as ( 17 | 18 | select 19 | {{ select_gb_cols }} 20 | {{ column_name }}, 21 | lag({{ column_name }}) over ( 22 | {{partition_gb_cols}} 23 | order by {{ column_name }} 24 | ) as {{ previous_column_name }} 25 | from {{ model }} 26 | ), 27 | 28 | validation_errors as ( 29 | select 30 | * 31 | from windowed 32 | {% if datepart %} 33 | where not(cast({{ column_name }} as {{ dbt.type_timestamp() }})= cast({{ dbt.dateadd(datepart, interval, previous_column_name) }} as {{ dbt.type_timestamp() }})) 34 | {% else %} 35 | where not({{ column_name }} = {{ previous_column_name }} + {{ interval }}) 36 | {% endif %} 37 | ) 38 | 39 | select * 40 | from validation_errors 41 | 42 | {% endmacro %} 43 | -------------------------------------------------------------------------------- /macros/generic_tests/unique_combination_of_columns.sql: -------------------------------------------------------------------------------- 1 | {% test unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %} 2 | {{ return(adapter.dispatch('test_unique_combination_of_columns', 'dbt_utils')(model, combination_of_columns, quote_columns)) }} 3 | {% endtest %} 4 | 5 | {% macro default__test_unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %} 6 | 7 | {% if not quote_columns %} 8 | {%- set column_list=combination_of_columns %} 9 | {% elif quote_columns %} 10 | {%- set column_list=[] %} 11 | {% for column in combination_of_columns -%} 12 | {% set column_list = column_list.append( adapter.quote(column) ) %} 13 | {%- endfor %} 14 | {% else %} 15 | {{ exceptions.raise_compiler_error( 16 | "`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '" ~ quote ~"'.'" 17 | ) }} 18 | {% endif %} 19 | 20 | {%- set columns_csv=column_list | join(', ') %} 21 | 22 | 23 | with validation_errors as ( 24 | 25 | select 26 | {{ columns_csv }} 27 | from {{ model }} 28 | group by {{ columns_csv }} 29 | having count(*) > 1 30 | 31 | ) 32 | 33 | select * 34 | from validation_errors 35 | 36 | 37 | {% endmacro %} 38 | -------------------------------------------------------------------------------- /macros/jinja_helpers/_is_ephemeral.sql: -------------------------------------------------------------------------------- 1 | {% macro _is_ephemeral(obj, macro) %} 2 | {%- if obj.is_cte -%} 3 | {% set ephemeral_prefix = api.Relation.add_ephemeral_prefix('') %} 4 | {% if obj.name.startswith(ephemeral_prefix) %} 5 | {% set model_name = obj.name[(ephemeral_prefix|length):] %} 6 | {% else %} 7 | {% set model_name = obj.name %} 8 | {%- endif -%} 9 | {% set error_message %} 10 | The `{{ macro }}` macro cannot be used with ephemeral models, as it relies on the information schema. 11 | 12 | `{{ model_name }}` is an ephemeral model. Consider making it a view or table instead. 13 | {% endset %} 14 | {%- do exceptions.raise_compiler_error(error_message) -%} 15 | {%- endif -%} 16 | {% endmacro %} 17 | -------------------------------------------------------------------------------- /macros/jinja_helpers/_is_relation.sql: -------------------------------------------------------------------------------- 1 | {% macro _is_relation(obj, macro) %} 2 | {%- if not (obj is mapping and obj.get('metadata', {}).get('type', '').endswith('Relation')) -%} 3 | {%- do exceptions.raise_compiler_error("Macro " ~ macro ~ " expected a Relation but received the value: " ~ obj) -%} 4 | {%- endif -%} 5 | {% endmacro %} 6 | -------------------------------------------------------------------------------- /macros/jinja_helpers/log_info.sql: -------------------------------------------------------------------------------- 1 | {% macro log_info(message) %} 2 | {{ return(adapter.dispatch('log_info', 'dbt_utils')(message)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__log_info(message) %} 6 | {{ log(dbt_utils.pretty_log_format(message), info=True) }} 7 | {% endmacro %} 8 | -------------------------------------------------------------------------------- /macros/jinja_helpers/pretty_log_format.sql: -------------------------------------------------------------------------------- 1 | {% macro pretty_log_format(message) %} 2 | {{ return(adapter.dispatch('pretty_log_format', 'dbt_utils')(message)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__pretty_log_format(message) %} 6 | {{ return( dbt_utils.pretty_time() ~ ' + ' ~ message) }} 7 | {% endmacro %} 8 | -------------------------------------------------------------------------------- /macros/jinja_helpers/pretty_time.sql: -------------------------------------------------------------------------------- 1 | {% macro pretty_time(format='%H:%M:%S') %} 2 | {{ return(adapter.dispatch('pretty_time', 'dbt_utils')(format)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__pretty_time(format='%H:%M:%S') %} 6 | {{ return(modules.datetime.datetime.now().strftime(format)) }} 7 | {% endmacro %} 8 | -------------------------------------------------------------------------------- /macros/jinja_helpers/slugify.sql: -------------------------------------------------------------------------------- 1 | {% macro slugify(string) %} 2 | 3 | {% if not string %} 4 | {{ return('') }} 5 | {% endif %} 6 | 7 | {#- Lower case the string -#} 8 | {% set string = string | lower %} 9 | {#- Replace spaces and dashes with underscores -#} 10 | {% set string = modules.re.sub('[ -]+', '_', string) %} 11 | {#- Only take letters, numbers, and underscores -#} 12 | {% set string = modules.re.sub('[^a-z0-9_]+', '', string) %} 13 | {#- Prepends "_" if string begins with a number -#} 14 | {% set string = modules.re.sub('^[0-9]', '_' + string[0], string) %} 15 | 16 | {{ return(string) }} 17 | 18 | {% endmacro %} 19 | -------------------------------------------------------------------------------- /macros/sql/date_spine.sql: -------------------------------------------------------------------------------- 1 | {% macro get_intervals_between(start_date, end_date, datepart) -%} 2 | {{ return(adapter.dispatch('get_intervals_between', 'dbt_utils')(start_date, end_date, datepart)) }} 3 | {%- endmacro %} 4 | 5 | {% macro default__get_intervals_between(start_date, end_date, datepart) -%} 6 | {%- call statement('get_intervals_between', fetch_result=True) %} 7 | 8 | select {{ dbt.datediff(start_date, end_date, datepart) }} 9 | 10 | {%- endcall -%} 11 | 12 | {%- set value_list = load_result('get_intervals_between') -%} 13 | 14 | {%- if value_list and value_list['data'] -%} 15 | {%- set values = value_list['data'] | map(attribute=0) | list %} 16 | {{ return(values[0]) }} 17 | {%- else -%} 18 | {{ return(1) }} 19 | {%- endif -%} 20 | 21 | {%- endmacro %} 22 | 23 | 24 | 25 | 26 | {% macro date_spine(datepart, start_date, end_date) %} 27 | {{ return(adapter.dispatch('date_spine', 'dbt_utils')(datepart, start_date, end_date)) }} 28 | {%- endmacro %} 29 | 30 | {% macro default__date_spine(datepart, start_date, end_date) %} 31 | 32 | 33 | {# call as follows: 34 | 35 | date_spine( 36 | "day", 37 | "to_date('01/01/2016', 'mm/dd/yyyy')", 38 | "dbt.dateadd(week, 1, current_date)" 39 | ) #} 40 | 41 | 42 | with rawdata as ( 43 | 44 | {{dbt_utils.generate_series( 45 | dbt_utils.get_intervals_between(start_date, end_date, datepart) 46 | )}} 47 | 48 | ), 49 | 50 | all_periods as ( 51 | 52 | select ( 53 | {{ 54 | dbt.dateadd( 55 | datepart, 56 | "row_number() over (order by 1) - 1", 57 | start_date 58 | ) 59 | }} 60 | ) as date_{{datepart}} 61 | from rawdata 62 | 63 | ), 64 | 65 | filtered as ( 66 | 67 | select * 68 | from all_periods 69 | where date_{{datepart}} <= {{ end_date }} 70 | 71 | ) 72 | 73 | select * from filtered 74 | 75 | {% endmacro %} 76 | -------------------------------------------------------------------------------- /macros/sql/deduplicate.sql: -------------------------------------------------------------------------------- 1 | {%- macro deduplicate(relation, partition_by, order_by) -%} 2 | {{ return(adapter.dispatch('deduplicate', 'dbt_utils')(relation, partition_by, order_by)) }} 3 | {% endmacro %} 4 | 5 | {%- macro default__deduplicate(relation, partition_by, order_by) -%} 6 | 7 | with row_numbered as ( 8 | select 9 | _inner.*, 10 | row_number() over ( 11 | partition by {{ partition_by }} 12 | order by {{ order_by }} 13 | ) as rn 14 | from {{ relation }} as _inner 15 | ) 16 | 17 | select 18 | distinct data.* 19 | from {{ relation }} as data 20 | {# 21 | -- Not all DBs will support natural joins but the ones that do include: 22 | -- Oracle, MySQL, SQLite, Redshift, Teradata, Materialize, Databricks 23 | -- Apache Spark, SingleStore, Vertica 24 | -- Those that do not appear to support natural joins include: 25 | -- SQLServer, Trino, Presto, Rockset, Athena 26 | #} 27 | natural join row_numbered 28 | where row_numbered.rn = 1 29 | 30 | {%- endmacro -%} 31 | 32 | -- Redshift has the `QUALIFY` syntax: 33 | -- https://docs.aws.amazon.com/redshift/latest/dg/r_QUALIFY_clause.html 34 | {% macro redshift__deduplicate(relation, partition_by, order_by) -%} 35 | 36 | select * 37 | from {{ relation }} as tt 38 | qualify 39 | row_number() over ( 40 | partition by {{ partition_by }} 41 | order by {{ order_by }} 42 | ) = 1 43 | 44 | {% endmacro %} 45 | 46 | {# 47 | -- Postgres has the `DISTINCT ON` syntax: 48 | -- https://www.postgresql.org/docs/current/sql-select.html#SQL-DISTINCT 49 | #} 50 | {%- macro postgres__deduplicate(relation, partition_by, order_by) -%} 51 | 52 | select 53 | distinct on ({{ partition_by }}) * 54 | from {{ relation }} 55 | order by {{ partition_by }}{{ ',' ~ order_by }} 56 | 57 | {%- endmacro -%} 58 | 59 | {# 60 | -- Snowflake has the `QUALIFY` syntax: 61 | -- https://docs.snowflake.com/en/sql-reference/constructs/qualify.html 62 | #} 63 | {%- macro snowflake__deduplicate(relation, partition_by, order_by) -%} 64 | 65 | select * 66 | from {{ relation }} 67 | qualify 68 | row_number() over ( 69 | partition by {{ partition_by }} 70 | order by {{ order_by }} 71 | ) = 1 72 | 73 | {%- endmacro -%} 74 | 75 | {# 76 | -- Databricks also has the `QUALIFY` syntax: 77 | -- https://docs.databricks.com/sql/language-manual/sql-ref-syntax-qry-select-qualify.html 78 | #} 79 | {%- macro databricks__deduplicate(relation, partition_by, order_by) -%} 80 | 81 | select * 82 | from {{ relation }} 83 | qualify 84 | row_number() over ( 85 | partition by {{ partition_by }} 86 | order by {{ order_by }} 87 | ) = 1 88 | 89 | {%- endmacro -%} 90 | 91 | {# 92 | -- It is more performant to deduplicate using `array_agg` with a limit 93 | -- clause in BigQuery: 94 | -- https://github.com/dbt-labs/dbt-utils/issues/335#issuecomment-788157572 95 | #} 96 | {%- macro bigquery__deduplicate(relation, partition_by, order_by) -%} 97 | 98 | select unique.* 99 | from ( 100 | select 101 | array_agg ( 102 | original 103 | order by {{ order_by }} 104 | limit 1 105 | )[offset(0)] unique 106 | from {{ relation }} original 107 | group by {{ partition_by }} 108 | ) 109 | 110 | {%- endmacro -%} 111 | -------------------------------------------------------------------------------- /macros/sql/generate_series.sql: -------------------------------------------------------------------------------- 1 | {% macro get_powers_of_two(upper_bound) %} 2 | {{ return(adapter.dispatch('get_powers_of_two', 'dbt_utils')(upper_bound)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_powers_of_two(upper_bound) %} 6 | 7 | {% if upper_bound <= 0 %} 8 | {{ exceptions.raise_compiler_error("upper bound must be positive") }} 9 | {% endif %} 10 | 11 | {% for _ in range(1, 100) %} 12 | {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %} 13 | {% endfor %} 14 | 15 | {% endmacro %} 16 | 17 | 18 | {% macro generate_series(upper_bound) %} 19 | {{ return(adapter.dispatch('generate_series', 'dbt_utils')(upper_bound)) }} 20 | {% endmacro %} 21 | 22 | {% macro default__generate_series(upper_bound) %} 23 | 24 | {% set n = dbt_utils.get_powers_of_two(upper_bound) %} 25 | 26 | with p as ( 27 | select 0 as generated_number union all select 1 28 | ), unioned as ( 29 | 30 | select 31 | 32 | {% for i in range(n) %} 33 | p{{i}}.generated_number * power(2, {{i}}) 34 | {% if not loop.last %} + {% endif %} 35 | {% endfor %} 36 | + 1 37 | as generated_number 38 | 39 | from 40 | 41 | {% for i in range(n) %} 42 | p as p{{i}} 43 | {% if not loop.last %} cross join {% endif %} 44 | {% endfor %} 45 | 46 | ) 47 | 48 | select * 49 | from unioned 50 | where generated_number <= {{upper_bound}} 51 | order by generated_number 52 | 53 | {% endmacro %} 54 | -------------------------------------------------------------------------------- /macros/sql/generate_surrogate_key.sql: -------------------------------------------------------------------------------- 1 | {%- macro generate_surrogate_key(field_list) -%} 2 | {{ return(adapter.dispatch('generate_surrogate_key', 'dbt_utils')(field_list)) }} 3 | {% endmacro %} 4 | 5 | {%- macro default__generate_surrogate_key(field_list) -%} 6 | 7 | {%- if var('surrogate_key_treat_nulls_as_empty_strings', False) -%} 8 | {%- set default_null_value = "" -%} 9 | {%- else -%} 10 | {%- set default_null_value = '_dbt_utils_surrogate_key_null_' -%} 11 | {%- endif -%} 12 | 13 | {%- set fields = [] -%} 14 | 15 | {%- for field in field_list -%} 16 | 17 | {%- do fields.append( 18 | "coalesce(cast(" ~ field ~ " as " ~ dbt.type_string() ~ "), '" ~ default_null_value ~"')" 19 | ) -%} 20 | 21 | {%- if not loop.last %} 22 | {%- do fields.append("'-'") -%} 23 | {%- endif -%} 24 | 25 | {%- endfor -%} 26 | 27 | {{ dbt.hash(dbt.concat(fields)) }} 28 | 29 | {%- endmacro -%} 30 | -------------------------------------------------------------------------------- /macros/sql/get_column_values.sql: -------------------------------------------------------------------------------- 1 | {% macro get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none, where=none) -%} 2 | {{ return(adapter.dispatch('get_column_values', 'dbt_utils')(table, column, order_by, max_records, default, where)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none, where=none) -%} 6 | {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} 7 | {%- if not execute -%} 8 | {% set default = [] if not default %} 9 | {{ return(default) }} 10 | {% endif %} 11 | 12 | {%- do dbt_utils._is_ephemeral(table, 'get_column_values') -%} 13 | 14 | {# Not all relations are tables. Renaming for internal clarity without breaking functionality for anyone using named arguments #} 15 | {# TODO: Change the method signature in a future 0.x.0 release #} 16 | {%- set target_relation = table -%} 17 | 18 | {# adapter.load_relation is a convenience wrapper to avoid building a Relation when we already have one #} 19 | {% set relation_exists = (load_relation(target_relation)) is not none %} 20 | 21 | {%- call statement('get_column_values', fetch_result=true) %} 22 | 23 | {%- if not relation_exists and default is none -%} 24 | 25 | {{ exceptions.raise_compiler_error("In get_column_values(): relation " ~ target_relation ~ " does not exist and no default value was provided.") }} 26 | 27 | {%- elif not relation_exists and default is not none -%} 28 | 29 | {{ log("Relation " ~ target_relation ~ " does not exist. Returning the default value: " ~ default) }} 30 | 31 | {{ return(default) }} 32 | 33 | {%- else -%} 34 | 35 | 36 | select 37 | {{ column }} as value 38 | 39 | from {{ target_relation }} 40 | 41 | {% if where is not none %} 42 | where {{ where }} 43 | {% endif %} 44 | 45 | group by {{ column }} 46 | order by {{ order_by }} 47 | 48 | {% if max_records is not none %} 49 | limit {{ max_records }} 50 | {% endif %} 51 | 52 | {% endif %} 53 | 54 | {%- endcall -%} 55 | 56 | {%- set value_list = load_result('get_column_values') -%} 57 | 58 | {%- if value_list and value_list['data'] -%} 59 | {%- set values = value_list['data'] | map(attribute=0) | list %} 60 | {{ return(values) }} 61 | {%- else -%} 62 | {{ return(default) }} 63 | {%- endif -%} 64 | 65 | {%- endmacro %} 66 | -------------------------------------------------------------------------------- /macros/sql/get_filtered_columns_in_relation.sql: -------------------------------------------------------------------------------- 1 | {% macro get_filtered_columns_in_relation(from, except=[]) -%} 2 | {{ return(adapter.dispatch('get_filtered_columns_in_relation', 'dbt_utils')(from, except)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_filtered_columns_in_relation(from, except=[]) -%} 6 | {%- do dbt_utils._is_relation(from, 'get_filtered_columns_in_relation') -%} 7 | {%- do dbt_utils._is_ephemeral(from, 'get_filtered_columns_in_relation') -%} 8 | 9 | {# -- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} 10 | {%- if not execute -%} 11 | {{ return('') }} 12 | {% endif %} 13 | 14 | {%- set include_cols = [] %} 15 | {%- set cols = adapter.get_columns_in_relation(from) -%} 16 | {%- set except = except | map("lower") | list %} 17 | {%- for col in cols -%} 18 | {%- if col.column|lower not in except -%} 19 | {% do include_cols.append(col.column) %} 20 | {%- endif %} 21 | {%- endfor %} 22 | 23 | {{ return(include_cols) }} 24 | 25 | {%- endmacro %} -------------------------------------------------------------------------------- /macros/sql/get_query_results_as_dict.sql: -------------------------------------------------------------------------------- 1 | {% macro get_query_results_as_dict(query) %} 2 | {{ return(adapter.dispatch('get_query_results_as_dict', 'dbt_utils')(query)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_query_results_as_dict(query) %} 6 | 7 | {# This macro returns a dictionary of the form {column_name: (tuple_of_results)} #} 8 | 9 | {%- call statement('get_query_results', fetch_result=True,auto_begin=false) -%} 10 | 11 | {{ query }} 12 | 13 | {%- endcall -%} 14 | 15 | {% set sql_results={} %} 16 | 17 | {%- if execute -%} 18 | {% set sql_results_table = load_result('get_query_results').table.columns %} 19 | {% for column_name, column in sql_results_table.items() %} 20 | {% do sql_results.update({column_name: column.values()}) %} 21 | {% endfor %} 22 | {%- endif -%} 23 | 24 | {{ return(sql_results) }} 25 | 26 | {% endmacro %} 27 | -------------------------------------------------------------------------------- /macros/sql/get_relations_by_pattern.sql: -------------------------------------------------------------------------------- 1 | {% macro get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} 2 | {{ return(adapter.dispatch('get_relations_by_pattern', 'dbt_utils')(schema_pattern, table_pattern, exclude, database)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} 6 | 7 | {%- call statement('get_tables', fetch_result=True) %} 8 | 9 | {{ dbt_utils.get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude, database) }} 10 | 11 | {%- endcall -%} 12 | 13 | {%- set table_list = load_result('get_tables') -%} 14 | 15 | {%- if table_list and table_list['table'] -%} 16 | {%- set tbl_relations = [] -%} 17 | {%- for row in table_list['table'] -%} 18 | {%- set tbl_relation = api.Relation.create( 19 | database=database, 20 | schema=row.table_schema, 21 | identifier=row.table_name, 22 | type=row.table_type 23 | ) -%} 24 | {%- do tbl_relations.append(tbl_relation) -%} 25 | {%- endfor -%} 26 | 27 | {{ return(tbl_relations) }} 28 | {%- else -%} 29 | {{ return([]) }} 30 | {%- endif -%} 31 | 32 | {% endmacro %} 33 | -------------------------------------------------------------------------------- /macros/sql/get_relations_by_prefix.sql: -------------------------------------------------------------------------------- 1 | {% macro get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %} 2 | {{ return(adapter.dispatch('get_relations_by_prefix', 'dbt_utils')(schema, prefix, exclude, database)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %} 6 | 7 | {%- call statement('get_tables', fetch_result=True) %} 8 | 9 | {{ dbt_utils.get_tables_by_prefix_sql(schema, prefix, exclude, database) }} 10 | 11 | {%- endcall -%} 12 | 13 | {%- set table_list = load_result('get_tables') -%} 14 | 15 | {%- if table_list and table_list['table'] -%} 16 | {%- set tbl_relations = [] -%} 17 | {%- for row in table_list['table'] -%} 18 | {%- set tbl_relation = api.Relation.create( 19 | database=database, 20 | schema=row.table_schema, 21 | identifier=row.table_name, 22 | type=row.table_type 23 | ) -%} 24 | {%- do tbl_relations.append(tbl_relation) -%} 25 | {%- endfor -%} 26 | 27 | {{ return(tbl_relations) }} 28 | {%- else -%} 29 | {{ return([]) }} 30 | {%- endif -%} 31 | 32 | {% endmacro %} 33 | -------------------------------------------------------------------------------- /macros/sql/get_single_value.sql: -------------------------------------------------------------------------------- 1 | {% macro get_single_value(query, default=none) %} 2 | {{ return(adapter.dispatch('get_single_value', 'dbt_utils')(query, default)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_single_value(query, default) %} 6 | 7 | {# This macro returns the (0, 0) record in a query, i.e. the first row of the first column #} 8 | 9 | {%- call statement('get_query_result', fetch_result=True, auto_begin=false) -%} 10 | 11 | {{ query }} 12 | 13 | {%- endcall -%} 14 | 15 | {%- if execute -%} 16 | 17 | {% set r = load_result('get_query_result').table.columns[0].values() %} 18 | {% if r | length == 0 %} 19 | {% do print('Query `' ~ query ~ '` returned no rows. Using the default value: ' ~ default) %} 20 | {% set sql_result = default %} 21 | {% else %} 22 | {% set sql_result = r[0] %} 23 | {% endif %} 24 | 25 | {%- else -%} 26 | 27 | {% set sql_result = default %} 28 | 29 | {%- endif -%} 30 | 31 | {% do return(sql_result) %} 32 | 33 | {% endmacro %} -------------------------------------------------------------------------------- /macros/sql/get_table_types_sql.sql: -------------------------------------------------------------------------------- 1 | {%- macro get_table_types_sql() -%} 2 | {{ return(adapter.dispatch('get_table_types_sql', 'dbt_utils')()) }} 3 | {%- endmacro -%} 4 | 5 | {% macro default__get_table_types_sql() %} 6 | case table_type 7 | when 'BASE TABLE' then 'table' 8 | when 'EXTERNAL TABLE' then 'external' 9 | when 'MATERIALIZED VIEW' then 'materializedview' 10 | else lower(table_type) 11 | end as {{ adapter.quote('table_type') }} 12 | {% endmacro %} 13 | 14 | 15 | {% macro postgres__get_table_types_sql() %} 16 | case table_type 17 | when 'BASE TABLE' then 'table' 18 | when 'FOREIGN' then 'external' 19 | when 'MATERIALIZED VIEW' then 'materializedview' 20 | else lower(table_type) 21 | end as {{ adapter.quote('table_type') }} 22 | {% endmacro %} 23 | 24 | 25 | {% macro databricks__get_table_types_sql() %} 26 | case table_type 27 | when 'MANAGED' then 'table' 28 | when 'BASE TABLE' then 'table' 29 | when 'MATERIALIZED VIEW' then 'materializedview' 30 | else lower(table_type) 31 | end as {{ adapter.quote('table_type') }} 32 | {% endmacro %} 33 | -------------------------------------------------------------------------------- /macros/sql/get_tables_by_pattern_sql.sql: -------------------------------------------------------------------------------- 1 | {% macro get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} 2 | {{ return(adapter.dispatch('get_tables_by_pattern_sql', 'dbt_utils') 3 | (schema_pattern, table_pattern, exclude, database)) }} 4 | {% endmacro %} 5 | 6 | {% macro default__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} 7 | 8 | select distinct 9 | table_schema as {{ adapter.quote('table_schema') }}, 10 | table_name as {{ adapter.quote('table_name') }}, 11 | {{ dbt_utils.get_table_types_sql() }} 12 | from {{ database }}.information_schema.tables 13 | where table_schema ilike '{{ schema_pattern }}' 14 | and table_name ilike '{{ table_pattern }}' 15 | and table_name not ilike '{{ exclude }}' 16 | 17 | {% endmacro %} 18 | 19 | {% macro redshift__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} 20 | 21 | {% set sql %} 22 | select distinct 23 | table_schema as {{ adapter.quote('table_schema') }}, 24 | table_name as {{ adapter.quote('table_name') }}, 25 | {{ dbt_utils.get_table_types_sql() }} 26 | from "{{ database }}"."information_schema"."tables" 27 | where table_schema ilike '{{ schema_pattern }}' 28 | and table_name ilike '{{ table_pattern }}' 29 | and table_name not ilike '{{ exclude }}' 30 | union all 31 | select distinct 32 | schemaname as {{ adapter.quote('table_schema') }}, 33 | tablename as {{ adapter.quote('table_name') }}, 34 | 'external' as {{ adapter.quote('table_type') }} 35 | from svv_external_tables 36 | where redshift_database_name = '{{ database }}' 37 | and schemaname ilike '{{ schema_pattern }}' 38 | and table_name ilike '{{ table_pattern }}' 39 | and table_name not ilike '{{ exclude }}' 40 | {% endset %} 41 | 42 | {{ return(sql) }} 43 | {% endmacro %} 44 | 45 | 46 | {% macro bigquery__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} 47 | 48 | {% if '%' in schema_pattern %} 49 | {% set schemata=dbt_utils._bigquery__get_matching_schemata(schema_pattern, database) %} 50 | {% else %} 51 | {% set schemata=[schema_pattern] %} 52 | {% endif %} 53 | 54 | {% set sql %} 55 | {% for schema in schemata %} 56 | select distinct 57 | table_schema, 58 | table_name, 59 | {{ dbt_utils.get_table_types_sql() }} 60 | 61 | from {{ adapter.quote(database) }}.{{ schema }}.INFORMATION_SCHEMA.TABLES 62 | where lower(table_name) like lower ('{{ table_pattern }}') 63 | and lower(table_name) not like lower ('{{ exclude }}') 64 | 65 | {% if not loop.last %} union all {% endif %} 66 | 67 | {% endfor %} 68 | {% endset %} 69 | 70 | {{ return(sql) }} 71 | 72 | {% endmacro %} 73 | 74 | 75 | {% macro _bigquery__get_matching_schemata(schema_pattern, database) %} 76 | {% if execute %} 77 | 78 | {% set sql %} 79 | select schema_name from {{ adapter.quote(database) }}.INFORMATION_SCHEMA.SCHEMATA 80 | where lower(schema_name) like lower('{{ schema_pattern }}') 81 | {% endset %} 82 | 83 | {% set results=run_query(sql) %} 84 | 85 | {% set schemata=results.columns['schema_name'].values() %} 86 | 87 | {{ return(schemata) }} 88 | 89 | {% else %} 90 | 91 | {{ return([]) }} 92 | 93 | {% endif %} 94 | 95 | 96 | {% endmacro %} 97 | -------------------------------------------------------------------------------- /macros/sql/get_tables_by_prefix_sql.sql: -------------------------------------------------------------------------------- 1 | {% macro get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} 2 | {{ return(adapter.dispatch('get_tables_by_prefix_sql', 'dbt_utils')(schema, prefix, exclude, database)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} 6 | 7 | {{ dbt_utils.get_tables_by_pattern_sql( 8 | schema_pattern = schema, 9 | table_pattern = prefix ~ '%', 10 | exclude = exclude, 11 | database = database 12 | ) }} 13 | 14 | {% endmacro %} 15 | -------------------------------------------------------------------------------- /macros/sql/groupby.sql: -------------------------------------------------------------------------------- 1 | {%- macro group_by(n) -%} 2 | {{ return(adapter.dispatch('group_by', 'dbt_utils')(n)) }} 3 | {% endmacro %} 4 | 5 | {%- macro default__group_by(n) -%} 6 | 7 | group by {% for i in range(1, n + 1) -%} 8 | {{ i }}{{ ',' if not loop.last }} 9 | {%- endfor -%} 10 | 11 | {%- endmacro -%} 12 | -------------------------------------------------------------------------------- /macros/sql/haversine_distance.sql: -------------------------------------------------------------------------------- 1 | {# 2 | This calculates the distance between two sets of latitude and longitude. 3 | The formula is from the following blog post: 4 | http://daynebatten.com/2015/09/latitude-longitude-distance-sql/ 5 | 6 | The arguments should be float type. 7 | #} 8 | 9 | {% macro degrees_to_radians(degrees) -%} 10 | acos(-1) * {{degrees}} / 180 11 | {%- endmacro %} 12 | 13 | {% macro haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} 14 | {{ return(adapter.dispatch('haversine_distance', 'dbt_utils')(lat1,lon1,lat2,lon2,unit)) }} 15 | {% endmacro %} 16 | 17 | {% macro default__haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} 18 | {%- if unit == 'mi' %} 19 | {% set conversion_rate = 1 %} 20 | {% elif unit == 'km' %} 21 | {% set conversion_rate = 1.60934 %} 22 | {% else %} 23 | {{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} 24 | {% endif %} 25 | 26 | 2 * 3961 * asin(sqrt(power((sin(radians(({{ lat2 }} - {{ lat1 }}) / 2))), 2) + 27 | cos(radians({{lat1}})) * cos(radians({{lat2}})) * 28 | power((sin(radians(({{ lon2 }} - {{ lon1 }}) / 2))), 2))) * {{ conversion_rate }} 29 | 30 | {%- endmacro %} 31 | 32 | 33 | 34 | {% macro bigquery__haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} 35 | {% set radians_lat1 = dbt_utils.degrees_to_radians(lat1) %} 36 | {% set radians_lat2 = dbt_utils.degrees_to_radians(lat2) %} 37 | {% set radians_lon1 = dbt_utils.degrees_to_radians(lon1) %} 38 | {% set radians_lon2 = dbt_utils.degrees_to_radians(lon2) %} 39 | {%- if unit == 'mi' %} 40 | {% set conversion_rate = 1 %} 41 | {% elif unit == 'km' %} 42 | {% set conversion_rate = 1.60934 %} 43 | {% else %} 44 | {{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} 45 | {% endif %} 46 | 2 * 3961 * asin(sqrt(power(sin(({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + 47 | cos({{ radians_lat1 }}) * cos({{ radians_lat2 }}) * 48 | power(sin(({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2))) * {{ conversion_rate }} 49 | 50 | {%- endmacro %} 51 | 52 | -------------------------------------------------------------------------------- /macros/sql/nullcheck.sql: -------------------------------------------------------------------------------- 1 | {% macro nullcheck(cols) %} 2 | {{ return(adapter.dispatch('nullcheck', 'dbt_utils')(cols)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__nullcheck(cols) %} 6 | {%- for col in cols %} 7 | 8 | {% if col.is_string() -%} 9 | 10 | nullif({{col.name}},'') as {{col.name}} 11 | 12 | {%- else -%} 13 | 14 | {{col.name}} 15 | 16 | {%- endif -%} 17 | 18 | {%- if not loop.last -%} , {%- endif -%} 19 | 20 | {%- endfor -%} 21 | {% endmacro %} 22 | -------------------------------------------------------------------------------- /macros/sql/nullcheck_table.sql: -------------------------------------------------------------------------------- 1 | {% macro nullcheck_table(relation) %} 2 | {{ return(adapter.dispatch('nullcheck_table', 'dbt_utils')(relation)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__nullcheck_table(relation) %} 6 | 7 | {%- do dbt_utils._is_relation(relation, 'nullcheck_table') -%} 8 | {%- do dbt_utils._is_ephemeral(relation, 'nullcheck_table') -%} 9 | {% set cols = adapter.get_columns_in_relation(relation) %} 10 | 11 | select {{ dbt_utils.nullcheck(cols) }} 12 | from {{relation}} 13 | 14 | {% endmacro %} 15 | -------------------------------------------------------------------------------- /macros/sql/pivot.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Pivot values from rows to columns. 3 | 4 | Example: 5 | 6 | Input: `public.test` 7 | 8 | | size | color | 9 | |------+-------| 10 | | S | red | 11 | | S | blue | 12 | | S | red | 13 | | M | red | 14 | 15 | select 16 | size, 17 | {{ dbt_utils.pivot('color', dbt_utils.get_column_values('public.test', 18 | 'color')) }} 19 | from public.test 20 | group by size 21 | 22 | Output: 23 | 24 | | size | red | blue | 25 | |------+-----+------| 26 | | S | 2 | 1 | 27 | | M | 1 | 0 | 28 | 29 | Arguments: 30 | column: Column name, required 31 | values: List of row values to turn into columns, required 32 | alias: Whether to create column aliases, default is True 33 | agg: SQL aggregation function, default is sum 34 | cmp: SQL value comparison, default is = 35 | prefix: Column alias prefix, default is blank 36 | suffix: Column alias postfix, default is blank 37 | then_value: Value to use if comparison succeeds, default is 1 38 | else_value: Value to use if comparison fails, default is 0 39 | quote_identifiers: Whether to surround column aliases with double quotes, default is true 40 | distinct: Whether to use distinct in the aggregation, default is False 41 | #} 42 | 43 | {% macro pivot(column, 44 | values, 45 | alias=True, 46 | agg='sum', 47 | cmp='=', 48 | prefix='', 49 | suffix='', 50 | then_value=1, 51 | else_value=0, 52 | quote_identifiers=True, 53 | distinct=False) %} 54 | {{ return(adapter.dispatch('pivot', 'dbt_utils')(column, values, alias, agg, cmp, prefix, suffix, then_value, else_value, quote_identifiers, distinct)) }} 55 | {% endmacro %} 56 | 57 | {% macro default__pivot(column, 58 | values, 59 | alias=True, 60 | agg='sum', 61 | cmp='=', 62 | prefix='', 63 | suffix='', 64 | then_value=1, 65 | else_value=0, 66 | quote_identifiers=True, 67 | distinct=False) %} 68 | {% for value in values %} 69 | {{ agg }}( 70 | {% if distinct %} distinct {% endif %} 71 | case 72 | when {{ column }} {{ cmp }} '{{ dbt.escape_single_quotes(value) }}' 73 | then {{ then_value }} 74 | else {{ else_value }} 75 | end 76 | ) 77 | {% if alias %} 78 | {% if quote_identifiers %} 79 | as {{ adapter.quote(prefix ~ value ~ suffix) }} 80 | {% else %} 81 | as {{ dbt_utils.slugify(prefix ~ value ~ suffix) }} 82 | {% endif %} 83 | {% endif %} 84 | {% if not loop.last %},{% endif %} 85 | {% endfor %} 86 | {% endmacro %} 87 | -------------------------------------------------------------------------------- /macros/sql/safe_add.sql: -------------------------------------------------------------------------------- 1 | {%- macro safe_add(field_list) -%} 2 | {{ return(adapter.dispatch('safe_add', 'dbt_utils')(field_list)) }} 3 | {% endmacro %} 4 | 5 | {%- macro default__safe_add(field_list) -%} 6 | 7 | {%- if field_list is not iterable or field_list is string or field_list is mapping -%} 8 | 9 | {%- set error_message = ' 10 | Warning: the `safe_add` macro now takes a single list argument instead of \ 11 | string arguments. The {}.{} model triggered this warning. \ 12 | '.format(model.package_name, model.name) -%} 13 | 14 | {%- do exceptions.warn(error_message) -%} 15 | 16 | {%- endif -%} 17 | 18 | {% set fields = [] %} 19 | 20 | {%- for field in field_list -%} 21 | 22 | {% do fields.append("coalesce(" ~ field ~ ", 0)") %} 23 | 24 | {%- endfor -%} 25 | 26 | {{ fields|join(' +\n ') }} 27 | 28 | {%- endmacro -%} 29 | -------------------------------------------------------------------------------- /macros/sql/safe_divide.sql: -------------------------------------------------------------------------------- 1 | {% macro safe_divide(numerator, denominator) -%} 2 | {{ return(adapter.dispatch('safe_divide', 'dbt_utils')(numerator, denominator)) }} 3 | {%- endmacro %} 4 | 5 | {% macro default__safe_divide(numerator, denominator) %} 6 | ( {{ numerator }} ) / nullif( ( {{ denominator }} ), 0) 7 | {% endmacro %} -------------------------------------------------------------------------------- /macros/sql/safe_subtract.sql: -------------------------------------------------------------------------------- 1 | {%- macro safe_subtract(field_list) -%} 2 | {{ return(adapter.dispatch('safe_subtract', 'dbt_utils')(field_list)) }} 3 | {% endmacro %} 4 | 5 | {%- macro default__safe_subtract(field_list) -%} 6 | 7 | {%- if field_list is not iterable or field_list is string or field_list is mapping -%} 8 | 9 | {%- set error_message = ' 10 | Warning: the `safe_subtract` macro takes a single list argument instead of \ 11 | string arguments. The {}.{} model triggered this warning. \ 12 | '.format(model.package_name, model.name) -%} 13 | 14 | {%- do exceptions.raise_compiler_error(error_message) -%} 15 | 16 | {%- endif -%} 17 | 18 | {% set fields = [] %} 19 | 20 | {%- for field in field_list -%} 21 | 22 | {% do fields.append("coalesce(" ~ field ~ ", 0)") %} 23 | 24 | {%- endfor -%} 25 | 26 | {{ fields|join(' -\n ') }} 27 | 28 | {%- endmacro -%} 29 | -------------------------------------------------------------------------------- /macros/sql/star.sql: -------------------------------------------------------------------------------- 1 | {% macro star(from, relation_alias=False, except=[], prefix='', suffix='', quote_identifiers=True) -%} 2 | {{ return(adapter.dispatch('star', 'dbt_utils')(from, relation_alias, except, prefix, suffix, quote_identifiers)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__star(from, relation_alias=False, except=[], prefix='', suffix='', quote_identifiers=True) -%} 6 | {%- do dbt_utils._is_relation(from, 'star') -%} 7 | {%- do dbt_utils._is_ephemeral(from, 'star') -%} 8 | 9 | {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} 10 | {%- if not execute -%} 11 | {% do return('*') %} 12 | {%- endif -%} 13 | 14 | {% set cols = dbt_utils.get_filtered_columns_in_relation(from, except) %} 15 | 16 | {%- if cols|length <= 0 -%} 17 | {% if flags.WHICH == 'compile' %} 18 | {% set response %} 19 | * 20 | /* No columns were returned. Maybe the relation doesn't exist yet 21 | or all columns were excluded. This star is only output during 22 | dbt compile, and exists to keep SQLFluff happy. */ 23 | {% endset %} 24 | {% do return(response) %} 25 | {% else %} 26 | {% do return("/* no columns returned from star() macro */") %} 27 | {% endif %} 28 | {%- else -%} 29 | {%- for col in cols %} 30 | {%- if relation_alias %}{{ relation_alias }}.{% else %}{%- endif -%} 31 | {%- if quote_identifiers -%} 32 | {{ adapter.quote(col)|trim }} {%- if prefix!='' or suffix!='' %} as {{ adapter.quote(prefix ~ col ~ suffix)|trim }} {%- endif -%} 33 | {%- else -%} 34 | {{ col|trim }} {%- if prefix!='' or suffix!='' %} as {{ (prefix ~ col ~ suffix)|trim }} {%- endif -%} 35 | {% endif %} 36 | {%- if not loop.last %},{{ '\n ' }}{%- endif -%} 37 | {%- endfor -%} 38 | {% endif %} 39 | {%- endmacro %} 40 | 41 | -------------------------------------------------------------------------------- /macros/sql/surrogate_key.sql: -------------------------------------------------------------------------------- 1 | {%- macro surrogate_key(field_list) -%} 2 | {% set frustrating_jinja_feature = varargs %} 3 | {{ return(adapter.dispatch('surrogate_key', 'dbt_utils')(field_list, *varargs)) }} 4 | {% endmacro %} 5 | 6 | {%- macro default__surrogate_key(field_list) -%} 7 | 8 | {%- set error_message = ' 9 | Warning: `dbt_utils.surrogate_key` has been replaced by \ 10 | `dbt_utils.generate_surrogate_key`. The new macro treats null values \ 11 | differently to empty strings. To restore the behaviour of the original \ 12 | macro, add a global variable in dbt_project.yml called \ 13 | `surrogate_key_treat_nulls_as_empty_strings` to your \ 14 | dbt_project.yml file with a value of True. \ 15 | The {}.{} model triggered this warning. \ 16 | '.format(model.package_name, model.name) -%} 17 | 18 | {%- do exceptions.raise_compiler_error(error_message) -%} 19 | 20 | {%- endmacro -%} 21 | -------------------------------------------------------------------------------- /macros/sql/union.sql: -------------------------------------------------------------------------------- 1 | {%- macro union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation', where=none) -%} 2 | {{ return(adapter.dispatch('union_relations', 'dbt_utils')(relations, column_override, include, exclude, source_column_name, where)) }} 3 | {% endmacro %} 4 | 5 | {%- macro default__union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation', where=none) -%} 6 | 7 | {%- if exclude and include -%} 8 | {{ exceptions.raise_compiler_error("Both an exclude and include list were provided to the `union` macro. Only one is allowed") }} 9 | {%- endif -%} 10 | 11 | {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#} 12 | {%- if not execute %} 13 | {{ return('') }} 14 | {% endif -%} 15 | 16 | {%- set column_override = column_override if column_override is not none else {} -%} 17 | 18 | {%- set relation_columns = {} -%} 19 | {%- set column_superset = {} -%} 20 | {%- set all_excludes = [] -%} 21 | {%- set all_includes = [] -%} 22 | 23 | {%- if exclude -%} 24 | {%- for exc in exclude -%} 25 | {%- do all_excludes.append(exc | lower) -%} 26 | {%- endfor -%} 27 | {%- endif -%} 28 | 29 | {%- if include -%} 30 | {%- for inc in include -%} 31 | {%- do all_includes.append(inc | lower) -%} 32 | {%- endfor -%} 33 | {%- endif -%} 34 | 35 | {%- for relation in relations -%} 36 | 37 | {%- do relation_columns.update({relation: []}) -%} 38 | 39 | {%- do dbt_utils._is_relation(relation, 'union_relations') -%} 40 | {%- do dbt_utils._is_ephemeral(relation, 'union_relations') -%} 41 | {%- set cols = adapter.get_columns_in_relation(relation) -%} 42 | {%- for col in cols -%} 43 | 44 | {#- If an exclude list was provided and the column is in the list, do nothing -#} 45 | {%- if exclude and col.column | lower in all_excludes -%} 46 | 47 | {#- If an include list was provided and the column is not in the list, do nothing -#} 48 | {%- elif include and col.column | lower not in all_includes -%} 49 | 50 | {#- Otherwise add the column to the column superset -#} 51 | {%- else -%} 52 | 53 | {#- update the list of columns in this relation -#} 54 | {%- do relation_columns[relation].append(col.column) -%} 55 | 56 | {%- if col.column in column_superset -%} 57 | 58 | {%- set stored = column_superset[col.column] -%} 59 | {%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%} 60 | 61 | {%- do column_superset.update({col.column: col}) -%} 62 | 63 | {%- endif %} 64 | 65 | {%- else -%} 66 | 67 | {%- do column_superset.update({col.column: col}) -%} 68 | 69 | {%- endif -%} 70 | 71 | {%- endif -%} 72 | 73 | {%- endfor -%} 74 | {%- endfor -%} 75 | 76 | {%- set ordered_column_names = column_superset.keys() -%} 77 | {%- set dbt_command = flags.WHICH -%} 78 | 79 | 80 | {% if dbt_command in ['run', 'build'] %} 81 | {% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %} 82 | {%- set relations_string -%} 83 | {%- for relation in relations -%} 84 | {{ relation.name }} 85 | {%- if not loop.last %}, {% endif -%} 86 | {%- endfor -%} 87 | {%- endset -%} 88 | 89 | {%- set error_message -%} 90 | There were no columns found to union for relations {{ relations_string }} 91 | {%- endset -%} 92 | 93 | {{ exceptions.raise_compiler_error(error_message) }} 94 | {%- endif -%} 95 | {%- endif -%} 96 | 97 | {%- for relation in relations %} 98 | 99 | ( 100 | select 101 | 102 | {%- if source_column_name is not none %} 103 | cast({{ dbt.string_literal(relation) }} as {{ dbt.type_string() }}) as {{ source_column_name }}, 104 | {%- endif %} 105 | 106 | {% for col_name in ordered_column_names -%} 107 | 108 | {%- set col = column_superset[col_name] %} 109 | {%- set col_type = column_override.get(col.column, col.data_type) %} 110 | {%- set col_name = adapter.quote(col_name) if col_name in relation_columns[relation] else 'null' %} 111 | cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} {% if not loop.last %},{% endif -%} 112 | 113 | {%- endfor %} 114 | 115 | from {{ relation }} 116 | 117 | {% if where -%} 118 | where {{ where }} 119 | {%- endif %} 120 | ) 121 | 122 | {% if not loop.last -%} 123 | union all 124 | {% endif -%} 125 | 126 | {%- endfor -%} 127 | 128 | {%- endmacro -%} 129 | -------------------------------------------------------------------------------- /macros/sql/unpivot.sql: -------------------------------------------------------------------------------- 1 | {# 2 | Pivot values from columns to rows. Similar to pandas DataFrame melt() function. 3 | 4 | Example Usage: {{ unpivot(relation=ref('users'), cast_to='integer', exclude=['id','created_at']) }} 5 | 6 | Arguments: 7 | relation: Relation object, required. 8 | cast_to: The datatype to cast all unpivoted columns to. Default is varchar. 9 | exclude: A list of columns to keep but exclude from the unpivot operation. Default is none. 10 | remove: A list of columns to remove from the resulting table. Default is none. 11 | field_name: Destination table column name for the source table column names. 12 | value_name: Destination table column name for the pivoted values 13 | #} 14 | 15 | {% macro unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', quote_identifiers=False) -%} 16 | {{ return(adapter.dispatch('unpivot', 'dbt_utils')(relation, cast_to, exclude, remove, field_name, value_name, quote_identifiers)) }} 17 | {% endmacro %} 18 | 19 | {% macro default__unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', quote_identifiers=False) -%} 20 | 21 | {% if not relation %} 22 | {{ exceptions.raise_compiler_error("Error: argument `relation` is required for `unpivot` macro.") }} 23 | {% endif %} 24 | 25 | {%- set exclude = exclude if exclude is not none else [] %} 26 | {%- set remove = remove if remove is not none else [] %} 27 | 28 | {%- set include_cols = [] %} 29 | 30 | {%- set table_columns = {} %} 31 | 32 | {%- do table_columns.update({relation: []}) %} 33 | 34 | {%- do dbt_utils._is_relation(relation, 'unpivot') -%} 35 | {%- do dbt_utils._is_ephemeral(relation, 'unpivot') -%} 36 | {%- set cols = adapter.get_columns_in_relation(relation) %} 37 | 38 | {%- for col in cols -%} 39 | {%- if col.column.lower() not in remove|map('lower') and col.column.lower() not in exclude|map('lower') -%} 40 | {% do include_cols.append(col) %} 41 | {%- endif %} 42 | {%- endfor %} 43 | 44 | 45 | {%- for col in include_cols -%} 46 | {%- set current_col_name = adapter.quote(col.column) if quote_identifiers else col.column -%} 47 | select 48 | {%- for exclude_col in exclude %} 49 | {{ adapter.quote(exclude_col) if quote_identifiers else exclude_col }}, 50 | {%- endfor %} 51 | 52 | cast('{{ col.column }}' as {{ dbt.type_string() }}) as {{ adapter.quote(field_name) if quote_identifiers else field_name }}, 53 | cast( {% if col.data_type == 'boolean' %} 54 | {{ dbt.cast_bool_to_text(current_col_name) }} 55 | {% else %} 56 | {{ current_col_name }} 57 | {% endif %} 58 | as {{ cast_to }}) as {{ adapter.quote(value_name) if quote_identifiers else value_name }} 59 | 60 | from {{ relation }} 61 | 62 | {% if not loop.last -%} 63 | union all 64 | {% endif -%} 65 | {%- endfor -%} 66 | 67 | {%- endmacro %} 68 | -------------------------------------------------------------------------------- /macros/sql/width_bucket.sql: -------------------------------------------------------------------------------- 1 | {% macro width_bucket(expr, min_value, max_value, num_buckets) %} 2 | {{ return(adapter.dispatch('width_bucket', 'dbt_utils') (expr, min_value, max_value, num_buckets)) }} 3 | {% endmacro %} 4 | 5 | 6 | {% macro default__width_bucket(expr, min_value, max_value, num_buckets) -%} 7 | 8 | {% set bin_size -%} 9 | (( {{ max_value }} - {{ min_value }} ) / {{ num_buckets }} ) 10 | {%- endset %} 11 | ( 12 | -- to break ties when the amount is eaxtly at the bucket egde 13 | case 14 | when 15 | mod( 16 | {{ dbt.safe_cast(expr, dbt.type_numeric() ) }}, 17 | {{ dbt.safe_cast(bin_size, dbt.type_numeric() ) }} 18 | ) = 0 19 | then 1 20 | else 0 21 | end 22 | ) + 23 | -- Anything over max_value goes the N+1 bucket 24 | least( 25 | ceil( 26 | ({{ expr }} - {{ min_value }})/{{ bin_size }} 27 | ), 28 | {{ num_buckets }} + 1 29 | ) 30 | {%- endmacro %} 31 | 32 | {% macro snowflake__width_bucket(expr, min_value, max_value, num_buckets) %} 33 | width_bucket({{ expr }}, {{ min_value }}, {{ max_value }}, {{ num_buckets }} ) 34 | {% endmacro %} 35 | -------------------------------------------------------------------------------- /macros/web/get_url_host.sql: -------------------------------------------------------------------------------- 1 | {% macro get_url_host(field) -%} 2 | {{ return(adapter.dispatch('get_url_host', 'dbt_utils')(field)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_url_host(field) -%} 6 | 7 | {%- set parsed = 8 | dbt.split_part( 9 | dbt.split_part( 10 | dbt.replace( 11 | dbt.replace( 12 | dbt.replace(field, "'android-app://'", "''" 13 | ), "'http://'", "''" 14 | ), "'https://'", "''" 15 | ), "'/'", 1 16 | ), "'?'", 1 17 | ) 18 | 19 | -%} 20 | 21 | 22 | {{ dbt.safe_cast( 23 | parsed, 24 | dbt.type_string() 25 | )}} 26 | 27 | {%- endmacro %} 28 | -------------------------------------------------------------------------------- /macros/web/get_url_parameter.sql: -------------------------------------------------------------------------------- 1 | {% macro get_url_parameter(field, url_parameter) -%} 2 | {{ return(adapter.dispatch('get_url_parameter', 'dbt_utils')(field, url_parameter)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_url_parameter(field, url_parameter) -%} 6 | 7 | {%- set formatted_url_parameter = "'" + url_parameter + "='" -%} 8 | 9 | {%- set split = dbt.split_part(dbt.split_part(field, formatted_url_parameter, 2), "'&'", 1) -%} 10 | 11 | nullif({{ split }},'') 12 | 13 | {%- endmacro %} 14 | -------------------------------------------------------------------------------- /macros/web/get_url_path.sql: -------------------------------------------------------------------------------- 1 | {% macro get_url_path(field) -%} 2 | {{ return(adapter.dispatch('get_url_path', 'dbt_utils')(field)) }} 3 | {% endmacro %} 4 | 5 | {% macro default__get_url_path(field) -%} 6 | 7 | {%- set stripped_url = 8 | dbt.replace( 9 | dbt.replace(field, "'http://'", "''"), "'https://'", "''") 10 | -%} 11 | 12 | {%- set first_slash_pos -%} 13 | coalesce( 14 | nullif({{ dbt.position("'/'", stripped_url) }}, 0), 15 | {{ dbt.position("'?'", stripped_url) }} - 1 16 | ) 17 | {%- endset -%} 18 | 19 | {%- set parsed_path = 20 | dbt.split_part( 21 | dbt.right( 22 | stripped_url, 23 | dbt.length(stripped_url) ~ "-" ~ first_slash_pos 24 | ), 25 | "'?'", 1 26 | ) 27 | -%} 28 | 29 | {{ dbt.safe_cast( 30 | parsed_path, 31 | dbt.type_string() 32 | )}} 33 | 34 | {%- endmacro %} 35 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | filterwarnings = 3 | ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning 4 | ignore:unclosed file .*:ResourceWarning 5 | env_files = 6 | test.env 7 | testpaths = 8 | tests/functional -------------------------------------------------------------------------------- /run_functional_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python3 -m pytest tests/functional -n4 --profile $1 4 | -------------------------------------------------------------------------------- /run_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Show location of local install of dbt 4 | echo $(which dbt) 5 | 6 | # Show version and installed adapters 7 | dbt --version 8 | 9 | # Set the profile 10 | cd integration_tests 11 | export DBT_PROFILES_DIR=. 12 | 13 | # Show the location of the profiles directory and test the connection 14 | dbt debug --target $1 15 | 16 | dbt deps --target $1 || exit 1 17 | dbt build --target $1 --full-refresh || exit 1 18 | -------------------------------------------------------------------------------- /supported_adapters.env: -------------------------------------------------------------------------------- 1 | SUPPORTED_ADAPTERS=postgres,snowflake,redshift,bigquery 2 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | skipsdist = True 3 | envlist = lint_all, testenv 4 | 5 | [testenv] 6 | passenv = 7 | # postgres env vars 8 | POSTGRES_HOST 9 | POSTGRES_USER 10 | DBT_ENV_SECRET_POSTGRES_PASS 11 | POSTGRES_PORT 12 | POSTGRES_DATABASE 13 | POSTGRES_SCHEMA 14 | # snowflake env vars 15 | SNOWFLAKE_ACCOUNT 16 | SNOWFLAKE_USER 17 | DBT_ENV_SECRET_SNOWFLAKE_PASS 18 | SNOWFLAKE_ROLE 19 | SNOWFLAKE_DATABASE 20 | SNOWFLAKE_WAREHOUSE 21 | SNOWFLAKE_SCHEMA 22 | # redshift 23 | REDSHIFT_HOST 24 | REDSHIFT_USER 25 | DBT_ENV_SECRET_REDSHIFT_PASS 26 | REDSHIFT_DATABASE 27 | REDSHIFT_SCHEMA 28 | REDSHIFT_PORT 29 | # bigquery 30 | BIGQUERY_PROJECT 31 | BIGQUERY_KEYFILE_JSON 32 | BIGQUERY_SCHEMA 33 | 34 | # Snowflake integration tests for centralized dbt testing 35 | # run dbt commands directly, assumes dbt is already installed in environment 36 | [testenv:dbt_integration_snowflake] 37 | changedir = integration_tests 38 | allowlist_externals = 39 | dbt 40 | skip_install = true 41 | commands = 42 | dbt --version 43 | dbt debug --target snowflake 44 | dbt deps --target snowflake 45 | dbt build --target snowflake --full-refresh 46 | 47 | 48 | # Postgres integration tests for centralized dbt testing 49 | # run dbt commands directly, assumes dbt is already installed in environment 50 | [testenv:dbt_integration_postgres] 51 | changedir = integration_tests 52 | allowlist_externals = 53 | dbt 54 | skip_install = true 55 | commands = 56 | dbt --version 57 | dbt debug --target postgres 58 | dbt deps --target postgres 59 | dbt build --target postgres --full-refresh 60 | 61 | # BigQuery integration tests for centralized dbt testing 62 | # run dbt commands directly, assumes dbt is already installed in environment 63 | [testenv:dbt_integration_bigquery] 64 | changedir = integration_tests 65 | allowlist_externals = 66 | dbt 67 | skip_install = true 68 | commands = 69 | dbt --version 70 | dbt debug --target bigquery 71 | dbt deps --target bigquery 72 | dbt build --target bigquery --full-refresh 73 | 74 | # redshift integration tests for centralized dbt testing 75 | # run dbt commands directly, assumes dbt is already installed in environment 76 | [testenv:dbt_integration_redshift] 77 | changedir = integration_tests 78 | allowlist_externals = 79 | dbt 80 | skip_install = true 81 | commands = 82 | dbt --version 83 | dbt debug --target redshift 84 | dbt deps --target redshift 85 | dbt build --target redshift --full-refresh 86 | --------------------------------------------------------------------------------