├── .changes ├── unreleased │ └── .gitkeep ├── 5.3.2.md ├── 4.1.1.md ├── 4.5.2.md ├── 1.2.1.md ├── 4.2.1.md ├── 4.3.0.md ├── 4.4.0.md ├── 4.5.0.md ├── 5.1.2.md ├── 5.4.2.md ├── 5.0.2.md ├── 1.5.0.md ├── 2.0.1.md ├── 5.0.4.md ├── 4.0.0.md ├── 4.3.1.md ├── 4.5.3.md ├── 5.2.1.md ├── 5.2.2.md ├── 5.4.3.md ├── 4.0.1.md ├── 5.3.3.md ├── 1.1.0.md ├── 1.6.1.md ├── 2.0.2.md ├── 1.4.3.md ├── 4.0.2.md ├── 1.6.0.md ├── 4.4.1.md ├── 4.5.1.md ├── 1.2.2.md ├── 5.4.4.md ├── 1.4.1.md ├── 1.4.2.md ├── 1.3.0.md ├── 5.0.1.md ├── 3.0.1.md ├── 1.3.1.md ├── 5.4.5.md ├── 4.2.0.md ├── 4.1.0.md ├── 5.3.1.md ├── 5.4.6.md ├── 5.4.1.md ├── 5.1.1.md ├── 5.3.0.md ├── 5.2.0.md ├── 5.4.0.md ├── 5.4.7.md ├── 5.5.0.md ├── header.tpl.md ├── 1.0.0.md ├── 5.0.3.md ├── 1.4.0.md ├── 1.2.3.md ├── 1.2.0.md ├── 2.0.0.md ├── 5.1.0.md ├── 4.6.0.md ├── 5.0.0.md └── 3.0.0.md ├── integration_test_project ├── packages.yml ├── models │ ├── incremental.sql │ ├── monthly_spend.sql │ ├── monthly_spend.yml │ └── sources.yml ├── tests │ ├── assert_unique_daily_rates.sql │ └── assert_no_queries_are_dropped.sql ├── seeds │ └── monthly_spend_fixture.csv ├── dbt_project.yml ├── profiles.yml └── macros │ └── advanced_equality.sql ├── .gitignore ├── documentation └── sample_queries.md ├── macros ├── query_comment.sql ├── account_name.sql ├── create_merge_objects_udf.sql └── query_tags.sql ├── packages.yml ├── models ├── staging │ ├── stg_stage_storage_usage_history.sql │ ├── stg_metering_daily_history.sql │ ├── stg_database_storage_usage_history.sql │ ├── stg_metering_history.sql │ ├── stg_storage_usage.sql │ ├── stg_stage_storage_usage_history.yml │ ├── stg_usage_in_currency_daily.sql │ ├── stg_remaining_balance_daily.sql │ ├── stg_metering_daily_history.yml │ ├── stg_rate_sheet_daily.sql │ ├── stg_access_history.sql │ ├── stg_serverless_task_history.sql │ ├── stg_warehouse_metering_history.sql │ ├── stg_database_storage_usage_history.yml │ ├── sources.yml │ ├── stg_access_history.yml │ ├── stg_metering_history.yml │ ├── stg_serverless_task_history.yml │ ├── stg_warehouse_metering_history.yml │ ├── stg_storage_usage.yml │ ├── stg_rate_sheet_daily.yml │ ├── stg_remaining_balance_daily.yml │ ├── stg_query_history.sql │ └── stg_query_history.yml ├── query_base_table_access.sql ├── query_direct_table_access.sql ├── remaining_balance_daily_without_contract_view.sql ├── daily_spend.sql ├── warehouse_credits_map.yml ├── warehouses_type2_dimension.yml ├── warehouse_credits_map.sql ├── query_base_table_access.yml ├── query_direct_table_access.yml ├── warehouses_type2_dimension.sql ├── daily_rates.yml ├── query_base_object_access.yml ├── query_direct_object_access.yml ├── hourly_spend.yml ├── query_base_object_access.sql ├── daily_spend.yml ├── cost_per_query.yml ├── query_direct_object_access.sql ├── dbt_queries.yml ├── dbt_queries.sql ├── daily_rates.sql ├── query_history_enriched.sql ├── cost_per_query.sql ├── query_history_enriched.yml └── hourly_spend.sql ├── .editorconfig ├── package-lock.yml ├── dbt_project.yml ├── .pre-commit-config.yaml ├── LICENSE ├── .github └── workflows │ ├── main_test_package.yml │ ├── ci_test_package.yml │ └── publish_docs.yml ├── README.md ├── tox.ini ├── .changie.yaml └── CHANGELOG.md /.changes/unreleased/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /integration_test_project/packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - local: ../ 3 | -------------------------------------------------------------------------------- /integration_test_project/models/incremental.sql: -------------------------------------------------------------------------------- 1 | {{ config(materialized='incremental') }} 2 | 3 | select 1 as a 4 | -------------------------------------------------------------------------------- /.changes/5.3.2.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.3.2 - January 31, 2025 2 | 3 | Several user-contributed fixes and improvements. 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | dbt_packages/ 3 | logs/ 4 | logfile 5 | .DS_Store 6 | env.sh 7 | uv.lock 8 | pyproject.toml 9 | .python-version -------------------------------------------------------------------------------- /documentation/sample_queries.md: -------------------------------------------------------------------------------- 1 | Example queries have moved to our new documentation site: https://select.dev/docs/dbt-snowflake-monitoring/example-usage 2 | -------------------------------------------------------------------------------- /macros/query_comment.sql: -------------------------------------------------------------------------------- 1 | {% macro get_query_comment(node) %} 2 | {{ return(dbt_snowflake_query_tags.get_query_comment(node)) }} 3 | {% endmacro %} 4 | -------------------------------------------------------------------------------- /integration_test_project/tests/assert_unique_daily_rates.sql: -------------------------------------------------------------------------------- 1 | select 2 | date, 3 | usage_type 4 | from {{ ref('daily_rates') }} 5 | group by 1,2 6 | having count(*) > 1 -------------------------------------------------------------------------------- /packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: [">=0.8.0", "<2.0.0"] 4 | - package: get-select/dbt_snowflake_query_tags 5 | version: [">=2.0.0", "<3.0.0"] 6 | -------------------------------------------------------------------------------- /.changes/4.1.1.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 4.1.1 - May 12, 2023 2 | 3 | ### Features 4 | 5 | - Support dbt 1.5.0 ([#106](https://github.com/get-select/dbt-snowflake-monitoring/pull/106)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/4.5.2.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 4.5.2 - October 30, 2023 2 | 3 | ### Fixes 4 | 5 | - Fix cluster number ([#134](https://github.com/get-select/dbt-snowflake-monitoring/pull/134)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/1.2.1.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 1.2.1 - December 04, 2022 2 | 3 | ### Fixes 4 | 5 | - Fix account_locator var ([#33](https://github.com/get-select/dbt-snowflake-monitoring/pull/33)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/4.2.1.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 4.2.1 - June 03, 2023 2 | 3 | ### Fixes 4 | 5 | - Daily rates improvements ([#114](https://github.com/get-select/dbt-snowflake-monitoring/pull/114)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/4.3.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 4.3.0 - June 21, 2023 2 | 3 | ### Features 4 | 5 | - Add hourly_spend model ([#116](https://github.com/get-select/dbt-snowflake-monitoring/pull/116)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/4.4.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 4.4.0 - July 26, 2023 2 | 3 | ### Features 4 | 5 | - Add Data Transfer costs ([#123](https://github.com/get-select/dbt-snowflake-monitoring/pull/123)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/4.5.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 4.5.0 - August 09, 2023 2 | 3 | ### Features 4 | 5 | - Add Snowpipe Streaming ([#125](https://github.com/get-select/dbt-snowflake-monitoring/pull/125)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/5.1.2.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.1.2 - May 23, 2024 2 | 3 | ### Fixes 4 | 5 | - Fix incremental filtering ([#254](https://github.com/get-select/dbt-snowflake-monitoring/pull/254)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/5.4.2.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.4.2 - February 27, 2025 2 | 3 | ### Fixes 4 | 5 | - Fix AI services spend ([#176](https://github.com/get-select/dbt-snowflake-monitoring/pull/176)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /macros/account_name.sql: -------------------------------------------------------------------------------- 1 | {% macro account_name() %} 2 | {%- if var('account_name', none) -%} 3 | '{{ var('account_name') }}' 4 | {%- else -%} 5 | current_account_name() 6 | {%- endif -%} 7 | {%- endmacro %} 8 | -------------------------------------------------------------------------------- /.changes/5.0.2.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.0.2 - January 15, 2024 2 | 3 | ### Fixes 4 | 5 | - Fix handling of service_type ([#143](https://github.com/get-select/dbt-snowflake-monitoring/pull/143)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/1.5.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 1.5.0 - January 25, 2023 2 | 3 | ### Fixes 4 | 5 | - Fix query_text_no_comments regex ([#72](https://github.com/get-select/dbt-snowflake-monitoring/pull/72)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/2.0.1.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 2.0.1 - February 17, 2023 2 | 3 | ### Features 4 | 5 | - Add spend_net_cloud_services ([#80](https://github.com/get-select/dbt-snowflake-monitoring/pull/80)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/5.0.4.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.0.4 - March 15, 2024 2 | 3 | ### Fixes 4 | 5 | - Support different order json comment ([#148](https://github.com/get-select/dbt-snowflake-monitoring/pull/148)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/4.0.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 4.0.0 - March 21, 2023 2 | 3 | ### Breaking Changes 4 | 5 | - Rename access_history models ([#92](https://github.com/get-select/dbt-snowflake-monitoring/pull/92)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/4.3.1.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 4.3.1 - June 23, 2023 2 | 3 | ### Features 4 | 5 | - Set range for dbt_snowflake_query_tags ([#121](https://github.com/get-select/dbt-snowflake-monitoring/pull/121)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/4.5.3.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 4.5.3 - November 06, 2023 2 | 3 | ### Features 4 | 5 | - Remove regex UDF to avoid timeout ([#135](https://github.com/get-select/dbt-snowflake-monitoring/pull/135)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/5.2.1.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.2.1 - June 09, 2024 2 | 3 | ### Features 4 | 5 | - Remove warehouse_cluster_status model ([#158](https://github.com/get-select/dbt-snowflake-monitoring/pull/158)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/5.2.2.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.2.2 - June 09, 2024 2 | 3 | ### Features 4 | 5 | - Pull out stg_warehouse_events_history ([#159](https://github.com/get-select/dbt-snowflake-monitoring/pull/159)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/5.4.3.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.4.3 - March 24, 2025 2 | 3 | ### Features 4 | 5 | - Update locator to use account name ([#177](https://github.com/get-select/dbt-snowflake-monitoring/pull/177)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/4.0.1.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 4.0.1 - April 07, 2023 2 | 3 | ### Fixes 4 | 5 | - Don't let daily_spend be held up by stale rates ([#95](https://github.com/get-select/dbt-snowflake-monitoring/pull/95)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/5.3.3.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.3.3 - January 31, 2025 2 | 3 | ### Fixes 4 | 5 | - Fix column dependencies in stg_query_history ([#171](https://github.com/get-select/dbt-snowflake-monitoring/pull/171)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/1.1.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 1.1.0 - October 28, 2022 2 | 3 | ### Features 4 | 5 | - Add daily_spend model which aligns with monthly bill ([#13](https://github.com/get-select/dbt-snowflake-monitoring/pull/13)) 6 | 7 | -------------------------------------------------------------------------------- /.changes/1.6.1.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 1.6.1 - February 03, 2023 2 | 3 | ### Fixes 4 | 5 | - Fixes is_incremental in get_query_comment macro ([#77](https://github.com/get-select/dbt-snowflake-monitoring/pull/77)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/2.0.2.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 2.0.2 - February 21, 2023 2 | 3 | ### Fixes 4 | 5 | - Use the model database and schema for UDF creation ([#83](https://github.com/get-select/dbt-snowflake-monitoring/pull/83)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/1.4.3.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 1.4.3 - January 20, 2023 2 | 3 | ### Fixes 4 | 5 | - Fix error on initial compile with no models created yet ([#70](https://github.com/get-select/dbt-snowflake-monitoring/pull/70)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/4.0.2.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 4.0.2 - April 08, 2023 2 | 3 | ### Fixes 4 | 5 | - Prevent daily_spend from being held up by missing data ([#97](https://github.com/get-select/dbt-snowflake-monitoring/pull/97)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /models/staging/stg_stage_storage_usage_history.sql: -------------------------------------------------------------------------------- 1 | {{ config(materialized='view') }} 2 | 3 | select 4 | usage_date as date, 5 | average_stage_bytes 6 | from {{ source('snowflake_account_usage', 'stage_storage_usage_history') }} 7 | -------------------------------------------------------------------------------- /.changes/1.6.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 1.6.0 - February 01, 2023 2 | 3 | ### Features 4 | 5 | - Add query_object_access and query_table_access models ([#73](https://github.com/get-select/dbt-snowflake-monitoring/pull/73)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/4.4.1.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 4.4.1 - July 28, 2023 2 | 3 | ### Features 4 | 5 | - Remove unneeded data_transfer_history source and stg model ([#124](https://github.com/get-select/dbt-snowflake-monitoring/pull/124)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/4.5.1.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 4.5.1 - August 24, 2023 2 | 3 | ### Fixes 4 | 5 | - Sort remaining balance daily with null contract numbers last ([#128](https://github.com/get-select/dbt-snowflake-monitoring/pull/128)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /integration_test_project/models/monthly_spend.sql: -------------------------------------------------------------------------------- 1 | select 2 | date_trunc(month, date) as month, 3 | service, 4 | sum(spend) as spend 5 | from {{ ref('daily_spend') }} 6 | where date_trunc(month, date) = '2022-04-01' group by 1, 2 7 | -------------------------------------------------------------------------------- /.changes/1.2.2.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 1.2.2 - December 05, 2022 2 | 3 | ### Fixes 4 | 5 | - Show only complete days in daily_spend and materialize as table ([#34](https://github.com/get-select/dbt-snowflake-monitoring/pull/34)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/5.4.4.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.4.4 - May 8, 2025 2 | 3 | ### Fixes 4 | 5 | - Fix nested json access where the casing of the key has inadvertently changed ([#174](https://github.com/get-select/dbt-snowflake-monitoring/pull/174)) 6 | -------------------------------------------------------------------------------- /integration_test_project/models/monthly_spend.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: monthly_spend 5 | tests: 6 | - advanced_equality: 7 | compare_model: ref('monthly_spend_fixture') 8 | round_columns: ['spend'] 9 | -------------------------------------------------------------------------------- /.changes/1.4.1.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 1.4.1 - January 06, 2023 2 | 3 | ### Fixes 4 | 5 | - Increase incremental lookback window to account for changing cost data ([#64](https://github.com/get-select/dbt-snowflake-monitoring/pull/64)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/1.4.2.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 1.4.2 - January 12, 2023 2 | 3 | ### Fixes 4 | 5 | - Inner join daily_rates to avoid $0 entries in daily_spend for incomplete days ([#66](https://github.com/get-select/dbt-snowflake-monitoring/pull/66)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/1.3.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 1.3.0 - December 21, 2022 2 | 3 | ### Features 4 | 5 | - Add dbt_queries model to easily understand dbt model costs with links to dbt Cloud ([#47](https://github.com/get-select/dbt-snowflake-monitoring/pull/47)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.changes/5.0.1.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.0.1 - January 15, 2024 2 | 3 | ### Fixes 4 | 5 | - Switch stg_metering_history to a table incrementalization to handle natural dupes ([#142](https://github.com/get-select/dbt-snowflake-monitoring/pull/142)) 6 | 7 | 8 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 4 6 | end_of_line = lf 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | 11 | [*.yml] 12 | indent_size = 2 13 | trim_trailing_whitespace = false 14 | -------------------------------------------------------------------------------- /package-lock.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - name: dbt_utils 3 | package: dbt-labs/dbt_utils 4 | version: 1.3.0 5 | - name: dbt_snowflake_query_tags 6 | package: get-select/dbt_snowflake_query_tags 7 | version: 2.5.1 8 | sha1_hash: 8f7dc6c807de3c3e5483e9f286aae43e4b7d7bc1 9 | -------------------------------------------------------------------------------- /.changes/3.0.1.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 3.0.1 - March 01, 2023 2 | 3 | ### Fixes 4 | 5 | - Fix dbt_cloud_url var ([#90](https://github.com/get-select/dbt-snowflake-monitoring/pull/90)) 6 | 7 | ### Contributors 8 | - [@vinooganesh](https://github.com/vinooganesh) (Fixes) 9 | 10 | -------------------------------------------------------------------------------- /models/staging/stg_metering_daily_history.sql: -------------------------------------------------------------------------------- 1 | {{ config(materialized='view') }} 2 | 3 | select 4 | usage_date as date, 5 | service_type, 6 | credits_used_cloud_services, 7 | credits_adjustment_cloud_services 8 | from {{ source('snowflake_account_usage', 'metering_daily_history') }} 9 | -------------------------------------------------------------------------------- /.changes/1.3.1.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 1.3.1 - December 31, 2022 2 | 3 | ### Features 4 | 5 | - Support dbt_utils 1.0.0 ([#48](https://github.com/get-select/dbt-snowflake-monitoring/pull/48)) 6 | 7 | ### Contributors 8 | - [@GtheSheep](https://github.com/GtheSheep) (Features) 9 | 10 | -------------------------------------------------------------------------------- /.changes/5.4.5.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.4.5 - May 09, 2025 2 | 3 | ### Features 4 | 5 | - Deduplicate query history ([#182](https://github.com/get-select/dbt-snowflake-monitoring/pull/182)) 6 | 7 | ### Contributors 8 | - [@sergeykhar](https://github.com/sergeykhar) (Features) 9 | 10 | -------------------------------------------------------------------------------- /.changes/4.2.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 4.2.0 - May 27, 2023 2 | 3 | ### Features 4 | 5 | - Add node_meta to dbt_query model ([#109](https://github.com/get-select/dbt-snowflake-monitoring/pull/109)) 6 | 7 | ### Contributors 8 | - [@pratik60](https://github.com/pratik60) (Features) 9 | 10 | -------------------------------------------------------------------------------- /.changes/4.1.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 4.1.0 - May 05, 2023 2 | 3 | ### Features 4 | 5 | - Add currency to all models with spend data ([#103](https://github.com/get-select/dbt-snowflake-monitoring/pull/103)) 6 | 7 | ### Contributors 8 | - [@calleo](https://github.com/calleo) (Features) 9 | 10 | -------------------------------------------------------------------------------- /.changes/5.3.1.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.3.1 - August 23, 2024 2 | 3 | ### Features 4 | 5 | - Add lookback variable to stg_query_history ([#163](https://github.com/get-select/dbt-snowflake-monitoring/pull/163)) 6 | 7 | ### Contributors 8 | - [@smitsrr](https://github.com/smitsrr) (Features) 9 | 10 | -------------------------------------------------------------------------------- /.changes/5.4.6.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.4.6 - June 06, 2025 2 | 3 | ### Fixes 4 | 5 | - Handles two json comments if dbt adds them ([#185](https://github.com/get-select/dbt-snowflake-monitoring/pull/185)) 6 | 7 | ### Contributors 8 | - [@parasbhavnani](https://github.com/parasbhavnani) (Fixes) 9 | 10 | -------------------------------------------------------------------------------- /.changes/5.4.1.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.4.1 - February 11, 2025 2 | 3 | ### Fixes 4 | 5 | - Fix bug in hourly_spend introduced on 5.4.0 ([#175](https://github.com/get-select/dbt-snowflake-monitoring/pull/175)) 6 | 7 | ### Contributors 8 | - [@fernandobrito](https://github.com/fernandobrito) (Fixes) 9 | 10 | -------------------------------------------------------------------------------- /.changes/5.1.1.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.1.1 - May 14, 2024 2 | 3 | ### Fixes 4 | 5 | - Include overage spend on services recently introduced ([#154](https://github.com/get-select/dbt-snowflake-monitoring/pull/154)) 6 | 7 | ### Contributors 8 | - [@fernandobrito](https://github.com/fernandobrito) (Fixes) 9 | 10 | -------------------------------------------------------------------------------- /.changes/5.3.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.3.0 - June 15, 2024 2 | 3 | ### Features 4 | 5 | - Add new fields for stg_serverless_task_history ([#160](https://github.com/get-select/dbt-snowflake-monitoring/pull/160)) 6 | 7 | ### Contributors 8 | - [@fernandobrito](https://github.com/fernandobrito) (Features) 9 | 10 | -------------------------------------------------------------------------------- /.changes/5.2.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.2.0 - May 28, 2024 2 | 3 | ### Features 4 | 5 | - Add root_query_id, parent_query_id to stg_access_history ([#156](https://github.com/get-select/dbt-snowflake-monitoring/pull/156)) 6 | 7 | ### Contributors 8 | - [@tnightengale](https://github.com/tnightengale) (Features) 9 | 10 | -------------------------------------------------------------------------------- /.changes/5.4.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.4.0 - February 06, 2025 2 | 3 | ### Features 4 | 5 | - Support all services by default in hourly_spend ([#173](https://github.com/get-select/dbt-snowflake-monitoring/pull/173)) 6 | 7 | ### Contributors 8 | - [@fernandobrito](https://github.com/fernandobrito) (Features) 9 | 10 | -------------------------------------------------------------------------------- /.changes/5.4.7.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.4.7 - June 18, 2025 2 | 3 | ### Fixes 4 | 5 | - pins the latest query_tag package in package-lock.yml ([#188](https://github.com/get-select/dbt-snowflake-monitoring/pull/188)) 6 | 7 | ### Contributors 8 | - [@jeff-skoldberg-gmds](https://github.com/jeff-skoldberg-gmds) (Fixes) 9 | 10 | -------------------------------------------------------------------------------- /.changes/5.5.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.5.0 - November 10, 2025 2 | 3 | ### Features 4 | 5 | - feat: add configuration variables for custom source ([#195](https://github.com/get-select/dbt-snowflake-monitoring/pull/195)) 6 | 7 | ### Contributors 8 | - [@erlendhbarstad](https://github.com/erlendhbarstad) (Features) 9 | 10 | -------------------------------------------------------------------------------- /.changes/header.tpl.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), 6 | and is generated by [Changie](https://github.com/miniscruff/changie). 7 | -------------------------------------------------------------------------------- /models/staging/stg_database_storage_usage_history.sql: -------------------------------------------------------------------------------- 1 | {{ config(materialized='view') }} 2 | 3 | select 4 | usage_date as date, 5 | database_name, 6 | average_database_bytes, 7 | average_failsafe_bytes, 8 | average_hybrid_table_storage_bytes 9 | from {{ source('snowflake_account_usage', 'database_storage_usage_history') }} 10 | -------------------------------------------------------------------------------- /.changes/1.0.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 1.0.0 - October 17, 2022 2 | 3 | ### Features 4 | 5 | - Add query_history_enriched model with additional dimensions and cost ([#12](https://github.com/get-select/dbt-snowflake-monitoring/pull/12)) 6 | - Add cost per query model ([#1](https://github.com/get-select/dbt-snowflake-monitoring/pull/1)) 7 | 8 | 9 | -------------------------------------------------------------------------------- /.changes/5.0.3.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.0.3 - January 16, 2024 2 | 3 | ### Features 4 | 5 | - Add dbt_snowflake_monitoring_incremental_days to set incremental lookback period ([#140](https://github.com/get-select/dbt-snowflake-monitoring/pull/140)) 6 | 7 | ### Contributors 8 | - [@smitsrr](https://github.com/smitsrr) (Features) 9 | 10 | -------------------------------------------------------------------------------- /models/staging/stg_metering_history.sql: -------------------------------------------------------------------------------- 1 | {{ config( 2 | materialized='table' 3 | ) }} 4 | 5 | select 6 | service_type, 7 | start_time, 8 | end_time, 9 | entity_id, 10 | name, 11 | credits_used_compute, 12 | credits_used_cloud_services, 13 | credits_used 14 | from {{ source('snowflake_account_usage', 'metering_history') }} 15 | order by start_time asc 16 | -------------------------------------------------------------------------------- /models/query_base_table_access.sql: -------------------------------------------------------------------------------- 1 | select 2 | _unique_id, 3 | query_id, 4 | query_start_time, 5 | user_name, 6 | object_name as full_table_name, 7 | table_id, 8 | columns_accessed 9 | from {{ ref('query_base_object_access') }} 10 | where 11 | object_domain = 'Table' -- removes secured views 12 | and table_id is not null -- removes tables from a data share 13 | -------------------------------------------------------------------------------- /models/query_direct_table_access.sql: -------------------------------------------------------------------------------- 1 | select 2 | _unique_id, 3 | query_id, 4 | query_start_time, 5 | user_name, 6 | object_name as full_table_name, 7 | table_id, 8 | columns_accessed 9 | from {{ ref('query_direct_object_access') }} 10 | where 11 | object_domain = 'Table' -- removes secured views 12 | and table_id is not null -- removes tables from a data share 13 | -------------------------------------------------------------------------------- /models/staging/stg_storage_usage.sql: -------------------------------------------------------------------------------- 1 | {{ config(materialized='view') }} 2 | 3 | select 4 | usage_date as date, 5 | storage_bytes, 6 | stage_bytes, 7 | failsafe_bytes, 8 | hybrid_table_storage_bytes, 9 | archive_storage_cool_bytes, 10 | archive_storage_cold_bytes, 11 | archive_storage_retrieval_temp_bytes 12 | from {{ source('snowflake_account_usage', 'storage_usage') }} 13 | -------------------------------------------------------------------------------- /dbt_project.yml: -------------------------------------------------------------------------------- 1 | name: 'dbt_snowflake_monitoring' 2 | version: '5.5.0' 3 | config-version: 2 4 | 5 | profile: dbt_snowflake_monitoring 6 | 7 | model-paths: ["models"] 8 | 9 | clean-targets: 10 | - target 11 | - dbt_packages 12 | 13 | models: 14 | # The config below applies only when running this package directly as a project, not when installed as a package in another project. 15 | +on_schema_change: "append_new_columns" 16 | -------------------------------------------------------------------------------- /models/staging/stg_stage_storage_usage_history.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: stg_stage_storage_usage_history 5 | description: See https://docs.snowflake.com/en/sql-reference/account-usage/stage_storage_usage_history.html 6 | columns: 7 | - name: date 8 | description: Date of this storage usage record. 9 | - name: average_stage_bytes 10 | description: Number of bytes of stage storage used. 11 | -------------------------------------------------------------------------------- /macros/create_merge_objects_udf.sql: -------------------------------------------------------------------------------- 1 | {% macro create_merge_objects_udf(relation) %} 2 | 3 | create or replace function {{ adapter.quote_as_configured(this.database, 'database') }}.{{ adapter.quote_as_configured(this.schema, 'schema') }}.merge_objects(obj1 variant, obj2 variant) 4 | returns variant 5 | language javascript 6 | comment = 'Created by dbt-snowflake-monitoring dbt package.' 7 | as 8 | $$ 9 | return x = Object.assign(OBJ1, OBJ2) 10 | $$ 11 | 12 | {% endmacro %} 13 | -------------------------------------------------------------------------------- /models/staging/stg_usage_in_currency_daily.sql: -------------------------------------------------------------------------------- 1 | select 2 | organization_name, 3 | contract_number, 4 | account_name, 5 | account_locator, 6 | region, 7 | service_level, 8 | usage_date, 9 | usage_type, 10 | currency, 11 | usage, 12 | usage_in_currency, 13 | balance_source, 14 | billing_type, 15 | rating_type, 16 | service_type, 17 | is_adjustment 18 | from {{ source('snowflake_organization_usage', 'usage_in_currency_daily') }} 19 | -------------------------------------------------------------------------------- /models/staging/stg_remaining_balance_daily.sql: -------------------------------------------------------------------------------- 1 | -- <1000 rows, will be more expensive to materialize incrementally with multiple SQL statements 2 | {{ config(materialized='table') }} 3 | 4 | select 5 | date, 6 | organization_name, 7 | contract_number, 8 | currency, 9 | free_usage_balance, 10 | capacity_balance, 11 | on_demand_consumption_balance, 12 | rollover_balance 13 | from {{ source('snowflake_organization_usage', 'remaining_balance_daily') }} 14 | order by date 15 | -------------------------------------------------------------------------------- /.changes/1.4.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 1.4.0 - January 06, 2023 2 | 3 | ### Features 4 | 5 | - Add remaining balance daily model ([#58](https://github.com/get-select/dbt-snowflake-monitoring/pull/58)) 6 | - Improve cost per query freshness by using latest rates ([#52](https://github.com/get-select/dbt-snowflake-monitoring/pull/52)) 7 | 8 | ### Fixes 9 | 10 | - Select correct overage vs. regular rate each day ([#59](https://github.com/get-select/dbt-snowflake-monitoring/pull/59)) 11 | 12 | 13 | -------------------------------------------------------------------------------- /models/staging/stg_metering_daily_history.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: stg_metering_daily_history 5 | description: See https://docs.snowflake.com/en/sql-reference/account-usage/metering_daily_history.html 6 | columns: 7 | - name: date 8 | description: The date (in the UTC time zone) in which the usage took place. 9 | - name: credits_adjustment_cloud_services 10 | description: Number of credits adjusted for included cloud services. This is a negative value (e.g. -9). 11 | -------------------------------------------------------------------------------- /models/staging/stg_rate_sheet_daily.sql: -------------------------------------------------------------------------------- 1 | -- <1000 rows, will be more expensive to materialize incrementally with multiple SQL statements 2 | {{ config(materialized='table') }} 3 | 4 | select 5 | date, 6 | organization_name, 7 | contract_number, 8 | account_name, 9 | account_locator, 10 | region, 11 | service_level, 12 | usage_type, 13 | currency, 14 | effective_rate, 15 | service_type 16 | from {{ source('snowflake_organization_usage', 'rate_sheet_daily') }} 17 | order by date 18 | -------------------------------------------------------------------------------- /.changes/1.2.3.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 1.2.3 - December 15, 2022 2 | 3 | ### Features 4 | 5 | - Add serverless tasks history and spend ([#37](https://github.com/get-select/dbt-snowflake-monitoring/pull/37)) 6 | - Add `warehouse_cluster_status`, `warehouse_credits_map` and `warehouses_type2_dimension` models ([#40](https://github.com/get-select/dbt-snowflake-monitoring/pull/40)) 7 | 8 | ### Contributors 9 | - [@gthesheep](https://github.com/gthesheep) (Features) 10 | - [@ian-whitestone](https://github.com/ian-whitestone) (Features) 11 | -------------------------------------------------------------------------------- /integration_test_project/models/sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: snowflake_account_usage 5 | overrides: dbt_snowflake_monitoring 6 | database: '{{ env_var("SOURCE_DATABASE_OVERRIDE", "snowflake") }}' 7 | schema: '{{ env_var("SOURCE_SCHEMA_OVERRIDE", "account_usage") }}' 8 | 9 | - name: snowflake_organization_usage 10 | overrides: dbt_snowflake_monitoring 11 | database: '{{ env_var("SOURCE_DATABASE_OVERRIDE", "snowflake") }}' 12 | schema: '{{ env_var("SOURCE_SCHEMA_OVERRIDE", "organization_usage") }}' 13 | -------------------------------------------------------------------------------- /models/staging/stg_access_history.sql: -------------------------------------------------------------------------------- 1 | {{ config(materialized='incremental') }} 2 | 3 | select 4 | query_id, 5 | parent_query_id, 6 | root_query_id, 7 | query_start_time, 8 | user_name, 9 | direct_objects_accessed, 10 | base_objects_accessed, 11 | objects_modified 12 | from {{ source('snowflake_account_usage', 'access_history') }} 13 | 14 | {% if is_incremental() %} 15 | where query_start_time > (select coalesce(max(query_start_time), '1970-01-01') from {{ this }}) 16 | {% endif %} 17 | 18 | order by query_start_time asc 19 | -------------------------------------------------------------------------------- /integration_test_project/seeds/monthly_spend_fixture.csv: -------------------------------------------------------------------------------- 1 | MONTH,SERVICE,SPEND 2 | 2022-04-01,Storage,1.136266305 3 | 2022-04-01,Compute,912.772222176 4 | 2022-04-01,Adj For Incl Cloud Services,-74.921528302 5 | 2022-04-01,Cloud Services,74.921528302 6 | 2022-04-01,Automatic Clustering,0.051113362 7 | 2022-04-01,Snowpipe,0 8 | 2022-04-01,Snowpipe Streaming,0 9 | 2022-04-01,Serverless Tasks,0 10 | 2022-04-01,Search Optimization,0 11 | 2022-04-01,Replication,0 12 | 2022-04-01,Query Acceleration,0 13 | 2022-04-01,Materialized Views,0 14 | 2022-04-01,Data Transfer,0 15 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: local 3 | hooks: 4 | - id: sqlfluff-fix 5 | name: SQLFluff Fix 6 | description: This hook runs sqlfluff fix. 7 | entry: tox -e fix -- 8 | files: ^models/ 9 | require_serial: true 10 | types: [sql] 11 | language: system 12 | - id: sqlfluff-lint 13 | name: SQLFluff Lint 14 | description: This hook runs sqlfluff lint. 15 | entry: tox -e lint -- 16 | files: ^models/ 17 | require_serial: true 18 | types: [sql] 19 | language: system 20 | -------------------------------------------------------------------------------- /.changes/1.2.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 1.2.0 - November 26, 2022 2 | 3 | ### Features 4 | 5 | - Add get_query_comment macro for model metadata in query history ([#29](https://github.com/get-select/dbt-snowflake-monitoring/pull/29)) 6 | - Support overage usage type for PAYG accounts ([#24](https://github.com/get-select/dbt-snowflake-monitoring/pull/24)) 7 | 8 | ### Fixes 9 | 10 | - Division by zero in cost_per_query model ([#27](https://github.com/get-select/dbt-snowflake-monitoring/pull/27)) 11 | 12 | ### Contributors 13 | - [@aphethean1](https://github.com/aphethean1) (Features, Fixes) 14 | 15 | -------------------------------------------------------------------------------- /.changes/2.0.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 2.0.0 - February 08, 2023 2 | The most significant change in 2.0.0 is using query tags instead of comments to attach dbt metadata to queries. To upgrade from previous versions: 3 | 4 | 1. Remove the query comment configuration from `dbt_project.yml` 5 | 2. Follow the instructions to configure the query tag in the [Quickstart](https://github.com/get-select/dbt-snowflake-monitoring#quickstart). 6 | 7 | ### Breaking Changes 8 | 9 | - Switch to using query tags instead of comments ([#78](https://github.com/get-select/dbt-snowflake-monitoring/pull/78)) 10 | 11 | 12 | -------------------------------------------------------------------------------- /models/staging/stg_serverless_task_history.sql: -------------------------------------------------------------------------------- 1 | {{ config( 2 | materialized='incremental', 3 | unique_key=['start_time', 'task_id'], 4 | ) }} 5 | 6 | select 7 | start_time, 8 | end_time, 9 | task_id, 10 | task_name, 11 | schema_id, 12 | schema_name, 13 | database_id, 14 | database_name, 15 | credits_used 16 | from {{ source('snowflake_account_usage', 'serverless_task_history') }} 17 | 18 | {% if is_incremental() %} 19 | where end_time > (select dateadd(day, -3, coalesce(max(end_time), '1970-01-01') ) from {{ this }}) 20 | {% endif %} 21 | 22 | order by start_time 23 | -------------------------------------------------------------------------------- /integration_test_project/tests/assert_no_queries_are_dropped.sql: -------------------------------------------------------------------------------- 1 | /* 2 | We expect all queries in query_history_enriched to be present in cost_per_query, and vice versa. 3 | 4 | Cost per query won't have some of the more recent queries, so we only compare queries 5 | before the current date 6 | */ 7 | select * 8 | from {{ ref('query_history_enriched') }} as a 9 | full outer join {{ ref('cost_per_query') }} as b 10 | on a.query_id = b.query_id 11 | where 12 | ( 13 | a.query_id is null 14 | or b.query_id is null 15 | ) 16 | and date(coalesce(a.start_time, b.start_time)) < current_date 17 | -------------------------------------------------------------------------------- /.changes/5.1.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.1.0 - May 06, 2024 2 | 3 | ### Features 4 | 5 | - Support more usage types ([#152](https://github.com/get-select/dbt-snowflake-monitoring/pull/152)) 6 | 7 | ### Fixes 8 | 9 | - Update to use new service_type values ([#151](https://github.com/get-select/dbt-snowflake-monitoring/pull/151)) 10 | - Calculate net cloud services costs for reader accounts ([#153](https://github.com/get-select/dbt-snowflake-monitoring/pull/153)) 11 | 12 | ### Contributors 13 | - [@fernandobrito](https://github.com/fernandobrito) (Features) 14 | - [@stumelius](https://github.com/stumelius) (Fixes) 15 | 16 | -------------------------------------------------------------------------------- /integration_test_project/dbt_project.yml: -------------------------------------------------------------------------------- 1 | name: 'dbt_snowflake_monitoring_tests' 2 | version: '1.0.0' 3 | config-version: 2 4 | 5 | profile: dbt_snowflake_monitoring 6 | 7 | models: 8 | dbt_snowflake_monitoring: 9 | enabled: true 10 | 11 | clean-targets: 12 | - target 13 | - dbt_packages 14 | 15 | dispatch: 16 | - macro_namespace: dbt 17 | search_order: 18 | - dbt_snowflake_monitoring_tests 19 | - dbt_snowflake_monitoring 20 | - dbt 21 | 22 | 23 | query-comment: 24 | comment: '{{ dbt_snowflake_monitoring.get_query_comment(node) }}' 25 | append: true # Snowflake removes prefixed comments. 26 | -------------------------------------------------------------------------------- /models/staging/stg_warehouse_metering_history.sql: -------------------------------------------------------------------------------- 1 | {{ config( 2 | materialized='incremental', 3 | unique_key=['start_time', 'warehouse_id'], 4 | ) }} 5 | 6 | select 7 | start_time, 8 | end_time, 9 | warehouse_id, 10 | warehouse_name, 11 | credits_used, 12 | credits_used_compute, 13 | credits_used_cloud_services 14 | from {{ source('snowflake_account_usage', 'warehouse_metering_history') }} 15 | 16 | {% if is_incremental() %} 17 | -- account for changing metering data 18 | where end_time > (select coalesce(dateadd(day, -7, max(end_time)), '1970-01-01') from {{ this }}) 19 | {% endif %} 20 | 21 | order by start_time 22 | -------------------------------------------------------------------------------- /.changes/4.6.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 4.6.0 - November 09, 2023 2 | 3 | ### Features 4 | 5 | - Add credit consumption to cost per query ([#127](https://github.com/get-select/dbt-snowflake-monitoring/pull/127)) 6 | - Add query parameterized hash ([#137](https://github.com/get-select/dbt-snowflake-monitoring/pull/137)) 7 | 8 | ### Fixes 9 | - Support quoting: true ([#139](https://github.com/get-select/dbt-snowflake-monitoring/pull/139)) 10 | 11 | ### Contributors 12 | - [@sidhreddy](https://github.com/sidhreddy) (Features) 13 | - [@yingyingqiqi](https://github.com/yingyingqiqi) (Features) 14 | - [@ernestoongaro](https://github.com/ernestoongaro) (Features) 15 | -------------------------------------------------------------------------------- /macros/query_tags.sql: -------------------------------------------------------------------------------- 1 | {% macro set_query_tag() -%} 2 | {{ return(adapter.dispatch('set_query_tag', 'dbt_snowflake_query_tags')()) }} 3 | {%- endmacro %} 4 | 5 | {% macro default__set_query_tag() -%} 6 | {{ return(adapter.dispatch('set_query_tag', 'dbt_snowflake_query_tags')()) }} 7 | {% endmacro %} 8 | 9 | {% macro unset_query_tag(original_query_tag) -%} 10 | {{ return(adapter.dispatch('unset_query_tag', 'dbt_snowflake_query_tags')(original_query_tag)) }} 11 | {%- endmacro %} 12 | 13 | {% macro default__unset_query_tag(original_query_tag) -%} 14 | {{ return(adapter.dispatch('unset_query_tag', 'dbt_snowflake_query_tags')(original_query_tag)) }} 15 | {% endmacro %} 16 | -------------------------------------------------------------------------------- /models/remaining_balance_daily_without_contract_view.sql: -------------------------------------------------------------------------------- 1 | select 2 | date, 3 | organization_name, 4 | currency, 5 | free_usage_balance, 6 | capacity_balance, 7 | on_demand_consumption_balance, 8 | rollover_balance 9 | from {{ ref('stg_remaining_balance_daily') }} 10 | {# 11 | From what I can tell, there will only ever be 1 organization_name in remaining_balance_daily. 12 | During a contract switchover, there may be two records with the same date, but different contract_numbers. 13 | Assume the higher contract_number is more recent. Chose not to group by date and aggregate balances in 14 | case the currency changes.. 15 | #} 16 | qualify row_number() over (partition by date 17 | order by contract_number desc nulls last) = 1 18 | -------------------------------------------------------------------------------- /models/staging/stg_database_storage_usage_history.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: stg_database_storage_usage_history 5 | description: See https://docs.snowflake.com/en/sql-reference/account-usage/database_storage_usage_history.html 6 | columns: 7 | - name: date 8 | description: Date (in the UTC time zone) of this storage usage record. 9 | - name: database_name 10 | description: Name of the database. 11 | - name: average_database_bytes 12 | description: Number of bytes of database storage used, including data in Time Travel. 13 | - name: average_failsafe_bytes 14 | description: Number of bytes of Fail-safe storage used. 15 | - name: average_hybrid_table_storage_bytes 16 | description: Number of bytes of hybrid storage used. 17 | -------------------------------------------------------------------------------- /.changes/5.0.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 5.0.0 - January 14, 2024 2 | 3 | ### Features 4 | 5 | - Add query acceleration costs and update cost per query algorithm to include them ([#141](https://github.com/get-select/dbt-snowflake-monitoring/pull/141)) 6 | - Make cost_per_query model incremental ([#141](https://github.com/get-select/dbt-snowflake-monitoring/pull/141)) 7 | 8 | ### Breaking Changes 9 | 10 | - Add entity_id to stg_metering_history ([#141](https://github.com/get-select/dbt-snowflake-monitoring/pull/141)) 11 | 12 | To upgrade from 4.x.x, you'll need to full refresh the `stg_metering_history` model. 13 | 14 | ### Fixes 15 | 16 | - Support quoting: true ([#139](https://github.com/get-select/dbt-snowflake-monitoring/pull/139)) 17 | 18 | ### Contributors 19 | - [@ernestoongaro](https://github.com/ernestoongaro) (Fixes) -------------------------------------------------------------------------------- /models/daily_spend.sql: -------------------------------------------------------------------------------- 1 | select 2 | convert_timezone('UTC', hour)::date as date, -- get UTC date to align with Snowflake billing 3 | service, 4 | storage_type, 5 | warehouse_name, 6 | database_name, 7 | sum(usage) as usage, 8 | sum(usage_net_cloud_services) as usage_net_cloud_services, 9 | sum(spend) as spend, 10 | sum(spend_net_cloud_services) as spend_net_cloud_services, 11 | any_value(currency) as currency, 12 | any_value(usage_unit) as usage_unit, 13 | -- Weighted average rates across the day. Typically rates are consistent 14 | -- throughout a day so this should still be fairly accurate. 15 | case 16 | when sum(usage) > 0 then sum(spend) / sum(usage) 17 | when sum(usage_net_cloud_services) > 0 then sum(spend_net_cloud_services) / sum(usage_net_cloud_services) 18 | else 0 19 | end as usage_rate 20 | from {{ ref('hourly_spend') }} 21 | group by 1, 2, 3, 4, 5 22 | -------------------------------------------------------------------------------- /.changes/3.0.0.md: -------------------------------------------------------------------------------- 1 | ## dbt-snowflake-monitoring 3.0.0 - February 26, 2023 2 | After attempting to use only query comments or query tags in versions 1 and 2, we've learned we actually need both. The reasons for that are: 3 | * The `is_incremental` macro is only available from within the query tag dbt context, and is needed to determine whether a model run is incremental or not. 4 | * Query tags have a maximum character limit of 2000, which is easily exceeded by a list of refs containing more than a few models. 5 | 6 | To upgrade from 2.x.x, follow the latest instructions in the [Quickstart](https://github.com/get-select/dbt-snowflake-monitoring#quickstart). The package is still able to process query metadata generated by previous versions of the package, so no data will be lost on upgrade. 7 | 8 | ### Breaking Changes 9 | 10 | - Use query comments and query tags to avoid query tag character limit ([#87](https://github.com/get-select/dbt-snowflake-monitoring/pull/87)) 11 | -------------------------------------------------------------------------------- /models/warehouse_credits_map.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: warehouse_credits_map 5 | description: This model contains the credit consumption for each type and size of warehouse. It contains 1 row per hour per warehouse size and type. 6 | columns: 7 | - name: warehouse_size 8 | description: The size of the warehouse. These values will join directly to the warehouse_size column in the query_history models. 9 | - name: warehouse_type 10 | description: The type of the warehouse. Will be STANDARD or SNOWPARK-OPTIMIZED. 11 | - name: credits_per_hour 12 | description: The hourly rate at which credits are charged for this warehouse configuration 13 | - name: credits_per_minute 14 | description: The minutely rate at which credits are charged for this warehouse configuration 15 | - name: credits_per_second 16 | description: The secondly rate at which credits are charged for this warehouse configuration 17 | 18 | -------------------------------------------------------------------------------- /models/staging/sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: snowflake_account_usage 5 | database: "{{ var('snowflake_account_usage_database', 'snowflake') }}" 6 | schema: "{{ var('snowflake_account_usage_schema', 'account_usage') }}" 7 | tables: 8 | - name: access_history 9 | - name: database_storage_usage_history 10 | - name: metering_daily_history 11 | - name: metering_history 12 | - name: query_history 13 | - name: serverless_task_history 14 | - name: stage_storage_usage_history 15 | - name: storage_usage 16 | - name: warehouse_events_history 17 | - name: warehouse_metering_history 18 | 19 | - name: snowflake_organization_usage 20 | database: "{{ var('snowflake_account_usage_database', 'snowflake') }}" 21 | schema: "{{ var('snowflake_account_usage_schema', 'organization_usage') }}" 22 | tables: 23 | - name: rate_sheet_daily 24 | - name: remaining_balance_daily 25 | - name: usage_in_currency_daily 26 | 27 | -------------------------------------------------------------------------------- /integration_test_project/profiles.yml: -------------------------------------------------------------------------------- 1 | # HEY! This file is used in the dbt-snowflake-monitoring integrations tests with GitHub Actions. 2 | # You should __NEVER__ check credentials into version control. Thanks for reading :) 3 | 4 | config: 5 | send_anonymous_usage_stats: False 6 | use_colors: True 7 | 8 | dbt_snowflake_monitoring: 9 | target: snowflake 10 | outputs: 11 | snowflake: 12 | type: snowflake 13 | account: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT') }}" 14 | user: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_USER') }}" 15 | password: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD') }}" 16 | role: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_ROLE') }}" 17 | database: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE') }}" 18 | warehouse: "{{ env_var('DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE') }}" 19 | schema: dbt_snowflake_monitoring_test_commit_{{ env_var('GITHUB_SHA_OVERRIDE', '') if env_var('GITHUB_SHA_OVERRIDE', '') else env_var('GITHUB_SHA') }} 20 | threads: 8 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 get-select 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /models/warehouses_type2_dimension.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: warehouses_type2_dimension 5 | description: | 6 | This model tracks the warehouse size and name over time. It contains 1 row per period that the warehouse state applies. The following caveats apply: 7 | 8 | - if user changes the warehouse size/name, we won't register the new attributes until the next query runs against it 9 | - this could also mean that warehouse size changes are not registered if they were changed and then reverted 10 | columns: 11 | - name: warehouse_id 12 | description: The unique warehouse ID (assigned by Snowflake) that corresponds to the warehouse name in your account. 13 | - name: warehouse_name 14 | description: The name of the warehouse during the given period. 15 | - name: warehouse_size 16 | description: The size of the warehouse during the given period 17 | - name: valid_from 18 | description: Timestamp for the beginning of the period. 19 | - name: valid_to 20 | description: Timestamp for the end of the period. 21 | - name: is_current 22 | description: Boolean indicator to filter to the most recent state of the warehouse 23 | -------------------------------------------------------------------------------- /models/staging/stg_access_history.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: stg_access_history 5 | description: See https://docs.snowflake.com/en/sql-reference/account-usage/access_history.html 6 | columns: 7 | - name: query_id 8 | description: An internal, system-generated identifier for the SQL statement. 9 | - name: query_start_time 10 | description: The statement start time (UTC time zone). 11 | - name: user_name 12 | description: The user who issued the query. 13 | - name: direct_objects_accessed 14 | description: A JSON array of data objects such as tables, views, and columns directly named in the query explicitly or through shortcuts such as using an asterisk (i.e. *). Virtual columns can be returned in this field. 15 | - name: base_objects_accessed 16 | description: A JSON array of all base data objects, specifically, columns of tables to execute the query. This field specifies view names or view columns, including virtual columns, if a shared view is accessed in a data sharing consumer account. 17 | - name: objects_modified 18 | description: A JSON array that specifies the objects that were associated with a write operation in the query. 19 | -------------------------------------------------------------------------------- /models/staging/stg_metering_history.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: stg_metering_history 5 | description: See https://docs.snowflake.com/en/sql-reference/account-usage/metering_history.html 6 | columns: 7 | - name: name 8 | description: Name of the service type. 9 | - name: credits_used_compute 10 | description: Number of credits used for virtual warehouses in the hour. 11 | - name: start_time 12 | description: The date and beginning of the hour (in the UTC time zone) in which the usage took place. 13 | - name: service_type 14 | description: The type of service, which can be one of AUTO_CLUSTERING, MATERIALIZED_VIEW, PIPE, QUERY_ACCELERATION, REPLICATION, SEARCH_OPTIMIZATION, WAREHOUSE_METERING, or WAREHOUSE_METERING_READER. 15 | - name: credits_used_cloud_services 16 | description: Number of credits used for cloud services in the hour. 17 | - name: credits_used 18 | description: Total number of credits used for the account in the hour. This is a sum of CREDITS_USED_COMPUTE and CREDITS_USED_CLOUD_SERVICES. This value does not take into account the adjustment for cloud services, and may therefore be greater than your actual credit consumption. 19 | -------------------------------------------------------------------------------- /models/warehouse_credits_map.sql: -------------------------------------------------------------------------------- 1 | {{ config(materialized='view') }} 2 | 3 | with 4 | warehouse_credits_map as ( 5 | select * from ( 6 | values 7 | ('X-Small', 'STANDARD', 1), 8 | ('Small', 'STANDARD', 2), 9 | ('Medium', 'STANDARD', 4), 10 | ('Large', 'STANDARD', 8), 11 | ('X-Large', 'STANDARD', 16), 12 | ('2X-Large', 'STANDARD', 32), 13 | ('3X-Large', 'STANDARD', 64), 14 | ('4X-Large', 'STANDARD', 128), 15 | ('5X-Large', 'STANDARD', 256), 16 | ('6X-Large', 'STANDARD', 512), 17 | ('Medium', 'SNOWPARK-OPTIMIZED', 6), 18 | ('Large', 'SNOWPARK-OPTIMIZED', 12), 19 | ('X-Large', 'SNOWPARK-OPTIMIZED', 24), 20 | ('2X-Large', 'SNOWPARK-OPTIMIZED', 48), 21 | ('3X-Large', 'SNOWPARK-OPTIMIZED', 96), 22 | ('4X-Large', 'SNOWPARK-OPTIMIZED', 192), 23 | ('5X-Large', 'SNOWPARK-OPTIMIZED', 384), 24 | ('6X-Large', 'SNOWPARK-OPTIMIZED', 768) 25 | ) as t (warehouse_size, warehouse_type, credits_per_hour) 26 | ) 27 | 28 | select 29 | warehouse_size, 30 | warehouse_type, 31 | credits_per_hour, 32 | credits_per_hour / 60 as credits_per_minute, 33 | credits_per_hour / 3600 as credits_per_second 34 | from warehouse_credits_map 35 | -------------------------------------------------------------------------------- /models/query_base_table_access.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: query_base_table_access 5 | description: This model tracks the base tables accessed by a query. It contains 1 row per query per table. It does not include tables from a data share. Access history is only available for Snowflake accounts on enterprise or higher, and therefore this model will be empty for standard accounts. See https://docs.snowflake.com/en/sql-reference/account-usage/access_history.html for more details. See https://select.dev/posts/snowflake-unused-tables#direct-versus-base-objects-accessed for more details on the difference between base versus direct tables accessed. 6 | columns: 7 | - name: _unique_id 8 | description: Unique identifier for each row in the table 9 | - name: query_id 10 | description: An internal, system-generated identifier for the SQL statement. 11 | - name: query_start_time 12 | description: The statement start time (UTC time zone). 13 | - name: user_name 14 | description: The user who issued the query. 15 | - name: table_name 16 | description: Fully qualified table name, i.e. .. 17 | - name: table_id 18 | description: Unique identifier for the table 19 | - name: columns_accessed 20 | description: Array of column_names accessed in the table 21 | -------------------------------------------------------------------------------- /models/query_direct_table_access.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: query_direct_table_access 5 | description: This model tracks the direct tables accessed by a query. It contains 1 row per query per table. It does not include tables from a data share. Access history is only available for Snowflake accounts on enterprise or higher, and therefore this model will be empty for standard accounts. See https://docs.snowflake.com/en/sql-reference/account-usage/access_history.html for more details. See https://select.dev/posts/snowflake-unused-tables#direct-versus-base-objects-accessed for more details on the difference between base versus direct tables accessed. 6 | columns: 7 | - name: _unique_id 8 | description: Unique identifier for each row in the table 9 | - name: query_id 10 | description: An internal, system-generated identifier for the SQL statement. 11 | - name: query_start_time 12 | description: The statement start time (UTC time zone). 13 | - name: user_name 14 | description: The user who issued the query. 15 | - name: full_table_name 16 | description: Fully qualified table name, i.e. .. 17 | - name: table_id 18 | description: Unique identifier for the table 19 | - name: columns_accessed 20 | description: Array of column_names accessed in the table 21 | -------------------------------------------------------------------------------- /models/staging/stg_serverless_task_history.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: stg_serverless_task_history 5 | description: An incrementally materialized copy of the snowflake.account_usage.serverless_task_history view. See https://docs.snowflake.com/en/sql-reference/account-usage/serverless_task_history.html 6 | columns: 7 | - name: start_time 8 | description: The date and beginning of the hour (in the UTC time zone) in which the serverless task took place. 9 | - name: end_time 10 | description: The date and end of the hour (in the UTC time zone) in which the serverless task took place. 11 | - name: task_id 12 | description: Internal/system-generated identifier for the task. 13 | - name: task_name 14 | description: Name of the task. 15 | - name: schema_id 16 | description: Internal/system-generated identifier for the schema that contains the serverless task. 17 | - name: schema_name 18 | description: Name of the schema that contains the serverless task. 19 | - name: database_id 20 | description: Internal/system-generated identifier for the database that contains the serverless task. 21 | - name: database_name 22 | description: Name of the database in which the task is located. 23 | - name: credits_used 24 | description: Total number of credits used for the task. 25 | -------------------------------------------------------------------------------- /models/staging/stg_warehouse_metering_history.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: stg_warehouse_metering_history 5 | description: An incrementally materialized copy of the snowflake.account_usage.warehouse_metering_history view. It can be used to return the hourly credit usage for a single warehouse (or all the warehouses in your account) within the last 365 days (1 year). It contains 1 row per hour per warehouse. 6 | columns: 7 | - name: start_time 8 | description: The date and beginning of the hour (in the UTC time zone) in which the warehouse usage took place. 9 | - name: end_time 10 | description: The date and end of the hour (in the UTC time zone) in which the warehouse usage took place. 11 | - name: warehouse_id 12 | description: Internal/system-generated identifier for the warehouse. 13 | - name: warehouse_name 14 | description: Name of the warehouse. 15 | - name: credits_used 16 | description: Total number of credits used for the warehouse in the hour. This is a sum of CREDITS_USED_COMPUTE and CREDITS_USED_CLOUD_SERVICES. This value does not take into account the adjustment for cloud services, and may therefore be greater than your actual credit consumption. 17 | - name: credits_used_compute 18 | description: Number of credits used for the warehouse in the hour. 19 | - name: credits_used_cloud_services 20 | description: Number of credits used for cloud services in the hour. 21 | -------------------------------------------------------------------------------- /models/staging/stg_storage_usage.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: stg_storage_usage 5 | description: See https://docs.snowflake.com/en/sql-reference/account-usage/storage_usage 6 | columns: 7 | - name: date 8 | description: Date (in the UTC time zone) of this storage usage record. 9 | - name: storage_bytes 10 | description: Number of bytes of database storage used, including data in Time Travel. 11 | - name: stage_bytes 12 | description: Number of bytes of stage storage used by files in all internal stages (named, table, and user). 13 | - name: failsafe_bytes 14 | description: Number of bytes of Fail-safe storage used. 15 | - name: hybrid_table_storage_bytes 16 | description: Number of bytes of hybrid table storage used (data in the row store). 17 | - name: archive_storage_cool_bytes 18 | description: Number of all bytes of table storage used in the COOL storage tier, including active bytes, Fail-safe bytes, Time Travel bytes, and bytes subject to minimum storage duration charges. 19 | - name: archive_storage_cold_bytes 20 | description: Number of all bytes of table storage used in the COLD storage tier, including active bytes, Fail-safe bytes, Time Travel bytes, and bytes subject to minimum storage duration charges. 21 | - name: archive_storage_retrieval_temp_bytes 22 | description: Number of bytes used in the standard storage tier, during data retrieval from the COLD storage tier. 23 | -------------------------------------------------------------------------------- /integration_test_project/macros/advanced_equality.sql: -------------------------------------------------------------------------------- 1 | {# Inspired by https://towardsdatascience.com/how-to-do-unit-testing-in-dbt-cb5fb660fbd8 #} 2 | 3 | {% test advanced_equality(model, compare_model, round_columns=None) %} 4 | 5 | {% set compare_columns = adapter.get_columns_in_relation(model) | map(attribute='quoted') %} 6 | {% set compare_cols_csv = compare_columns | join(', ') %} 7 | 8 | {% if round_columns %} 9 | {% set round_columns_enriched = [] %} 10 | {% for col in round_columns %} 11 | {% do round_columns_enriched.append('trunc('+col+', 8)') %} 12 | {% endfor %} 13 | {% set selected_columns = '* exclude(' + round_columns|join(', ') + "), " + round_columns_enriched|join(', ') %} 14 | {% else %} 15 | {% set round_columns_csv = None %} 16 | {% set selected_columns = '*' %} 17 | {% endif %} 18 | 19 | with a as ( 20 | select {{compare_cols_csv}} from {{ model }} 21 | ), 22 | b as ( 23 | select {{compare_cols_csv}} from {{ compare_model }} 24 | ), 25 | a_minus_b as ( 26 | select {{ selected_columns }} from a 27 | {{ except() }} 28 | select {{ selected_columns }} from b 29 | ), 30 | b_minus_a as ( 31 | select {{ selected_columns }} from b 32 | {{ except() }} 33 | select {{ selected_columns }} from a 34 | ), 35 | 36 | unioned as ( 37 | select 'in_actual_not_in_expected' as which_diff, a_minus_b.* from a_minus_b 38 | union all 39 | select 'in_expected_not_in_actual' as which_diff, b_minus_a.* from b_minus_a 40 | ) 41 | select * from unioned 42 | {% endtest %} 43 | -------------------------------------------------------------------------------- /models/warehouses_type2_dimension.sql: -------------------------------------------------------------------------------- 1 | {{ config(materialized='table') }} 2 | 3 | with 4 | stop_threshold as ( 5 | select max(start_time) as timestamp 6 | from {{ ref('stg_query_history') }} 7 | ), 8 | 9 | warehouse_snapshots_base as ( 10 | select 11 | warehouse_id, 12 | warehouse_size, 13 | warehouse_name, 14 | start_time as timestamp, 15 | lag(warehouse_size) over (partition by warehouse_id 16 | order by start_time) as prev_warehouse_size, 17 | lag(warehouse_name) over (partition by warehouse_id 18 | order by start_time) as prev_warehouse_name 19 | from {{ ref('stg_query_history') }} 20 | where 21 | warehouse_size is not null 22 | ), 23 | 24 | warehouse_snapshots as ( 25 | select 26 | warehouse_id, 27 | warehouse_name, 28 | warehouse_size, 29 | timestamp as valid_from, 30 | lead(timestamp) over (partition by warehouse_id 31 | order by timestamp) as _valid_to 32 | from warehouse_snapshots_base 33 | where 34 | warehouse_size != coalesce(prev_warehouse_size, '') 35 | or warehouse_name != coalesce(prev_warehouse_name, '') 36 | ) 37 | 38 | select 39 | warehouse_snapshots.warehouse_id, 40 | warehouse_snapshots.warehouse_name, 41 | warehouse_snapshots.warehouse_size, 42 | warehouse_snapshots.valid_from, 43 | coalesce(warehouse_snapshots._valid_to, stop_threshold.timestamp) as valid_to, 44 | warehouse_snapshots._valid_to is null as is_current 45 | from warehouse_snapshots 46 | cross join stop_threshold 47 | -------------------------------------------------------------------------------- /.github/workflows/main_test_package.yml: -------------------------------------------------------------------------------- 1 | name: Main branch test package 2 | 3 | # triggers for the workflow 4 | on: 5 | workflow_dispatch: 6 | push: 7 | branches: 8 | - main 9 | 10 | env: 11 | # These are configured in GitHub secrets 12 | DBT_PROFILES_DIR: /home/runner/work/${{ github.event.repository.name }}/${{ github.event.repository.name }}/integration_test_project 13 | GITHUB_SHA_OVERRIDE: ${{ github.event.pull_request.head.sha }} # We need the commit hash of the pull request branch's head, the GITHUB_SHA env var is always the base branch in a pull_request_target trigger 14 | DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} 15 | DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }} 16 | DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} 17 | DBT_ENV_SECRET_SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }} 18 | DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }} 19 | DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }} 20 | 21 | jobs: 22 | integration-snowflake: 23 | runs-on: ubuntu-latest 24 | 25 | steps: 26 | - name: Checkout 27 | uses: actions/checkout@v2 28 | 29 | - name: Install tox 30 | run: python3 -m pip install tox 31 | 32 | # TODO: Re-enable this once we add seed data back 33 | # - name: Run Snowflake Integration Tests 34 | # run: tox -e integration_snowflake 35 | 36 | - name: Run Snowflake Tests 37 | run: tox -e snowflake 38 | -------------------------------------------------------------------------------- /models/daily_rates.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: daily_rates 5 | description: > 6 | A copy of snowflake.organization_usage.rate_sheet_daily for the current Snowflake account, with missing dates imputed. 7 | This model accounts for the fact that multiple rates will appear on a given day when the account is in overage. This 8 | happens for on-demand accounts, or fixed contract accounts who have exceeded their remaining balance. When the account 9 | is in overage, the usage_type will appear as 'compute' (instead of 'overage-compute'), but the effective_rate will be 10 | the one associated with `overage-compute`. You can identify these instances using `is_overage_rate` to help understand 11 | why a rate may have suddenly changed. 12 | columns: 13 | - name: date 14 | description: Date (in the UTC time zone) for the effective price. 15 | - name: service_type 16 | description: The type of service, which can be one of compute or storage. 17 | - name: usage_type 18 | description: The type of usage, which can be one of compute, storage, etc. 19 | - name: effective_rate 20 | description: The rate after applying any applicable discounts per the contract for the organization. 21 | - name: currency 22 | description: Currency of effect rate, retrieved from Snowflake's daily rate sheet 23 | - name: is_overage_rate 24 | description: Indicator for whether the effective_rate is an overage rate. 25 | - name: is_latest_rate 26 | description: Indicator for whether the effective_rate is the most recent. 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # [dbt-snowflake-monitoring](https://select.dev/docs/dbt-snowflake-monitoring) 2 | 3 | From the [SELECT](https://select.dev) team, a dbt package to help you monitor Snowflake performance and costs. 4 | 5 | ## Documentation 6 | 7 | Documentation for the package resides on the SELECT website for greater rendering flexibility. For questions and support, please either create an issue or reach out via the Intercom chat bubble on the website 🙂 8 | 9 | * [Setup instructions and example queries](https://select.dev/docs/resources/dbt-snowflake-monitoring) 10 | * [Generated dbt Docs for the package's resources](https://get-select.github.io/dbt-snowflake-monitoring/#!/overview) 11 | 12 | 13 | ## Configuration Variables 14 | 15 | This package supports the following optional variables: 16 | 17 | | Variable | Description | Default | 18 | |----------|-------------|---------| 19 | | `snowflake_account_usage_database` | Database for account_usage schema | `snowflake` | 20 | | `snowflake_account_usage_schema` | Schema for account usage data | `account_usage` | 21 | | `snowflake_organization_usage_database` | Database for organization_usage schema | `snowflake` | 22 | | `snowflake_organization_usage_schema` | Schema for organization usage data | `organization_usage` | 23 | 24 | 25 | Point your project to your custom database with the following defined in your own `dbt_project.yml`: 26 | ```yaml 27 | vars: 28 | snowflake_account_usage_database: "my_custom_db" 29 | snowflake_account_usage_schema: "custom_account_usage" 30 | snowflake_organization_usage_database: "my_custom_db" 31 | snowflake_organization_usage_schema: "custom_organization_usage" 32 | ``` -------------------------------------------------------------------------------- /models/staging/stg_rate_sheet_daily.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: stg_rate_sheet_daily 5 | description: A materialized copy of the snowflake.organization.rate_sheet_daily view. It can be used to get the effective rates used for calculating usage in the organization currency based on credits used for all Snowflake accounts in your organization. It contains 1 row per day per rate in Snowflake account (some days can have more than 1 rate for a given usage_type, usually when the contract is updated). 6 | columns: 7 | - name: date 8 | description: Date (in the UTC time zone) for the effective price. 9 | - name: organization_name 10 | description: Name of the organization. 11 | - name: contract_number 12 | description: Snowflake contract number for the organization. 13 | - name: account_name 14 | description: Name of the account. 15 | - name: account_locator 16 | description: Locator for the account. 17 | - name: region 18 | description: Name of the region where the account is located. 19 | - name: service_level 20 | description: Service level of the Snowflake account (Standard, Enterprise, Business Critical, etc.). 21 | - name: usage_type 22 | description: The type of usage, which can be one of compute, storage, etc. 23 | - name: currency 24 | description: The currency of the effective_rate. 25 | - name: effective_rate 26 | description: The rate after applying any applicable discounts per the contract for the organization. 27 | - name: service_type 28 | description: The type of service, which can be one of compute or storage. 29 | -------------------------------------------------------------------------------- /models/query_base_object_access.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: query_base_object_access 5 | description: This model tracks the base objects accessed by a query. It contains 1 row per query per object. An object can be a regular table owned/created by your account, a table from a data share, a secured view, or an external table. See https://docs.snowflake.com/en/sql-reference/account-usage/access_history.html for more details. Access history is only available for Snowflake accounts on enterprise or higher, and therefore this model will be empty for standard accounts. See https://select.dev/posts/snowflake-unused-tables#direct-versus-base-objects-accessed for more details on the difference between base versus direct objects accessed. 6 | columns: 7 | - name: _unique_id 8 | description: Unique identifier for each row in the table 9 | - name: query_id 10 | description: An internal, system-generated identifier for the SQL statement. 11 | - name: query_start_time 12 | description: The statement start time (UTC time zone). 13 | - name: user_name 14 | description: The user who issued the query. 15 | - name: object_name 16 | description: Fully qualified object name. For a table, this would be .. 17 | - name: object_domain 18 | description: "One of the following: Table, View, Materialized view, External table, Stream, or Stage." 19 | - name: table_id 20 | description: Unique identifier for the table/view. Will be null for secured views and tables from a data share. 21 | - name: columns_accessed 22 | description: Array of column_names accessed in the table 23 | -------------------------------------------------------------------------------- /models/query_direct_object_access.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: query_direct_object_access 5 | description: This model tracks the direct objects accessed by a query. It contains 1 row per query per object. An object can be a regular table owned/created by your account, a table from a data share, a secured view, or an external table. See https://docs.snowflake.com/en/sql-reference/account-usage/access_history.html for more details. Access history is only available for Snowflake accounts on enterprise or higher, and therefore this model will be empty for standard accounts. See https://select.dev/posts/snowflake-unused-tables#direct-versus-base-objects-accessed for more details on the difference between base versus direct objects accessed. 6 | columns: 7 | - name: _unique_id 8 | description: Unique identifier for each row in the table 9 | - name: query_id 10 | description: An internal, system-generated identifier for the SQL statement. 11 | - name: query_start_time 12 | description: The statement start time (UTC time zone). 13 | - name: user_name 14 | description: The user who issued the query. 15 | - name: object_name 16 | description: Fully qualified object name. For a table, this would be .. 17 | - name: object_domain 18 | description: "One of the following: Table, View, Materialized view, External table, Stream, or Stage." 19 | - name: table_id 20 | description: Unique identifier for the table/view. Will be null for secured views and tables from a data share. 21 | - name: columns_accessed 22 | description: Array of column_names accessed in the table 23 | -------------------------------------------------------------------------------- /.github/workflows/ci_test_package.yml: -------------------------------------------------------------------------------- 1 | name: CI test package 2 | 3 | on: 4 | workflow_dispatch: 5 | # all PRs, important to note that `pull_request_target` workflows 6 | # will run in the context of the target branch of a PR 7 | pull_request_target: 8 | 9 | env: 10 | # These are configured in GitHub secrets 11 | DBT_PROFILES_DIR: /home/runner/work/${{ github.event.repository.name }}/${{ github.event.repository.name }}/integration_test_project 12 | GITHUB_SHA_OVERRIDE: ${{ github.event.pull_request.head.sha }} # We need the commit hash of the pull request branch's head, the GITHUB_SHA env var is always the base branch in a pull_request_target trigger 13 | DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} 14 | DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }} 15 | DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} 16 | DBT_ENV_SECRET_SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }} 17 | DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }} 18 | DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }} 19 | 20 | 21 | jobs: 22 | integration-snowflake: 23 | runs-on: ubuntu-latest 24 | environment: 25 | name: Approve Integration Tests 26 | 27 | steps: 28 | - name: Checkout 29 | uses: actions/checkout@v2 30 | with: 31 | ref: ${{ github.event.pull_request.head.sha }} # Check out the code of the PR 32 | 33 | - name: Install tox 34 | run: python3 -m pip install tox 35 | 36 | # TODO: Re-enable this once we add seed data back 37 | # - name: Run Snowflake Integration Tests 38 | # run: tox -e integration_snowflake 39 | 40 | - name: Run Snowflake Tests 41 | run: tox -e snowflake 42 | -------------------------------------------------------------------------------- /models/hourly_spend.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: hourly_spend 5 | description: Table of hourly spend broken down by services as listed on the invoice, with additional granularity. 6 | columns: 7 | - name: hour 8 | description: Hour of spend. 9 | - name: service 10 | description: Top level spend category per Snowflake invoice. One of Storage, Compute, Cloud Services, Adj For Incl Cloud Services, Automatic Clustering, Materialized Views, Snowpipe, Query Acceleration, Replication, or Search Optimization. 11 | - name: storage_type 12 | description: Subcategories where service = "Storage". 13 | - name: warehouse_name 14 | description: Subcategories where service = "Compute" or "Cloud Services". 15 | - name: database_name 16 | description: Subcategories where service = "Serverless Tasks" or service = "Storage" and storage_type = "Table and Time Travel" or "Failsafe". 17 | - name: usage 18 | description: Usage of the service in the units described by the 'usage_unit' column (i.e. TB, Credits, etc.). 19 | - name: usage_net_cloud_services 20 | description: Usage with cloud service adjustments taken into account. The service 'Adj For Incl Cloud Services' has a value of 0 for this column. 21 | - name: spend 22 | description: Spend in the currency described by the 'currency' column 23 | - name: spend_net_cloud_services 24 | description: Spend with cloud service adjustments taken into account. The service 'Adj For Incl Cloud Services' has a value of 0 for this column. 25 | - name: currency 26 | description: Spend currency, retrieved from Snowflake's daily rate sheet 27 | - name: usage_unit 28 | description: The units of the usage column (i.e. TB, Credits, etc.). 29 | - name: usage_rate 30 | description: Usage rate, retrieved from the rate sheet, in units per currency. 31 | -------------------------------------------------------------------------------- /models/query_base_object_access.sql: -------------------------------------------------------------------------------- 1 | {{ config( 2 | materialized='incremental', 3 | unique_key=['_unique_id', 'query_start_time'], 4 | ) }} 5 | 6 | with 7 | access_history as ( 8 | select * 9 | from {{ ref('stg_access_history') }} 10 | 11 | {% if is_incremental() %} 12 | where query_start_time > (select coalesce(dateadd('day', -1, max(query_start_time)), '1970-01-01') from {{ this }}) 13 | {% endif %} 14 | 15 | ), 16 | 17 | access_history_flattened as ( 18 | select 19 | access_history.query_id, 20 | access_history.query_start_time, 21 | access_history.user_name, 22 | objects_accessed.value:objectId::integer as table_id, -- will be null for secured views or tables from a data share 23 | objects_accessed.value:objectName::text as object_name, 24 | objects_accessed.value:objectDomain::text as object_domain, 25 | objects_accessed.value:columns as columns_array 26 | 27 | from access_history, lateral flatten(access_history.base_objects_accessed) as objects_accessed 28 | ), 29 | 30 | access_history_flattened_w_columns as ( 31 | select 32 | access_history_flattened.query_id, 33 | access_history_flattened.query_start_time, 34 | access_history_flattened.user_name, 35 | access_history_flattened.table_id, 36 | access_history_flattened.object_name, 37 | access_history_flattened.object_domain, 38 | array_agg(distinct columns.value:columnName::text) as columns_accessed 39 | from access_history_flattened, lateral flatten(access_history_flattened.columns_array) as columns 40 | where 41 | access_history_flattened.object_name is not null 42 | group by 1, 2, 3, 4, 5, 6 43 | ) 44 | 45 | select 46 | md5(concat(query_id, object_name)) as _unique_id, 47 | * 48 | from access_history_flattened_w_columns 49 | order by query_start_time asc 50 | -------------------------------------------------------------------------------- /models/staging/stg_remaining_balance_daily.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: stg_remaining_balance_daily 5 | description: A materialized copy of the snowflake.organization.rate_sheet_daily view. It can be used to get the effective rates used for calculating usage in the organization currency based on credits used for all Snowflake accounts in your organization. It contains 1 row per day per rate in Snowflake account (some days can have more than 1 rate for a given usage_type, usually when the contract is updated). 6 | columns: 7 | - name: date 8 | description: The date of the FREE_USAGE_BALANCE or CAPACITY_BALANCE in the UTC time zone. 9 | - name: organization_name 10 | description: Name of the organization. 11 | - name: contract_number 12 | description: Contract number for the organization. 13 | - name: currency 14 | description: The currency of the FREE_USAGE_BALANCE or CAPACITY_BALANCE or ON_DEMAND_CONSUMPTION_BALANCE. 15 | - name: free_usage_balance 16 | description: The amount of free usage in currency that is available for use as of the date. This is the end of day balance. 17 | - name: capacity_balance 18 | description: The amount of capacity in currency that is available for use as of the date. This is the end of day balance. 19 | - name: on_demand_consumption_balance 20 | description: The amount of consumption at on demand prices that will be invoiced given that all the free usage and capacity balances have been exhausted. This is a negative value (e.g. -250) until the invoice is paid. This is the end of day balance. 21 | - name: rollover_balance 22 | description: The amount of rollover balance in currency that is available for use at the end of the date. At the end of a contract term, it is calculated as sum(AMOUNT) from the CONTRACT_ITEMS view - sum(USAGE_IN_CURRENCY) from the USAGE_IN_CURRENCY_DAILY view. 23 | -------------------------------------------------------------------------------- /models/daily_spend.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: daily_spend 5 | description: Table of daily spend broken down by services as listed on the invoice, with additional granularity. 6 | columns: 7 | - name: date 8 | description: Date of spend in UTC timezone to match with Snowflake's billing timezone. 9 | - name: service 10 | description: Top level spend category per Snowflake invoice. One of Storage, Compute, Cloud Services, Adj For Incl Cloud Services, Automatic Clustering, Materialized Views, Snowpipe, Query Acceleration, Replication, or Search Optimization. 11 | - name: storage_type 12 | description: Subcategories where service = "Storage". 13 | - name: warehouse_name 14 | description: Subcategories where service = "Compute" or "Cloud Services". 15 | - name: database_name 16 | description: Subcategories where service = "Serverless Tasks" or service = "Storage" and storage_type = "Table and Time Travel" or "Failsafe". 17 | - name: usage 18 | description: Usage of the service in the units described by the 'usage_unit' column (i.e. TB, Credits, etc.). 19 | - name: usage_net_cloud_services 20 | description: Usage with cloud service adjustments taken into account. The service 'Adj For Incl Cloud Services' has a value of 0 for this column. 21 | - name: spend 22 | description: Spend in the currency described by the 'currency' column 23 | - name: spend_net_cloud_services 24 | description: Spend with cloud service adjustments taken into account. The service 'Adj For Incl Cloud Services' has a value of 0 for this column. 25 | - name: currency 26 | description: Spend currency, retrieved from Snowflake's daily rate sheet 27 | - name: usage_unit 28 | description: The units of the usage column (i.e. TB, Credits, etc.). 29 | - name: usage_rate 30 | description: Usage rate, retrieved from the rate sheet, in units per currency. 31 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | skipsdist = True 3 | envlist = integration_snowflake 4 | 5 | [sqlfluff] 6 | exclude_rules = LT05, ST06, RF04, AM06, ST05, LT02 7 | dialect = snowflake 8 | templater = dbt 9 | deps = 10 | sqlfluff-templater-dbt==3.* 11 | dbt-snowflake~=1.9.0 12 | 13 | [testenv] 14 | passenv = 15 | DBT_PROFILES_DIR 16 | GITHUB_SHA 17 | GITHUB_SHA_OVERRIDE 18 | DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT 19 | DBT_ENV_SECRET_SNOWFLAKE_TEST_USER 20 | DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD 21 | DBT_ENV_SECRET_SNOWFLAKE_TEST_ROLE 22 | DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE 23 | DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE 24 | 25 | [testenv:lint] 26 | deps = {[sqlfluff]deps} 27 | commands = sqlfluff lint {posargs} --ignore parsing 28 | 29 | [testenv:lint_all] 30 | deps = {[sqlfluff]deps} 31 | commands = 32 | dbt deps 33 | sqlfluff lint models --ignore parsing 34 | 35 | [testenv:fix] 36 | deps = {[sqlfluff]deps} 37 | commands = sqlfluff fix {posargs} --ignore parsing -f 38 | 39 | [testenv:fix_all] 40 | deps = {[sqlfluff]deps} 41 | commands = sqlfluff fix models --ignore parsing -f 42 | 43 | [testenv:generate_docs] 44 | deps = dbt-snowflake~=1.9.0 45 | commands = dbt docs generate 46 | 47 | [testenv:integration_snowflake] 48 | ; This test env uses a test dataset and asserted expected output 49 | changedir = integration_test_project 50 | setenv = 51 | SOURCE_DATABASE_OVERRIDE = DBT_SNOWFLAKE_MONITORING_CI 52 | SOURCE_SCHEMA_OVERRIDE = MOCK_DATA 53 | deps = dbt-snowflake~=1.9.0 54 | commands = 55 | dbt deps 56 | dbt build -s +daily_spend --full-refresh --vars 'account_name: a09e1' 57 | dbt build --exclude dbt_snowflake_monitoring --full-refresh --vars 'account_name: a09e1' 58 | dbt build -s +daily_spend --vars 'account_name: a09e1' 59 | dbt build --exclude dbt_snowflake_monitoring --vars 'account_name: a09e1' 60 | 61 | [testenv:snowflake] 62 | ; This test env just runs the package as it would normally be installed, using the snowflake database as a source 63 | changedir = integration_test_project 64 | deps = dbt-snowflake~=1.9.0 65 | commands = 66 | dbt deps 67 | dbt build -s dbt_snowflake_monitoring --full-refresh --empty 68 | dbt build -s dbt_snowflake_monitoring --empty 69 | -------------------------------------------------------------------------------- /.github/workflows/publish_docs.yml: -------------------------------------------------------------------------------- 1 | name: Publish dbt Documentation on Release 2 | 3 | # triggers for the workflow 4 | on: 5 | workflow_dispatch: 6 | release: 7 | types: [published] 8 | 9 | env: 10 | # These are configured in GitHub secrets 11 | DBT_PROFILES_DIR: /home/runner/work/${{ github.event.repository.name }}/${{ github.event.repository.name }}/integration_test_project 12 | GITHUB_SHA_OVERRIDE: ${{ github.event.pull_request.head.sha }} # We need the commit hash of the pull request branch's head, the GITHUB_SHA env var is always the base branch in a pull_request_target trigger 13 | DBT_ENV_SECRET_SNOWFLAKE_TEST_ACCOUNT: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT }} 14 | DBT_ENV_SECRET_SNOWFLAKE_TEST_USER: ${{ secrets.SNOWFLAKE_TEST_USER }} 15 | DBT_ENV_SECRET_SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} 16 | DBT_ENV_SECRET_SNOWFLAKE_TEST_ROLE: ${{ secrets.SNOWFLAKE_TEST_ROLE }} 17 | DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }} 18 | DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }} 19 | 20 | jobs: 21 | generate-dbt-docs: 22 | name: Generate dbt docs 23 | runs-on: ubuntu-latest 24 | 25 | # Grant GITHUB_TOKEN the permissions required to make a Pages deployment 26 | permissions: 27 | pages: write # to deploy to Pages 28 | id-token: write # to verify the deployment originates from an appropriate source 29 | 30 | environment: 31 | name: github-pages 32 | url: ${{ steps.deployment.outputs.page_url }} 33 | 34 | steps: 35 | - name: Checkout Branch 36 | uses: actions/checkout@v3 37 | with: 38 | ref: ${{ github.event.release.tag_name }} 39 | 40 | - name: Install Python packages 41 | run: python -m pip install dbt-snowflake~=1.9.0 42 | 43 | - name: Test database connection 44 | run: dbt debug 45 | 46 | - name: Install dbt packages 47 | run: dbt deps 48 | 49 | - name: Generate dbt docs 50 | run: dbt docs generate 51 | 52 | - name: Setup Pages 53 | uses: actions/configure-pages@v2 54 | 55 | - name: Upload artifact 56 | uses: actions/upload-pages-artifact@v1 57 | with: 58 | path: 'target' 59 | 60 | - name: Deploy to GitHub Pages 61 | id: deployment 62 | uses: actions/deploy-pages@v1 63 | -------------------------------------------------------------------------------- /models/cost_per_query.yml: -------------------------------------------------------------------------------- 1 | 2 | version: 2 3 | 4 | models: 5 | - name: cost_per_query 6 | description: A model to help you identify expensive Snowflake queries. It includes 1 row per query, along with the estimated query cost. 7 | columns: 8 | - name: query_id 9 | description: Primary key. Internal/system-generated identifier for the SQL statement. 10 | tests: 11 | - unique 12 | - not_null 13 | - name: start_time 14 | description: Query statement start time (in the UTC time zone). The table is naturally clustered on this column, meaning your queries will run much faster if you filter records using this column. 15 | - name: end_time 16 | description: Query statement end time (in the UTC time zone). 17 | - name: execution_start_time 18 | description: When the query began executing on the warehouse (in the UTC time zone). This will always be after the start_time. 19 | - name: compute_cost 20 | description: Compute costs associated with the query, in the primary currency of your account. Can be 0 if the query did not run on a warehouse. 21 | - name: compute_credits 22 | description: Compute credits associated with the query. Can be 0 if the query did not run on a warehouse. 23 | - name: cloud_services_cost 24 | description: Cloud service costs associated with the query, in the primary currency of your account. Can be 0 if total cloud services credits consumption was less than 10% of total compute credits consumption on that day. 25 | - name: cloud_services_credits 26 | description: Cloud service credits associated with the query. Can be 0 if total cloud services credits consumption was less than 10% of total compute credits consumption on that day. 27 | - name: query_cost 28 | description: Total cost associated with the query, calculated as sum of compute_cost and cloud_services_cost, in the primary currency of your account. 29 | - name: query_credits 30 | description: Total credits associated with the query, calculated as sum of compute_credits and cloud_services_credits. 31 | - name: ran_on_warehouse 32 | description: Indicator for whether the query ran on a warehouse. Certain queries, such as metadata queries, can be entirely processed in cloud services. 33 | - name: currency 34 | description: Spend currency, retrieved from Snowflake's daily rate sheet 35 | -------------------------------------------------------------------------------- /models/query_direct_object_access.sql: -------------------------------------------------------------------------------- 1 | {{ 2 | config( 3 | materialized="incremental", 4 | unique_key=["_unique_id", "query_start_time"], 5 | ) 6 | }} 7 | 8 | with 9 | access_history as ( 10 | select * 11 | from {{ ref("stg_access_history") }} 12 | 13 | {% if is_incremental() %} 14 | where 15 | query_start_time > ( 16 | select 17 | coalesce( 18 | dateadd('day', -1, max(query_start_time)), '1970-01-01' 19 | ) 20 | from {{ this }} 21 | ) 22 | {% endif %} 23 | 24 | ), 25 | 26 | access_history_flattened as ( 27 | select 28 | access_history.query_id, 29 | access_history.query_start_time, 30 | access_history.user_name, 31 | objects_accessed.value:"objectId"::integer as table_id, -- will be null for secured views or tables from a data share 32 | objects_accessed.value:"objectName"::text as object_name, 33 | objects_accessed.value:"objectDomain"::text as object_domain, 34 | objects_accessed.value:columns as columns_array 35 | 36 | from 37 | access_history, 38 | lateral flatten(access_history.direct_objects_accessed) as objects_accessed 39 | ), 40 | 41 | access_history_flattened_w_columns as ( 42 | select 43 | access_history_flattened.query_id, 44 | access_history_flattened.query_start_time, 45 | access_history_flattened.user_name, 46 | access_history_flattened.table_id, 47 | access_history_flattened.object_name, 48 | access_history_flattened.object_domain, 49 | array_agg(distinct columns.value:"columnName"::text) as columns_accessed 50 | from 51 | access_history_flattened, 52 | lateral flatten(access_history_flattened.columns_array) as columns 53 | where access_history_flattened.object_name is not null 54 | group by 1, 2, 3, 4, 5, 6 55 | ) 56 | 57 | select 58 | md5(concat(query_id, object_name)) as _unique_id, 59 | * 60 | from access_history_flattened_w_columns 61 | qualify -- added by affinaquest to ensure uniqueness 62 | row_number() over ( 63 | partition by md5(concat(query_id, object_name)), query_start_time 64 | order by query_start_time asc 65 | ) 66 | = 1 67 | order by query_start_time asc 68 | -------------------------------------------------------------------------------- /.changie.yaml: -------------------------------------------------------------------------------- 1 | # Mainly from https://github.com/dbt-labs/dbt-core/blob/main/.changie.yaml 2 | 3 | changesDir: .changes 4 | unreleasedDir: unreleased 5 | headerPath: header.tpl.md 6 | changelogPath: CHANGELOG.md 7 | versionExt: md 8 | versionFormat: '## dbt-snowflake-monitoring {{.Version}} - {{.Time.Format "January 02, 2006"}}' 9 | kindFormat: '### {{.Kind}}' 10 | changeFormat: '- {{.Body}} ([#{{.Custom.PR}}](https://github.com/get-select/dbt-snowflake-monitoring/pull/{{.Custom.PR}}))' 11 | kinds: 12 | - label: Breaking Changes 13 | - label: Features 14 | - label: Fixes 15 | - label: Docs 16 | newlines: 17 | afterChangelogHeader: 1 18 | afterKind: 1 19 | afterChangelogVersion: 1 20 | beforeKind: 1 21 | endOfVersion: 1 22 | custom: 23 | - key: Author 24 | label: GitHub Username(s) (separated by a single space if multiple) 25 | type: string 26 | minLength: 3 27 | - key: PR 28 | label: GitHub Pull Request Number 29 | type: int 30 | minInt: 1 31 | 32 | footerFormat: | 33 | {{- $contributorDict := dict }} 34 | {{- /* any names added to this list should be all lowercase for later matching purposes */}} 35 | {{- $core_team := list "niallrees" "ian-whitestone" }} 36 | {{- range $change := .Changes }} 37 | {{- $authorList := splitList " " $change.Custom.Author }} 38 | {{- /* loop through all authors for a PR */}} 39 | {{- range $author := $authorList }} 40 | {{- $authorLower := lower $author }} 41 | {{- /* we only want to include non-core team contributors */}} 42 | {{- if not (has $authorLower $core_team)}} 43 | {{- /* Docs kind link back to dbt-docs instead of dbt-core PRs */}} 44 | {{- $prLink := $change.Kind }} 45 | {{- /* check if this contributor has other PRs associated with them already */}} 46 | {{- if hasKey $contributorDict $author }} 47 | {{- $prList := get $contributorDict $author }} 48 | {{- $prList = append $prList $prLink }} 49 | {{- $contributorDict := set $contributorDict $author $prList }} 50 | {{- else }} 51 | {{- $prList := list $prLink }} 52 | {{- $contributorDict := set $contributorDict $author $prList }} 53 | {{- end }} 54 | {{- end}} 55 | {{- end}} 56 | {{- end }} 57 | {{- /* no indentation here for formatting so the final markdown doesn't have unneeded indentations */}} 58 | {{- if $contributorDict}} 59 | ### Contributors 60 | {{- range $k,$v := $contributorDict }} 61 | - [@{{$k}}](https://github.com/{{$k}}) ({{ range $index, $element := $v }}{{if $index}}, {{end}}{{$element}}{{end}}) 62 | {{- end }} 63 | {{- end }} 64 | -------------------------------------------------------------------------------- /models/staging/stg_query_history.sql: -------------------------------------------------------------------------------- 1 | {{ config( 2 | materialized='incremental', 3 | unique_key=['query_id', 'start_time'], 4 | ) }} 5 | 6 | select 7 | query_id, 8 | query_text, 9 | database_id, 10 | database_name, 11 | schema_id, 12 | schema_name, 13 | query_type, 14 | session_id, 15 | user_name, 16 | role_name, 17 | warehouse_id, 18 | warehouse_name, 19 | warehouse_size, 20 | warehouse_type, 21 | cluster_number, 22 | query_tag, 23 | execution_status, 24 | error_code, 25 | error_message, 26 | start_time, 27 | end_time, 28 | total_elapsed_time, 29 | bytes_scanned, 30 | percentage_scanned_from_cache, 31 | bytes_written, 32 | bytes_written_to_result, 33 | bytes_read_from_result, 34 | rows_produced, 35 | rows_inserted, 36 | rows_updated, 37 | rows_deleted, 38 | rows_unloaded, 39 | bytes_deleted, 40 | partitions_scanned, 41 | partitions_total, 42 | bytes_spilled_to_local_storage, 43 | bytes_spilled_to_remote_storage, 44 | bytes_sent_over_the_network, 45 | compilation_time, 46 | execution_time, 47 | queued_provisioning_time, 48 | queued_repair_time, 49 | queued_overload_time, 50 | transaction_blocked_time, 51 | outbound_data_transfer_cloud, 52 | outbound_data_transfer_region, 53 | outbound_data_transfer_bytes, 54 | inbound_data_transfer_cloud, 55 | inbound_data_transfer_region, 56 | inbound_data_transfer_bytes, 57 | list_external_files_time, 58 | credits_used_cloud_services, 59 | release_version, 60 | external_function_total_invocations, 61 | external_function_total_sent_rows, 62 | external_function_total_received_rows, 63 | external_function_total_sent_bytes, 64 | external_function_total_received_bytes, 65 | query_load_percent, 66 | is_client_generated_statement, 67 | query_acceleration_bytes_scanned, 68 | query_acceleration_partitions_scanned, 69 | query_acceleration_upper_limit_scale_factor, 70 | query_hash, 71 | query_hash_version, 72 | query_parameterized_hash, 73 | query_parameterized_hash_version, 74 | query_retry_time, 75 | query_retry_cause, 76 | fault_handling_time 77 | from {{ source('snowflake_account_usage', 'query_history') }} 78 | 79 | {% if is_incremental() %} 80 | -- must use end time in case query hasn't completed 81 | -- add lookback window of 2 days to account for late arriving queries 82 | where end_time > (select dateadd(day, -{{ var('dbt_snowflake_monitoring_incremental_days', '2') }}, coalesce(max(end_time), '1970-01-01') ) from {{ this }}) 83 | {% endif %} 84 | 85 | qualify row_number() over (partition by query_id order by start_time ) = 1 86 | 87 | order by start_time 88 | -------------------------------------------------------------------------------- /models/dbt_queries.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: dbt_queries 5 | description: Filtered version of query_history_enriched just for queries issued by dbt. Adds additional dbt-specific columns. Incrementally pulls the last 3 days by default to account for late arriving rates data. This can be overriden by passing a value for dbt_snowflake_monitoring_incremental_days. 6 | columns: 7 | - name: dbt_snowflake_query_tags_version 8 | description: Version of the dbt-snowflake-query-tags package that generated the metadata 9 | - name: dbt_invocation_id 10 | description: The id of the dbt invocation. 11 | - name: dbt_node_id 12 | description: The identifier for the node that the query relates to. 13 | - name: dbt_node_resource_type 14 | description: The resource type of the node that the query relates to. 15 | - name: dbt_node_name 16 | description: The name of the node that the query relates to. 17 | - name: dbt_node_materialized 18 | description: The materialization of the node that the query relates to. 19 | - name: dbt_node_is_incremental 20 | description: Boolean describing if the node run was incremental. 21 | - name: dbt_node_alias 22 | description: Alias set for the node. 23 | - name: dbt_node_meta 24 | description: Dict of any meta set for the node. 25 | - name: dbt_node_tags 26 | description: Array of all tags set for the node. 27 | - name: dbt_node_refs 28 | description: Array of all refs used by the node. 29 | - name: dbt_node_database 30 | description: The database configured for the node. 31 | - name: dbt_node_schema 32 | description: The schema configured for the node. 33 | - name: dbt_version 34 | description: Version of dbt in use. 35 | - name: dbt_project_name 36 | description: Name of the dbt project. 37 | - name: dbt_target_name 38 | description: The target name for the dbt invocation. 39 | - name: dbt_target_database 40 | description: The target database for the dbt invocation. 41 | - name: dbt_target_schema 42 | description: The target schema for the dbt invocation. 43 | - name: dbt_node_package_name 44 | description: The package name of the dbt node. 45 | - name: dbt_node_original_file_path 46 | description: The file path of the dbt node. 47 | - name: dbt_cloud_project_id 48 | description: If using dbt Cloud, the ID of the project. 49 | - name: dbt_cloud_job_id 50 | description: If using dbt Cloud, the ID of the issuing job. 51 | - name: dbt_cloud_run_id 52 | description: If using dbt Cloud, the ID of the issuing run. 53 | - name: dbt_cloud_run_reason_category 54 | description: If using dbt Cloud, the run reason category for the issuing run. 55 | - name: dbt_cloud_run_reason 56 | description: If using dbt Cloud, the run reason for the issuing run. 57 | - name: dbt_cloud_job_url 58 | description: If using dbt Cloud, the URL of the issuing job. The dbt_cloud_account_id dbt variable must be set for this field to populate. 59 | - name: dbt_cloud_run_url 60 | description: If using dbt Cloud, the URL of the issuing run. The dbt_cloud_account_id dbt variable must be set for this field to populate. 61 | -------------------------------------------------------------------------------- /models/dbt_queries.sql: -------------------------------------------------------------------------------- 1 | {{ config( 2 | materialized='incremental', 3 | unique_key=['query_id', 'start_time'] 4 | ) }} 5 | 6 | select 7 | dbt_metadata['dbt_snowflake_query_tags_version']::string as dbt_snowflake_query_tags_version, -- this will be null where the metadata came from a query comment in dbt-snowflake-monitoring versions <2.0.0 8 | dbt_metadata['invocation_id']::string as dbt_invocation_id, 9 | dbt_metadata['node_id']::string as dbt_node_id, 10 | dbt_metadata['node_resource_type']::string as dbt_node_resource_type, 11 | coalesce(dbt_metadata['node_name']::string, replace(array_slice(split(dbt_node_id, '.'), -1, array_size(split(dbt_node_id, '.')))[0], '"')) as dbt_node_name, -- we can just use node_name once enough time has been that users have migrated to v2.0.0 12 | dbt_metadata['materialized']::string as dbt_node_materialized, 13 | dbt_metadata['is_incremental']::boolean as dbt_node_is_incremental, 14 | dbt_metadata['node_alias']::string as dbt_node_alias, 15 | dbt_metadata['node_meta']::variant as dbt_node_meta, 16 | dbt_metadata['node_tags']::array as node_tags, 17 | iff(dbt_snowflake_query_tags_version >= '1.1.3', dbt_metadata['node_refs']::array, []) as dbt_node_refs, -- correct refs available from 1.1.3 onwards 18 | dbt_metadata['node_database']::string as dbt_node_database, 19 | dbt_metadata['node_schema']::string as dbt_node_schema, 20 | dbt_metadata['dbt_version']::string as dbt_version, 21 | dbt_metadata['project_name']::string as dbt_project_name, 22 | dbt_metadata['target_name']::string as dbt_target_name, 23 | dbt_metadata['target_database']::string as dbt_target_database, 24 | dbt_metadata['target_schema']::string as dbt_target_schema, 25 | dbt_metadata['node_package_name']::string as dbt_node_package_name, 26 | dbt_metadata['node_original_file_path']::string as dbt_node_original_file_path, 27 | dbt_metadata['dbt_cloud_project_id']::string as dbt_cloud_project_id, 28 | dbt_metadata['dbt_cloud_job_id']::string as dbt_cloud_job_id, 29 | dbt_metadata['dbt_cloud_run_id']::string as dbt_cloud_run_id, 30 | dbt_metadata['dbt_cloud_run_reason_category']::string as dbt_cloud_run_reason_category, 31 | dbt_metadata['dbt_cloud_run_reason']::string as dbt_cloud_run_reason, 32 | case 33 | when dbt_cloud_project_id is not null 34 | then 35 | {% if var('dbt_cloud_account_id', none) -%} 36 | '{{ var('dbt_cloud_url', 'https://cloud.getdbt.com/deploy/') }}' || '{{ var('dbt_cloud_account_id') }}' || '/projects/' || dbt_cloud_project_id || '/jobs/' || dbt_cloud_job_id 37 | {%- else -%} 38 | 'Required dbt_cloud_account_id variable not set' -- noqa 39 | {%- endif %} 40 | end as dbt_cloud_job_url, 41 | case 42 | when dbt_cloud_project_id is not null 43 | then 44 | {% if var('dbt_cloud_account_id', none) -%} 45 | '{{ var('dbt_cloud_url', 'https://cloud.getdbt.com/deploy/') }}' || '{{ var('dbt_cloud_account_id') }}' || '/projects/' || dbt_cloud_project_id || '/runs/' || dbt_cloud_run_id 46 | {%- else -%} 47 | 'Required dbt_cloud_account_id variable not set' -- noqa 48 | {%- endif %} 49 | end as dbt_cloud_run_url, 50 | * exclude dbt_metadata 51 | from {{ ref('query_history_enriched') }} 52 | where dbt_metadata is not null 53 | {% if is_incremental() %} 54 | -- Conservatively re-process the last 3 days to account for late arriving rates data which changes the cost per query. 55 | -- Allow an override from project variable 56 | and end_time > (select coalesce(dateadd(day, -{{ var('dbt_snowflake_monitoring_incremental_days', '3') }}, max(end_time)), '1970-01-01') from {{ this }}) 57 | {% endif %} 58 | -------------------------------------------------------------------------------- /models/daily_rates.sql: -------------------------------------------------------------------------------- 1 | {{ config(materialized='table') }} 2 | 3 | /* 4 | snowflake.organization_usage.rate_sheet_daily isn't guaranteed to have 1 row per day per usage type. 5 | 6 | If you don't consume any compute resources on a given day, there won't be a record. 7 | 8 | This model guarantees 1 row per day per usage type, by filling in missing values with rates from the last 9 | known day. 10 | */ 11 | 12 | with 13 | dates_base as ( 14 | select date_day as date from ( 15 | {{ dbt_utils.date_spine( 16 | datepart="day", 17 | start_date="'2018-01-01'", 18 | end_date="dateadd(day, 1, current_date)" 19 | ) 20 | }} 21 | ) 22 | ), 23 | 24 | rate_sheet_daily_base as ( 25 | select 26 | date, 27 | usage_type, 28 | currency, 29 | effective_rate, 30 | service_type 31 | from {{ ref('stg_rate_sheet_daily') }} 32 | where 33 | account_name = {{ account_name() }} 34 | ), 35 | 36 | stop_thresholds as ( 37 | select min(date) as start_date 38 | from rate_sheet_daily_base 39 | 40 | union all 41 | 42 | select min(date) as start_date 43 | from {{ ref('remaining_balance_daily_without_contract_view') }} 44 | ), 45 | 46 | date_range as ( 47 | select 48 | max(start_date) as start_date, 49 | current_date as end_date 50 | from stop_thresholds 51 | ), 52 | 53 | remaining_balance_daily as ( 54 | select 55 | date, 56 | free_usage_balance + capacity_balance + on_demand_consumption_balance + rollover_balance as remaining_balance, 57 | remaining_balance < 0 as is_account_in_overage 58 | from {{ ref('remaining_balance_daily_without_contract_view') }} 59 | ), 60 | 61 | latest_remaining_balance_daily as ( 62 | select 63 | date, 64 | remaining_balance, 65 | is_account_in_overage 66 | from remaining_balance_daily 67 | qualify row_number() over ( 68 | order by date desc) = 1 69 | ), 70 | 71 | rate_sheet_daily as ( 72 | select rate_sheet_daily_base.* 73 | from rate_sheet_daily_base 74 | inner join date_range 75 | on rate_sheet_daily_base.date between date_range.start_date and date_range.end_date 76 | ), 77 | 78 | rates_date_range_w_usage_types as ( 79 | select 80 | date_range.start_date, 81 | date_range.end_date, 82 | usage_types.usage_type 83 | from date_range 84 | cross join (select distinct rate_sheet_daily.usage_type from rate_sheet_daily) as usage_types 85 | ), 86 | 87 | base as ( 88 | select 89 | db.date, 90 | dr.usage_type 91 | from dates_base as db 92 | inner join rates_date_range_w_usage_types as dr 93 | on db.date between dr.start_date and dr.end_date 94 | ), 95 | 96 | rates_w_overage as ( 97 | select 98 | base.date, 99 | base.usage_type, 100 | coalesce( 101 | rate_sheet_daily.service_type, 102 | lag(rate_sheet_daily.service_type) ignore nulls over (partition by base.usage_type 103 | order by base.date), 104 | lead(rate_sheet_daily.service_type) ignore nulls over (partition by base.usage_type 105 | order by base.date) 106 | ) as service_type, 107 | coalesce( 108 | rate_sheet_daily.effective_rate, 109 | lag(rate_sheet_daily.effective_rate) ignore nulls over (partition by base.usage_type 110 | order by base.date), 111 | lead(rate_sheet_daily.effective_rate) ignore nulls over (partition by base.usage_type 112 | order by base.date) 113 | ) as effective_rate, 114 | coalesce( 115 | rate_sheet_daily.currency, 116 | lag(rate_sheet_daily.currency) ignore nulls over (partition by base.usage_type 117 | order by base.date), 118 | lead(rate_sheet_daily.currency) ignore nulls over (partition by base.usage_type 119 | order by base.date) 120 | ) as currency, 121 | base.usage_type like 'overage-%' as is_overage_rate, 122 | replace(base.usage_type, 'overage-', '') as associated_usage_type, 123 | coalesce(remaining_balance_daily.is_account_in_overage, latest_remaining_balance_daily.is_account_in_overage, false) as _is_account_in_overage, 124 | case 125 | when _is_account_in_overage and is_overage_rate then 1 126 | when not _is_account_in_overage and not is_overage_rate then 1 127 | else 0 128 | end as rate_priority 129 | 130 | from base 131 | left join latest_remaining_balance_daily on latest_remaining_balance_daily.date is not null 132 | left join remaining_balance_daily 133 | on base.date = remaining_balance_daily.date 134 | left join rate_sheet_daily 135 | on base.date = rate_sheet_daily.date 136 | and base.usage_type = rate_sheet_daily.usage_type 137 | ), 138 | 139 | rates as ( 140 | select 141 | date, 142 | usage_type, 143 | associated_usage_type, 144 | service_type, 145 | effective_rate, 146 | currency, 147 | is_overage_rate 148 | from rates_w_overage 149 | qualify row_number() over (partition by date, service_type, associated_usage_type 150 | order by rate_priority desc) = 1 151 | ) 152 | 153 | select 154 | date, 155 | associated_usage_type as usage_type, 156 | service_type, 157 | effective_rate, 158 | currency, 159 | is_overage_rate, 160 | row_number() over (partition by service_type, associated_usage_type 161 | order by date desc) = 1 as is_latest_rate 162 | from rates 163 | order by date 164 | -------------------------------------------------------------------------------- /models/query_history_enriched.sql: -------------------------------------------------------------------------------- 1 | {{ config( 2 | static_analysis="off", 3 | materialized='incremental', 4 | unique_key=['query_id', 'start_time'], 5 | pre_hook=["{{ create_merge_objects_udf(this) }}"] 6 | ) }} 7 | 8 | with 9 | query_history as ( 10 | select 11 | *, 12 | 13 | -- this removes comments enclosed by /* */ and single line comments starting with -- and either ending with a new line or end of string 14 | regexp_replace(query_text, $$(\/\*(.|\n|\r)*?\*\/)|(--.*$)|(--.*(\n|\r))|;$$, '') as query_text_no_comments, 15 | 16 | try_parse_json(regexp_substr(query_text, $$\/\*\s*({[^*]*?"app":\s*"dbt"[^*]*})\s*\*\/$$, 1, 1, 'ie')) as _dbt_json_comment_meta, 17 | case 18 | when try_parse_json(query_tag)['dbt_snowflake_query_tags_version'] is not null then try_parse_json(query_tag) 19 | end as _dbt_json_query_tag_meta, 20 | case 21 | when _dbt_json_comment_meta is not null or _dbt_json_query_tag_meta is not null then 22 | {{ adapter.quote_as_configured(this.database, 'database') }}.{{ adapter.quote_as_configured(this.schema, 'schema') }}.merge_objects(coalesce(_dbt_json_comment_meta, { }), coalesce(_dbt_json_query_tag_meta, { })) 23 | end as dbt_metadata 24 | 25 | from {{ ref('stg_query_history') }} 26 | 27 | {% if is_incremental() %} 28 | -- Conservatively re-process the last 3 days to account for late arriving rates data. Allow an override from project variable 29 | -- which changes the cost per query 30 | where end_time > (select dateadd(day, -{{ var('dbt_snowflake_monitoring_incremental_days', '3') }}, max(end_time)) from {{ this }}) 31 | {% endif %} 32 | ), 33 | 34 | cost_per_query as ( 35 | select * 36 | from {{ ref('cost_per_query') }} 37 | {% if is_incremental() %} 38 | -- Conservatively re-process the last 3 days to account for late arriving rates data. Allow an override from project variable 39 | -- which changes the cost per query 40 | where end_time > (select dateadd(day, -{{ var('dbt_snowflake_monitoring_incremental_days', '3') }}, max(end_time)) from {{ this }}) 41 | {% endif %} 42 | ) 43 | 44 | select 45 | cost_per_query.query_id, 46 | cost_per_query.compute_cost, 47 | cost_per_query.compute_credits, 48 | cost_per_query.query_acceleration_cost, 49 | cost_per_query.query_acceleration_credits, 50 | cost_per_query.cloud_services_cost, 51 | cost_per_query.cloud_services_credits, 52 | cost_per_query.query_cost, 53 | cost_per_query.query_credits, 54 | cost_per_query.execution_start_time, 55 | 56 | -- Grab all columns from query_history (except the query time columns which we rename below) 57 | query_history.query_text, 58 | query_history.database_id, 59 | query_history.database_name, 60 | query_history.schema_id, 61 | query_history.schema_name, 62 | query_history.query_type, 63 | query_history.session_id, 64 | query_history.user_name, 65 | query_history.role_name, 66 | query_history.warehouse_id, 67 | query_history.warehouse_name, 68 | query_history.warehouse_size, 69 | query_history.warehouse_type, 70 | query_history.cluster_number, 71 | query_history.query_tag, 72 | query_history.execution_status, 73 | query_history.error_code, 74 | query_history.error_message, 75 | query_history.start_time, 76 | query_history.end_time, 77 | query_history.total_elapsed_time, 78 | query_history.bytes_scanned, 79 | query_history.percentage_scanned_from_cache, 80 | query_history.bytes_written, 81 | query_history.bytes_written_to_result, 82 | query_history.bytes_read_from_result, 83 | query_history.rows_produced, 84 | query_history.rows_inserted, 85 | query_history.rows_updated, 86 | query_history.rows_deleted, 87 | query_history.rows_unloaded, 88 | query_history.bytes_deleted, 89 | query_history.partitions_scanned, 90 | query_history.partitions_total, 91 | query_history.bytes_spilled_to_local_storage, 92 | query_history.bytes_spilled_to_remote_storage, 93 | query_history.bytes_sent_over_the_network, 94 | query_history.outbound_data_transfer_cloud, 95 | query_history.outbound_data_transfer_region, 96 | query_history.outbound_data_transfer_bytes, 97 | query_history.inbound_data_transfer_cloud, 98 | query_history.inbound_data_transfer_region, 99 | query_history.inbound_data_transfer_bytes, 100 | query_history.credits_used_cloud_services, 101 | query_history.release_version, 102 | query_history.external_function_total_invocations, 103 | query_history.external_function_total_sent_rows, 104 | query_history.external_function_total_received_rows, 105 | query_history.external_function_total_sent_bytes, 106 | query_history.external_function_total_received_bytes, 107 | query_history.query_load_percent, 108 | query_history.is_client_generated_statement, 109 | query_history.query_acceleration_bytes_scanned, 110 | query_history.query_acceleration_partitions_scanned, 111 | query_history.query_acceleration_upper_limit_scale_factor, 112 | query_history.query_hash, 113 | query_history.query_hash_version, 114 | query_history.query_parameterized_hash, 115 | query_history.query_parameterized_hash_version, 116 | 117 | -- Rename some existing columns for clarity 118 | query_history.total_elapsed_time as total_elapsed_time_ms, 119 | query_history.compilation_time as compilation_time_ms, 120 | query_history.queued_provisioning_time as queued_provisioning_time_ms, 121 | query_history.queued_repair_time as queued_repair_time_ms, 122 | query_history.queued_overload_time as queued_overload_time_ms, 123 | query_history.transaction_blocked_time as transaction_blocked_time_ms, 124 | query_history.list_external_files_time as list_external_files_time_ms, 125 | query_history.execution_time as execution_time_ms, 126 | 127 | -- New columns 128 | query_history.warehouse_size is not null as ran_on_warehouse, 129 | query_history.bytes_scanned / power(1024, 3) as data_scanned_gb, 130 | data_scanned_gb * query_history.percentage_scanned_from_cache as data_scanned_from_cache_gb, 131 | query_history.bytes_spilled_to_local_storage / power(1024, 3) as data_spilled_to_local_storage_gb, 132 | query_history.bytes_spilled_to_remote_storage / power(1024, 3) as data_spilled_to_remote_storage_gb, 133 | query_history.bytes_sent_over_the_network / power(1024, 3) as data_sent_over_the_network_gb, 134 | query_history.query_text_no_comments, 135 | query_history.dbt_metadata, 136 | 137 | query_history.total_elapsed_time / 1000 as total_elapsed_time_s, 138 | query_history.compilation_time / 1000 as compilation_time_s, 139 | query_history.queued_provisioning_time / 1000 as queued_provisioning_time_s, 140 | query_history.queued_repair_time / 1000 as queued_repair_time_s, 141 | query_history.queued_overload_time / 1000 as queued_overload_time_s, 142 | query_history.transaction_blocked_time / 1000 as transaction_blocked_time_s, 143 | query_history.list_external_files_time / 1000 as list_external_files_time_s, 144 | query_history.execution_time / 1000 as execution_time_s, 145 | cost_per_query.currency, 146 | query_history.query_retry_time as query_retry_time_ms, 147 | query_history.query_retry_time / 1000 as query_retry_time_s, 148 | query_history.query_retry_cause, 149 | query_history.fault_handling_time as fault_handling_time_ms, 150 | query_history.fault_handling_time / 1000 as fault_handling_time_s 151 | 152 | from query_history 153 | inner join cost_per_query 154 | on query_history.query_id = cost_per_query.query_id 155 | order by query_history.start_time 156 | -------------------------------------------------------------------------------- /models/cost_per_query.sql: -------------------------------------------------------------------------------- 1 | {{ config( 2 | materialized='incremental', 3 | unique_key=['query_id', 'start_time'], 4 | ) }} 5 | 6 | with 7 | stop_threshold as ( 8 | select max(end_time) as latest_ts 9 | from {{ ref('stg_warehouse_metering_history') }} 10 | ), 11 | 12 | filtered_queries as ( 13 | select 14 | query_id, 15 | query_text as original_query_text, 16 | credits_used_cloud_services, 17 | warehouse_id, 18 | warehouse_size is not null as ran_on_warehouse, 19 | timeadd( 20 | 'millisecond', 21 | queued_overload_time + compilation_time 22 | + queued_provisioning_time + queued_repair_time 23 | + list_external_files_time, 24 | start_time 25 | ) as execution_start_time, 26 | start_time, 27 | end_time, 28 | query_acceleration_bytes_scanned 29 | from {{ ref('stg_query_history') }} 30 | where true 31 | and end_time <= (select stop_threshold.latest_ts from stop_threshold) 32 | {% if is_incremental() %} 33 | -- account for late arriving queries 34 | and end_time > (select coalesce(dateadd(day, -3, max(end_time)), '1970-01-01') from {{ this }}) 35 | {% endif %} 36 | ), 37 | 38 | hours_list as ( 39 | select 40 | dateadd( 41 | 'hour', 42 | '-' || row_number() over ( 43 | order by seq4() asc), 44 | dateadd('day', '+1', current_date::timestamp_tz) 45 | ) as hour_start, 46 | dateadd('hour', '+1', hour_start) as hour_end 47 | 48 | {% if is_incremental() %} 49 | from table(generator(rowcount => (24 * 7))) 50 | {% else %} 51 | from table(generator(rowcount => (24 * 1095))) 52 | {% endif %} 53 | ), 54 | 55 | -- 1 row per hour a query ran 56 | query_hours as ( 57 | select 58 | hours_list.hour_start, 59 | hours_list.hour_end, 60 | queries.* 61 | from hours_list 62 | inner join filtered_queries as queries 63 | on hours_list.hour_start >= date_trunc('hour', queries.execution_start_time) 64 | and hours_list.hour_start < queries.end_time 65 | and queries.ran_on_warehouse 66 | ), 67 | 68 | query_seconds_per_hour as ( 69 | select 70 | *, 71 | datediff('millisecond', greatest(execution_start_time, hour_start), least(end_time, hour_end)) as num_milliseconds_query_ran, 72 | sum(num_milliseconds_query_ran) over (partition by warehouse_id, hour_start) as total_query_milliseconds_in_hour, 73 | div0(num_milliseconds_query_ran, total_query_milliseconds_in_hour) as fraction_of_total_query_time_in_hour, 74 | sum(query_acceleration_bytes_scanned) over (partition by warehouse_id, hour_start) as total_query_acceleration_bytes_scanned_in_hour, 75 | div0(query_acceleration_bytes_scanned, total_query_acceleration_bytes_scanned_in_hour) as fraction_of_total_query_acceleration_bytes_scanned_in_hour, 76 | hour_start as hour 77 | from query_hours 78 | ), 79 | 80 | credits_billed_hourly as ( 81 | select 82 | start_time as hour, 83 | entity_id as warehouse_id, 84 | sum(iff(service_type = 'WAREHOUSE_METERING', credits_used_compute, 0)) as credits_used_compute, 85 | sum(iff(service_type = 'WAREHOUSE_METERING', credits_used_cloud_services, 0)) as credits_used_cloud_services, 86 | sum(iff(service_type = 'QUERY_ACCELERATION', credits_used_compute, 0)) as credits_used_query_acceleration 87 | from {{ ref('stg_metering_history') }} 88 | where true 89 | and service_type in ('QUERY_ACCELERATION', 'WAREHOUSE_METERING') 90 | group by 1, 2 91 | ), 92 | 93 | query_cost as ( 94 | select 95 | query_seconds_per_hour.*, 96 | credits_billed_hourly.credits_used_compute * query_seconds_per_hour.fraction_of_total_query_time_in_hour as allocated_compute_credits_in_hour, 97 | allocated_compute_credits_in_hour * daily_rates.effective_rate as allocated_compute_cost_in_hour, 98 | credits_billed_hourly.credits_used_query_acceleration * query_seconds_per_hour.fraction_of_total_query_acceleration_bytes_scanned_in_hour as allocated_query_acceleration_credits_in_hour, 99 | allocated_query_acceleration_credits_in_hour * daily_rates.effective_rate as allocated_query_acceleration_cost_in_hour 100 | from query_seconds_per_hour 101 | inner join credits_billed_hourly 102 | on query_seconds_per_hour.warehouse_id = credits_billed_hourly.warehouse_id 103 | and query_seconds_per_hour.hour = credits_billed_hourly.hour 104 | inner join {{ ref('daily_rates') }} as daily_rates 105 | on date(query_seconds_per_hour.start_time) = daily_rates.date 106 | and daily_rates.service_type = 'WAREHOUSE_METERING' 107 | and daily_rates.usage_type = 'compute' 108 | ), 109 | 110 | cost_per_query as ( 111 | select 112 | query_id, 113 | any_value(start_time) as start_time, 114 | any_value(end_time) as end_time, 115 | any_value(execution_start_time) as execution_start_time, 116 | sum(allocated_compute_cost_in_hour) as compute_cost, 117 | sum(allocated_compute_credits_in_hour) as compute_credits, 118 | sum(allocated_query_acceleration_cost_in_hour) as query_acceleration_cost, 119 | sum(allocated_query_acceleration_credits_in_hour) as query_acceleration_credits, 120 | any_value(credits_used_cloud_services) as credits_used_cloud_services, 121 | any_value(ran_on_warehouse) as ran_on_warehouse 122 | from query_cost 123 | group by 1 124 | ), 125 | 126 | credits_billed_daily as ( 127 | select 128 | date(hour) as date, 129 | sum(credits_used_compute) as daily_credits_used_compute, 130 | sum(credits_used_cloud_services) as daily_credits_used_cloud_services, 131 | greatest(daily_credits_used_cloud_services - daily_credits_used_compute * 0.1, 0) as daily_billable_cloud_services 132 | from credits_billed_hourly 133 | group by 1 134 | ), 135 | 136 | all_queries as ( 137 | select 138 | query_id, 139 | start_time, 140 | end_time, 141 | execution_start_time, 142 | compute_cost, 143 | compute_credits, 144 | query_acceleration_cost, 145 | query_acceleration_credits, 146 | credits_used_cloud_services, 147 | ran_on_warehouse 148 | from cost_per_query 149 | 150 | union all 151 | 152 | select 153 | query_id, 154 | start_time, 155 | end_time, 156 | execution_start_time, 157 | 0 as compute_cost, 158 | 0 as compute_credits, 159 | 0 as query_acceleration_cost, 160 | 0 as query_acceleration_credits, 161 | credits_used_cloud_services, 162 | ran_on_warehouse 163 | from filtered_queries 164 | where 165 | not ran_on_warehouse 166 | ) 167 | 168 | select 169 | all_queries.query_id, 170 | all_queries.start_time, 171 | all_queries.end_time, 172 | all_queries.execution_start_time, 173 | all_queries.compute_cost, 174 | all_queries.compute_credits, 175 | all_queries.query_acceleration_cost, 176 | all_queries.query_acceleration_credits, 177 | -- For the most recent day, which is not yet complete, this calculation won't be perfect. 178 | -- For example, at 12PM on the latest day, it's possible that cloud credits make up <10% of compute cost, so the queries 179 | -- from that day are not allocated any cloud_services_cost. The next time the model runs, after we have the full day of data, 180 | -- this may change if cloud credits make up >10% of compute cost. 181 | (div0(all_queries.credits_used_cloud_services, credits_billed_daily.daily_credits_used_cloud_services) * credits_billed_daily.daily_billable_cloud_services) * coalesce(daily_rates.effective_rate, current_rates.effective_rate) as cloud_services_cost, 182 | div0(all_queries.credits_used_cloud_services, credits_billed_daily.daily_credits_used_cloud_services) * credits_billed_daily.daily_billable_cloud_services as cloud_services_credits, 183 | all_queries.compute_cost + all_queries.query_acceleration_cost + cloud_services_cost as query_cost, 184 | all_queries.compute_credits + all_queries.query_acceleration_credits + cloud_services_credits as query_credits, 185 | all_queries.ran_on_warehouse, 186 | coalesce(daily_rates.currency, current_rates.currency) as currency 187 | from all_queries 188 | inner join credits_billed_daily 189 | on date(all_queries.start_time) = credits_billed_daily.date 190 | left join {{ ref('daily_rates') }} as daily_rates 191 | on date(all_queries.start_time) = daily_rates.date 192 | and daily_rates.service_type = 'CLOUD_SERVICES' 193 | and daily_rates.usage_type = 'cloud services' 194 | inner join {{ ref('daily_rates') }} as current_rates 195 | on current_rates.is_latest_rate 196 | and current_rates.service_type = 'CLOUD_SERVICES' 197 | and current_rates.usage_type = 'cloud services' 198 | order by all_queries.start_time asc 199 | -------------------------------------------------------------------------------- /models/staging/stg_query_history.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: stg_query_history 5 | description: An incrementally materialized copy of the snowflake.account_usage.query_history view. It can be used to query Snowflake query history by various dimensions (time range, session, user, warehouse, etc.) within the last 365 days (1 year). It contains 1 row per query (query_id). 6 | columns: 7 | - name: query_id 8 | description: Internal/system-generated identifier for the SQL statement. 9 | - name: query_text 10 | description: Text of the SQL statement. 11 | - name: database_id 12 | description: Internal/system-generated identifier for the database that was in use. 13 | - name: database_name 14 | description: Database that was in use at the time of the query. 15 | - name: schema_id 16 | description: Internal/system-generated identifier for the schema that was in use. 17 | - name: schema_name 18 | description: Schema that was in use at the time of the query. 19 | - name: query_type 20 | description: DML, query, etc. If the query failed, then the query type may be UNKNOWN. 21 | - name: session_id 22 | description: Session that executed the statement. 23 | - name: user_name 24 | description: User who issued the query. 25 | - name: role_name 26 | description: Role that was active in the session at the time of the query. 27 | - name: warehouse_id 28 | description: Internal/system-generated identifier for the warehouse that was used. 29 | - name: warehouse_name 30 | description: Warehouse that the query executed on, if any. 31 | - name: warehouse_size 32 | description: Size of the warehouse when this statement executed. 33 | - name: warehouse_type 34 | description: Type of the warehouse when this statement executed. 35 | - name: cluster_number 36 | description: The cluster (in a multi-cluster warehouse) that this statement executed on. 37 | - name: query_tag 38 | description: Query tag set for this statement through the QUERY_TAG session parameter. 39 | - name: execution_status 40 | description: "Execution status for the query. Valid values: `success`, `fail`, `incident`." 41 | - name: error_code 42 | description: Error code, if the query returned an error. 43 | - name: error_message 44 | description: Error message, if the query returned an error. 45 | - name: start_time 46 | description: Statement start time (in the UTC time zone). The table is naturally clustered on this column, meaning your queries will run much faster if you filter records using this column. 47 | - name: end_time 48 | description: Statement end time (in the UTC time zone). 49 | - name: total_elapsed_time 50 | description: Elapsed time (in milliseconds). 51 | - name: bytes_scanned 52 | description: Number of bytes scanned by this statement. 53 | - name: percentage_scanned_from_cache 54 | description: The percentage of data scanned from the local disk cache. The value ranges from 0.0 to 1.0. Multiply by 100 to get a true percentage. 55 | - name: bytes_written 56 | description: Number of bytes written (e.g. when loading into a table). 57 | - name: bytes_written_to_result 58 | description: Number of bytes written to a result object. For example, `select * from . . .` would produce a set of results in tabular format representing each field in the selection. In general, the results object represents whatever is produced as a result of the query, and BYTES_WRITTEN_TO_RESULT represents the size of the returned result. 59 | - name: bytes_read_from_result 60 | description: Number of bytes read from a result object. 61 | - name: rows_produced 62 | description: Number of rows produced by this statement. 63 | - name: rows_inserted 64 | description: Number of rows inserted by the query. 65 | - name: rows_updated 66 | description: Number of rows updated by the query. 67 | - name: rows_deleted 68 | description: Number of rows deleted by the query. 69 | - name: rows_unloaded 70 | description: Number of rows unloaded during data export. 71 | - name: bytes_deleted 72 | description: Number of bytes deleted by the query. 73 | - name: partitions_scanned 74 | description: Number of micro-partitions scanned. 75 | - name: partitions_total 76 | description: Total micro-partitions of all tables included in this query. 77 | - name: bytes_spilled_to_local_storage 78 | description: Volume of data spilled to local disk on the warehouse nodes. 79 | - name: bytes_spilled_to_remote_storage 80 | description: Volume of data spilled to remote disk (i.e. AWS S3, Google Cloud Storage, Azure Blob). 81 | - name: bytes_sent_over_the_network 82 | description: Volume of data sent over the network. 83 | - name: compilation_time 84 | description: Compilation time (in milliseconds). 85 | - name: execution_time 86 | description: Execution time (in milliseconds). 87 | - name: queued_provisioning_time 88 | description: Time (in milliseconds) spent in the warehouse queue, waiting for the warehouse compute resources to provision, due to warehouse creation, resume, or resize. 89 | - name: queued_repair_time 90 | description: Time (in milliseconds) spent in the warehouse queue, waiting for compute resources in the warehouse to be repaired. 91 | - name: queued_overload_time 92 | description: Time (in milliseconds) spent in the warehouse queue, due to the warehouse being overloaded by the current query workload. 93 | - name: transaction_blocked_time 94 | description: Time (in milliseconds) spent blocked by a concurrent DML. 95 | - name: outbound_data_transfer_cloud 96 | description: Target cloud provider for statements that unload data to another region and/or cloud. 97 | - name: outbound_data_transfer_region 98 | description: Target region for statements that unload data to another region and/or cloud. 99 | - name: outbound_data_transfer_bytes 100 | description: Number of bytes transferred in statements that unload data to another region and/or cloud. 101 | - name: inbound_data_transfer_cloud 102 | description: Source cloud provider for statements that load data from another region and/or cloud. 103 | - name: inbound_data_transfer_region 104 | description: Source region for statements that load data from another region and/or cloud. 105 | - name: inbound_data_transfer_bytes 106 | description: Number of bytes transferred in statements that load data from another region and/or cloud. 107 | - name: list_external_files_time 108 | description: Time (in milliseconds) spent listing external files. 109 | - name: credits_used_cloud_services 110 | description: Number of credits used for cloud services. 111 | - name: release_version 112 | description: "Release version in the format of `..`." 113 | - name: external_function_total_invocations 114 | description: The aggregate number of times that this query called remote services. For important details, see the Usage Notes. 115 | - name: external_function_total_sent_rows 116 | description: The total number of rows that this query sent in all calls to all remote services. 117 | - name: external_function_total_received_rows 118 | description: The total number of rows that this query received from all calls to all remote services. 119 | - name: external_function_total_sent_bytes 120 | description: The total number of bytes that this query sent in all calls to all remote services. 121 | - name: external_function_total_received_bytes 122 | description: The total number of bytes that this query received from all calls to all remote services. 123 | - name: query_load_percent 124 | description: The approximate percentage of active compute resources in the warehouse for this query execution. 125 | - name: is_client_generated_statement 126 | description: Indicates whether the query was client-generated. These tend to be queries from the Snowflake UI (i.e. Snowsight or the Classic Console). 127 | - name: query_acceleration_bytes_scanned 128 | description: Number of bytes scanned by the query acceleration service. 129 | - name: query_acceleration_partitions_scanned 130 | description: Number of partitions scanned by the query acceleration service. 131 | - name: query_acceleration_upper_limit_scale_factor 132 | description: Upper limit scale factor that a query would have benefited from. 133 | - name: query_hash 134 | description: The hash value computed based on the canonicalized SQL text. 135 | - name: query_hash_version 136 | description: The version of the logic used to compute QUERY_HASH. 137 | - name: query_parameterized_hash 138 | description: The hash value computed based on the parameterized query. 139 | - name: query_parameterized_hash_version 140 | description: The version of the logic used to compute QUERY_PARAMETERIZED_HASH. 141 | -------------------------------------------------------------------------------- /models/query_history_enriched.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | models: 4 | - name: query_history_enriched 5 | description: An enriched version of the query_history model. Contains 1 row per query (query_id). 6 | columns: 7 | - name: query_id 8 | description: Primary key. Internal/system-generated identifier for the SQL statement. 9 | - name: query_text 10 | description: Text of the SQL statement. 11 | - name: database_id 12 | description: Internal/system-generated identifier for the database that was in use. 13 | - name: database_name 14 | description: Database that was in use at the time of the query. 15 | - name: schema_id 16 | description: Internal/system-generated identifier for the schema that was in use. 17 | - name: schema_name 18 | description: Schema that was in use at the time of the query. 19 | - name: query_type 20 | description: DML, query, etc. If the query failed, then the query type may be UNKNOWN. 21 | - name: session_id 22 | description: Session that executed the statement. 23 | - name: user_name 24 | description: User who issued the query. 25 | - name: role_name 26 | description: Role that was active in the session at the time of the query. 27 | - name: warehouse_id 28 | description: Internal/system-generated identifier for the warehouse that was used. 29 | - name: warehouse_name 30 | description: Warehouse that the query executed on, if any. 31 | - name: warehouse_size 32 | description: Size of the warehouse when this statement executed. 33 | - name: warehouse_type 34 | description: Type of the warehouse when this statement executed. 35 | - name: cluster_number 36 | description: The cluster (in a multi-cluster warehouse) that this statement executed on. 37 | - name: query_tag 38 | description: Query tag set for this statement through the QUERY_TAG session parameter. 39 | - name: execution_status 40 | description: "Execution status for the query. Valid values: `success`, `fail`, `incident`." 41 | - name: error_code 42 | description: Error code, if the query returned an error. 43 | - name: error_message 44 | description: Error message, if the query returned an error. 45 | - name: start_time 46 | description: Statement start time (in the UTC time zone). The table is naturally clustered on this column, meaning your queries will run much faster if you filter records using this column. 47 | - name: end_time 48 | description: Statement end time (in the UTC time zone). 49 | - name: bytes_scanned 50 | description: Number of bytes scanned by this statement. 51 | - name: percentage_scanned_from_cache 52 | description: The percentage of data scanned from the local disk cache. The value ranges from 0.0 to 1.0. Multiply by 100 to get a true percentage. 53 | - name: bytes_written 54 | description: Number of bytes written (e.g. when loading into a table). 55 | - name: bytes_written_to_result 56 | description: Number of bytes written to a result object. For example, `select * from . . .` would produce a set of results in tabular format representing each field in the selection. In general, the results object represents whatever is produced as a result of the query, and BYTES_WRITTEN_TO_RESULT represents the size of the returned result. 57 | - name: bytes_read_from_result 58 | description: Number of bytes read from a result object. 59 | - name: rows_produced 60 | description: Number of rows produced by this statement. 61 | - name: rows_inserted 62 | description: Number of rows inserted by the query. 63 | - name: rows_updated 64 | description: Number of rows updated by the query. 65 | - name: rows_deleted 66 | description: Number of rows deleted by the query. 67 | - name: rows_unloaded 68 | description: Number of rows unloaded during data export. 69 | - name: bytes_deleted 70 | description: Number of bytes deleted by the query. 71 | - name: partitions_scanned 72 | description: Number of micro-partitions scanned. 73 | - name: partitions_total 74 | description: Total micro-partitions of all tables included in this query. 75 | - name: bytes_spilled_to_local_storage 76 | description: Volume of data spilled to local disk on the warehouse nodes. 77 | - name: bytes_spilled_to_remote_storage 78 | description: Volume of data spilled to remote disk (i.e. AWS S3, Google Cloud Storage, Azure Blob). 79 | - name: bytes_sent_over_the_network 80 | description: Volume of data sent over the network. 81 | - name: outbound_data_transfer_cloud 82 | description: Target cloud provider for statements that unload data to another region and/or cloud. 83 | - name: outbound_data_transfer_region 84 | description: Target region for statements that unload data to another region and/or cloud. 85 | - name: outbound_data_transfer_bytes 86 | description: Number of bytes transferred in statements that unload data to another region and/or cloud. 87 | - name: inbound_data_transfer_cloud 88 | description: Source cloud provider for statements that load data from another region and/or cloud. 89 | - name: inbound_data_transfer_region 90 | description: Source region for statements that load data from another region and/or cloud. 91 | - name: inbound_data_transfer_bytes 92 | description: Number of bytes transferred in statements that load data from another region and/or cloud. 93 | - name: credits_used_cloud_services 94 | description: Number of credits used for cloud services. 95 | - name: release_version 96 | description: "Release version in the format of `..`." 97 | - name: external_function_total_invocations 98 | description: The aggregate number of times that this query called remote services. For important details, see the Usage Notes. 99 | - name: external_function_total_sent_rows 100 | description: The total number of rows that this query sent in all calls to all remote services. 101 | - name: external_function_total_received_rows 102 | description: The total number of rows that this query received from all calls to all remote services. 103 | - name: external_function_total_sent_bytes 104 | description: The total number of bytes that this query sent in all calls to all remote services. 105 | - name: external_function_total_received_bytes 106 | description: The total number of bytes that this query received from all calls to all remote services. 107 | - name: query_load_percent 108 | description: The approximate percentage of active compute resources in the warehouse for this query execution. 109 | - name: is_client_generated_statement 110 | description: Indicates whether the query was client-generated. These tend to be queries from the Snowflake UI (i.e. Snowsight or the Classic Console). 111 | - name: query_acceleration_bytes_scanned 112 | description: Number of bytes scanned by the query acceleration service. 113 | - name: query_acceleration_partitions_scanned 114 | description: Number of partitions scanned by the query acceleration service. 115 | - name: query_acceleration_upper_limit_scale_factor 116 | description: Upper limit scale factor that a query would have benefited from. 117 | 118 | # Snowflake's Query Hash & Parameterized Query Hash columns 119 | - name: query_hash 120 | description: The hash value computed based on the canonicalized SQL text. 121 | - name: query_hash_version 122 | description: The version of the logic used to compute QUERY_HASH. 123 | - name: query_parameterized_hash 124 | description: The hash value computed based on the parameterized query. 125 | - name: query_parameterized_hash_version 126 | description: The version of the logic used to compute QUERY_PARAMETERIZED_HASH. 127 | 128 | # Renamed existing columns 129 | - name: total_elapsed_time_ms 130 | description: Elapsed time (in milliseconds). 131 | - name: compilation_time_ms 132 | description: Compilation time (in milliseconds). 133 | - name: execution_time_ms 134 | description: Execution time (in milliseconds). 135 | - name: queued_provisioning_time_ms 136 | description: Time (in milliseconds) spent in the warehouse queue, waiting for the warehouse compute resources to provision, due to warehouse creation, resume, or resize. 137 | - name: queued_repair_time_ms 138 | description: Time (in milliseconds) spent in the warehouse queue, waiting for compute resources in the warehouse to be repaired. 139 | - name: queued_overload_time_ms 140 | description: Time (in milliseconds) spent in the warehouse queue, due to the warehouse being overloaded by the current query workload. 141 | - name: transaction_blocked_time_ms 142 | description: Time (in milliseconds) spent blocked by a concurrent DML. 143 | - name: list_external_files_time_ms 144 | description: Time (in milliseconds) spent listing external files. 145 | 146 | # Columns above, but in seconds 147 | - name: total_elapsed_time_s 148 | description: Elapsed time (in seconds). 149 | - name: compilation_time_s 150 | description: Compilation time (in seconds). 151 | - name: execution_time_s 152 | description: Execution time (in seconds). 153 | - name: queued_provisioning_time_s 154 | description: Time (in seconds) spent in the warehouse queue, waiting for the warehouse compute resources to provision, due to warehouse creation, resume, or resize. 155 | - name: queued_repair_time_s 156 | description: Time (in seconds) spent in the warehouse queue, waiting for compute resources in the warehouse to be repaired. 157 | - name: queued_overload_time_s 158 | description: Time (in seconds) spent in the warehouse queue, due to the warehouse being overloaded by the current query workload. 159 | - name: transaction_blocked_time_s 160 | description: Time (in seconds) spent blocked by a concurrent DML. 161 | - name: list_external_files_time_s 162 | description: Time (in seconds) spent listing external files. 163 | 164 | # From cost_per_query model 165 | - name: execution_start_time 166 | description: When the query began executing on the warehouse (in the UTC time zone). This will always be after the start_time. 167 | - name: compute_cost 168 | description: Compute costs associated with the query, in the primary currency of your account. Can be 0 if the query did not run on a warehouse. 169 | - name: compute_credits 170 | description: Compute credits associated with the query. Can be 0 if the query did not run on a warehouse. 171 | - name: cloud_services_cost 172 | description: Cloud service costs associated with the query, in the primary currency of your account. Can be 0 if total cloud services credits consumption was less than 10% of total compute credits consumption on that day. 173 | - name: cloud_services_credits 174 | description: Cloud service credits associated with the query. Can be 0 if total cloud services credits consumption was less than 10% of total compute credits consumption on that day. 175 | - name: query_cost 176 | description: Total cost associated with the query, calculated as sum of compute_cost and cloud_services_cost, in the primary currency of your account. 177 | - name: query_credits 178 | description: Total credits associated with the query, calculated as sum of compute_credits and cloud_services_credits. 179 | - name: currency 180 | description: Spend currency, retrieved from Snowflake's daily rate sheet 181 | 182 | 183 | # New columns 184 | - name: ran_on_warehouse 185 | description: True if the query executed in a virtual warehouse. 186 | - name: data_scanned_gb 187 | description: Number of gb scanned by this statement. 188 | - name: data_spilled_to_local_storage_gb 189 | description: Volume of data (in GB) spilled to local disk on the warehouse nodes. 190 | - name: data_spilled_to_remote_storage_gb 191 | description: Volume of data (in GB) spilled to remote disk (i.e. AWS S3, Google Cloud Storage, Azure Blob). 192 | - name: data_sent_over_the_network_gb 193 | description: Volume of data (in GB) sent over the network. 194 | - name: data_scanned_from_cache_gb 195 | description: Volume of data (in GB) scanned from the local disk cache. 196 | - name: query_text_no_comments 197 | description: Contains the original query text stripped of any comments. 198 | - name: dbt_metadata 199 | description: Metadata from the JSON string added to the query by dbt. Null if no metadata was added. 200 | - name: query_retry_time_ms 201 | description: Total execution time (in milliseconds) for query retries caused by actionable errors. 202 | - name: query_retry_time_s 203 | description: Total execution time (in seconds) for query retries caused by actionable errors. 204 | - name: query_retry_cause 205 | description: Error that caused the query to retry. If there is no query retry, the field is NULL. 206 | - name: fault_handling_time_ms 207 | description: Total execution time (in milliseconds) for query retries caused by errors that are not actionable. 208 | - name: fault_handling_time_s 209 | description: Total execution time (in seconds) for query retries caused by errors that are not actionable. 210 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), 6 | and is generated by [Changie](https://github.com/miniscruff/changie). 7 | 8 | ## dbt-snowflake-monitoring 5.5.0 - November 10, 2025 9 | 10 | ### Features 11 | 12 | - feat: add configuration variables for custom source ([#195](https://github.com/get-select/dbt-snowflake-monitoring/pull/195)) 13 | 14 | ### Contributors 15 | - [@erlendhbarstad](https://github.com/erlendhbarstad) (Features) 16 | 17 | 18 | ## dbt-snowflake-monitoring 5.4.7 - June 18, 2025 19 | 20 | ### Fixes 21 | 22 | - pins the latest query_tag package in package-lock.yml ([#188](https://github.com/get-select/dbt-snowflake-monitoring/pull/188)) 23 | 24 | ### Contributors 25 | - [@jeff-skoldberg-gmds](https://github.com/jeff-skoldberg-gmds) (Fixes) 26 | 27 | 28 | ## dbt-snowflake-monitoring 5.4.6 - June 06, 2025 29 | 30 | ### Fixes 31 | 32 | - Handles two json comments if dbt adds them ([#185](https://github.com/get-select/dbt-snowflake-monitoring/pull/185)) 33 | 34 | ### Contributors 35 | - [@parasbhavnani](https://github.com/parasbhavnani) (Fixes) 36 | 37 | 38 | ## dbt-snowflake-monitoring 5.4.5 - May 09, 2025 39 | 40 | ### Features 41 | 42 | - Deduplicate query history ([#182](https://github.com/get-select/dbt-snowflake-monitoring/pull/182)) 43 | 44 | ### Contributors 45 | - [@sergeykhar](https://github.com/sergeykhar) (Features) 46 | 47 | 48 | ## dbt-snowflake-monitoring 5.4.4 - May 8, 2025 49 | 50 | ### Fixes 51 | 52 | - Fix nested json access where the casing of the key has inadvertently changed ([#174](https://github.com/get-select/dbt-snowflake-monitoring/pull/174)) 53 | 54 | ## dbt-snowflake-monitoring 5.4.3 - March 24, 2025 55 | 56 | ### Features 57 | 58 | - Update locator to use account name ([#177](https://github.com/get-select/dbt-snowflake-monitoring/pull/177)) 59 | 60 | 61 | 62 | ## dbt-snowflake-monitoring 5.4.2 - February 27, 2025 63 | 64 | ### Fixes 65 | 66 | - Fix AI services spend ([#176](https://github.com/get-select/dbt-snowflake-monitoring/pull/176)) 67 | 68 | 69 | 70 | ## dbt-snowflake-monitoring 5.4.1 - February 11, 2025 71 | 72 | ### Fixes 73 | 74 | - Fix bug in hourly_spend introduced on 5.4.0 ([#175](https://github.com/get-select/dbt-snowflake-monitoring/pull/175)) 75 | 76 | ### Contributors 77 | - [@fernandobrito](https://github.com/fernandobrito) (Fixes) 78 | 79 | 80 | ## dbt-snowflake-monitoring 5.4.0 - February 06, 2025 81 | 82 | ### Features 83 | 84 | - Support all services by default in hourly_spend ([#173](https://github.com/get-select/dbt-snowflake-monitoring/pull/173)) 85 | 86 | ### Contributors 87 | - [@fernandobrito](https://github.com/fernandobrito) (Features) 88 | 89 | 90 | ## dbt-snowflake-monitoring 5.3.3 - January 31, 2025 91 | 92 | ### Fixes 93 | 94 | - Fix column dependencies in stg_query_history ([#171](https://github.com/get-select/dbt-snowflake-monitoring/pull/171)) 95 | 96 | 97 | 98 | ## dbt-snowflake-monitoring 5.3.2 - January 31, 2025 99 | 100 | Several user-contributed fixes and improvements. 101 | 102 | ## dbt-snowflake-monitoring 5.3.1 - August 23, 2024 103 | 104 | ### Features 105 | 106 | - Add lookback variable to stg_query_history ([#163](https://github.com/get-select/dbt-snowflake-monitoring/pull/163)) 107 | 108 | ### Contributors 109 | - [@smitsrr](https://github.com/smitsrr) (Features) 110 | 111 | 112 | ## dbt-snowflake-monitoring 5.3.0 - June 15, 2024 113 | 114 | ### Features 115 | 116 | - Add new fields for stg_serverless_task_history ([#160](https://github.com/get-select/dbt-snowflake-monitoring/pull/160)) 117 | 118 | ### Contributors 119 | - [@fernandobrito](https://github.com/fernandobrito) (Features) 120 | 121 | 122 | ## dbt-snowflake-monitoring 5.2.2 - June 09, 2024 123 | 124 | ### Features 125 | 126 | - Pull out stg_warehouse_events_history ([#159](https://github.com/get-select/dbt-snowflake-monitoring/pull/159)) 127 | 128 | 129 | 130 | ## dbt-snowflake-monitoring 5.2.1 - June 09, 2024 131 | 132 | ### Features 133 | 134 | - Remove warehouse_cluster_status model ([#158](https://github.com/get-select/dbt-snowflake-monitoring/pull/158)) 135 | 136 | 137 | 138 | ## dbt-snowflake-monitoring 5.2.0 - May 28, 2024 139 | 140 | ### Features 141 | 142 | - Add root_query_id, parent_query_id to stg_access_history ([#156](https://github.com/get-select/dbt-snowflake-monitoring/pull/156)) 143 | 144 | ### Contributors 145 | - [@tnightengale](https://github.com/tnightengale) (Features) 146 | 147 | 148 | ## dbt-snowflake-monitoring 5.1.2 - May 23, 2024 149 | 150 | ### Fixes 151 | 152 | - Fix incremental filtering ([#254](https://github.com/get-select/dbt-snowflake-monitoring/pull/254)) 153 | 154 | 155 | 156 | ## dbt-snowflake-monitoring 5.1.1 - May 14, 2024 157 | 158 | ### Fixes 159 | 160 | - Include overage spend on services recently introduced ([#154](https://github.com/get-select/dbt-snowflake-monitoring/pull/154)) 161 | 162 | ### Contributors 163 | - [@fernandobrito](https://github.com/fernandobrito) (Fixes) 164 | 165 | 166 | ## dbt-snowflake-monitoring 5.1.0 - May 06, 2024 167 | 168 | ### Features 169 | 170 | - Support more usage types ([#152](https://github.com/get-select/dbt-snowflake-monitoring/pull/152)) 171 | 172 | ### Fixes 173 | 174 | - Update to use new service_type values ([#151](https://github.com/get-select/dbt-snowflake-monitoring/pull/151)) 175 | - Calculate net cloud services costs for reader accounts ([#153](https://github.com/get-select/dbt-snowflake-monitoring/pull/153)) 176 | 177 | ### Contributors 178 | - [@fernandobrito](https://github.com/fernandobrito) (Features) 179 | - [@stumelius](https://github.com/stumelius) (Fixes) 180 | 181 | 182 | ## dbt-snowflake-monitoring 5.0.4 - March 15, 2024 183 | 184 | ### Fixes 185 | 186 | - Support different order json comment ([#148](https://github.com/get-select/dbt-snowflake-monitoring/pull/148)) 187 | 188 | 189 | 190 | ## dbt-snowflake-monitoring 5.0.3 - January 16, 2024 191 | 192 | ### Features 193 | 194 | - Add dbt_snowflake_monitoring_incremental_days to set incremental lookback period ([#140](https://github.com/get-select/dbt-snowflake-monitoring/pull/140)) 195 | 196 | ### Contributors 197 | - [@smitsrr](https://github.com/smitsrr) (Features) 198 | 199 | 200 | ## dbt-snowflake-monitoring 5.0.2 - January 15, 2024 201 | 202 | ### Fixes 203 | 204 | - Fix handling of service_type ([#143](https://github.com/get-select/dbt-snowflake-monitoring/pull/143)) 205 | 206 | 207 | 208 | ## dbt-snowflake-monitoring 5.0.1 - January 15, 2024 209 | 210 | ### Fixes 211 | 212 | - Switch stg_metering_history to a table incrementalization to handle natural dupes ([#142](https://github.com/get-select/dbt-snowflake-monitoring/pull/142)) 213 | 214 | 215 | 216 | ## dbt-snowflake-monitoring 5.0.0 - January 14, 2024 217 | 218 | ### Features 219 | 220 | - Add query acceleration costs and update cost per query algorithm to include them ([#141](https://github.com/get-select/dbt-snowflake-monitoring/pull/141)) 221 | - Make cost_per_query model incremental ([#141](https://github.com/get-select/dbt-snowflake-monitoring/pull/141)) 222 | 223 | ### Breaking Changes 224 | 225 | - Add entity_id to stg_metering_history ([#141](https://github.com/get-select/dbt-snowflake-monitoring/pull/141)) 226 | 227 | To upgrade from 4.x.x, you'll need to full refresh the `stg_metering_history` model. 228 | 229 | ### Fixes 230 | 231 | - Support quoting: true ([#139](https://github.com/get-select/dbt-snowflake-monitoring/pull/139)) 232 | 233 | ### Contributors 234 | - [@ernestoongaro](https://github.com/ernestoongaro) (Fixes) 235 | ## dbt-snowflake-monitoring 4.6.0 - November 09, 2023 236 | 237 | ### Features 238 | 239 | - Add credit consumption to cost per query ([#127](https://github.com/get-select/dbt-snowflake-monitoring/pull/127)) 240 | - Add query parameterized hash ([#137](https://github.com/get-select/dbt-snowflake-monitoring/pull/137)) 241 | 242 | ### Fixes 243 | - Support quoting: true ([#139](https://github.com/get-select/dbt-snowflake-monitoring/pull/139)) 244 | 245 | ### Contributors 246 | - [@sidhreddy](https://github.com/sidhreddy) (Features) 247 | - [@yingyingqiqi](https://github.com/yingyingqiqi) (Features) 248 | - [@ernestoongaro](https://github.com/ernestoongaro) (Features) 249 | 250 | ## dbt-snowflake-monitoring 4.5.3 - November 06, 2023 251 | 252 | ### Features 253 | 254 | - Remove regex UDF to avoid timeout ([#135](https://github.com/get-select/dbt-snowflake-monitoring/pull/135)) 255 | 256 | 257 | 258 | ## dbt-snowflake-monitoring 4.5.2 - October 30, 2023 259 | 260 | ### Fixes 261 | 262 | - Fix cluster number ([#134](https://github.com/get-select/dbt-snowflake-monitoring/pull/134)) 263 | 264 | 265 | 266 | ## dbt-snowflake-monitoring 4.5.1 - August 24, 2023 267 | 268 | ### Fixes 269 | 270 | - Sort remaining balance daily with null contract numbers last ([#128](https://github.com/get-select/dbt-snowflake-monitoring/pull/128)) 271 | 272 | 273 | 274 | ## dbt-snowflake-monitoring 4.5.0 - August 09, 2023 275 | 276 | ### Features 277 | 278 | - Add Snowpipe Streaming ([#125](https://github.com/get-select/dbt-snowflake-monitoring/pull/125)) 279 | 280 | 281 | 282 | ## dbt-snowflake-monitoring 4.4.1 - July 28, 2023 283 | 284 | ### Features 285 | 286 | - Remove unneeded data_transfer_history source and stg model ([#124](https://github.com/get-select/dbt-snowflake-monitoring/pull/124)) 287 | 288 | 289 | 290 | ## dbt-snowflake-monitoring 4.4.0 - July 26, 2023 291 | 292 | ### Features 293 | 294 | - Add Data Transfer costs ([#123](https://github.com/get-select/dbt-snowflake-monitoring/pull/123)) 295 | 296 | 297 | 298 | ## dbt-snowflake-monitoring 4.3.1 - June 23, 2023 299 | 300 | ### Features 301 | 302 | - Set range for dbt_snowflake_query_tags ([#121](https://github.com/get-select/dbt-snowflake-monitoring/pull/121)) 303 | 304 | 305 | 306 | ## dbt-snowflake-monitoring 4.3.0 - June 21, 2023 307 | 308 | ### Features 309 | 310 | - Add hourly_spend model ([#116](https://github.com/get-select/dbt-snowflake-monitoring/pull/116)) 311 | 312 | 313 | 314 | ## dbt-snowflake-monitoring 4.2.1 - June 03, 2023 315 | 316 | ### Fixes 317 | 318 | - Daily rates improvements ([#114](https://github.com/get-select/dbt-snowflake-monitoring/pull/114)) 319 | 320 | 321 | 322 | ## dbt-snowflake-monitoring 4.2.0 - May 27, 2023 323 | 324 | ### Features 325 | 326 | - Add node_meta to dbt_query model ([#109](https://github.com/get-select/dbt-snowflake-monitoring/pull/109)) 327 | 328 | ### Contributors 329 | - [@pratik60](https://github.com/pratik60) (Features) 330 | 331 | 332 | ## dbt-snowflake-monitoring 4.1.1 - May 12, 2023 333 | 334 | ### Features 335 | 336 | - Support dbt 1.5.0 ([#106](https://github.com/get-select/dbt-snowflake-monitoring/pull/106)) 337 | 338 | 339 | 340 | ## dbt-snowflake-monitoring 4.1.0 - May 05, 2023 341 | 342 | ### Features 343 | 344 | - Add currency to all models with spend data ([#103](https://github.com/get-select/dbt-snowflake-monitoring/pull/103)) 345 | 346 | ### Contributors 347 | - [@calleo](https://github.com/calleo) (Features) 348 | 349 | 350 | ## dbt-snowflake-monitoring 4.0.2 - April 08, 2023 351 | 352 | ### Fixes 353 | 354 | - Prevent daily_spend from being held up by missing data ([#97](https://github.com/get-select/dbt-snowflake-monitoring/pull/97)) 355 | 356 | 357 | 358 | ## dbt-snowflake-monitoring 4.0.1 - April 07, 2023 359 | 360 | ### Fixes 361 | 362 | - Don't let daily_spend be held up by stale rates ([#95](https://github.com/get-select/dbt-snowflake-monitoring/pull/95)) 363 | 364 | 365 | 366 | ## dbt-snowflake-monitoring 4.0.0 - March 21, 2023 367 | 368 | ### Breaking Changes 369 | 370 | - Rename access_history models ([#92](https://github.com/get-select/dbt-snowflake-monitoring/pull/92)) 371 | 372 | 373 | 374 | ## dbt-snowflake-monitoring 3.0.1 - March 01, 2023 375 | 376 | ### Fixes 377 | 378 | - Fix dbt_cloud_url var ([#90](https://github.com/get-select/dbt-snowflake-monitoring/pull/90)) 379 | 380 | ### Contributors 381 | - [@vinooganesh](https://github.com/vinooganesh) (Fixes) 382 | 383 | 384 | ## dbt-snowflake-monitoring 3.0.0 - February 26, 2023 385 | After attempting to use only query comments or query tags in versions 1 and 2, we've learned we actually need both. The reasons for that are: 386 | * The `is_incremental` macro is only available from within the query tag dbt context, and is needed to determine whether a model run is incremental or not. 387 | * Query tags have a maximum character limit of 2000, which is easily exceeded by a list of refs containing more than a few models. 388 | 389 | To upgrade from 2.x.x, follow the latest instructions in the [Quickstart](https://github.com/get-select/dbt-snowflake-monitoring#quickstart). The package is still able to process query metadata generated by previous versions of the package, so no data will be lost on upgrade. 390 | 391 | ### Breaking Changes 392 | 393 | - Use query comments and query tags to avoid query tag character limit ([#87](https://github.com/get-select/dbt-snowflake-monitoring/pull/87)) 394 | 395 | ## dbt-snowflake-monitoring 2.0.2 - February 21, 2023 396 | 397 | ### Fixes 398 | 399 | - Use the model database and schema for UDF creation ([#83](https://github.com/get-select/dbt-snowflake-monitoring/pull/83)) 400 | 401 | 402 | 403 | ## dbt-snowflake-monitoring 2.0.1 - February 17, 2023 404 | 405 | ### Features 406 | 407 | - Add spend_net_cloud_services ([#80](https://github.com/get-select/dbt-snowflake-monitoring/pull/80)) 408 | 409 | 410 | 411 | ## dbt-snowflake-monitoring 2.0.0 - February 08, 2023 412 | The most significant change in 2.0.0 is using query tags instead of comments to attach dbt metadata to queries. To upgrade from previous versions: 413 | 414 | 1. Remove the query comment configuration from `dbt_project.yml` 415 | 2. Follow the instructions to configure the query tag in the [Quickstart](https://github.com/get-select/dbt-snowflake-monitoring#quickstart). 416 | 417 | ### Breaking Changes 418 | 419 | - Switch to using query tags instead of comments ([#78](https://github.com/get-select/dbt-snowflake-monitoring/pull/78)) 420 | 421 | 422 | 423 | ## dbt-snowflake-monitoring 1.6.1 - February 03, 2023 424 | 425 | ### Fixes 426 | 427 | - Fixes is_incremental in get_query_comment macro ([#77](https://github.com/get-select/dbt-snowflake-monitoring/pull/77)) 428 | 429 | 430 | 431 | ## dbt-snowflake-monitoring 1.6.0 - February 01, 2023 432 | 433 | ### Features 434 | 435 | - Add query_object_access and query_table_access models ([#73](https://github.com/get-select/dbt-snowflake-monitoring/pull/73)) 436 | 437 | 438 | 439 | ## dbt-snowflake-monitoring 1.5.0 - January 25, 2023 440 | 441 | ### Fixes 442 | 443 | - Fix query_text_no_comments regex ([#72](https://github.com/get-select/dbt-snowflake-monitoring/pull/72)) 444 | 445 | 446 | 447 | ## dbt-snowflake-monitoring 1.4.3 - January 20, 2023 448 | 449 | ### Fixes 450 | 451 | - Fix error on initial compile with no models created yet ([#70](https://github.com/get-select/dbt-snowflake-monitoring/pull/70)) 452 | 453 | 454 | 455 | ## dbt-snowflake-monitoring 1.4.2 - January 12, 2023 456 | 457 | ### Fixes 458 | 459 | - Inner join daily_rates to avoid $0 entries in daily_spend for incomplete days ([#66](https://github.com/get-select/dbt-snowflake-monitoring/pull/66)) 460 | 461 | 462 | 463 | ## dbt-snowflake-monitoring 1.4.1 - January 06, 2023 464 | 465 | ### Fixes 466 | 467 | - Increase incremental lookback window to account for changing cost data ([#64](https://github.com/get-select/dbt-snowflake-monitoring/pull/64)) 468 | 469 | 470 | 471 | ## dbt-snowflake-monitoring 1.4.0 - January 06, 2023 472 | 473 | ### Features 474 | 475 | - Add remaining balance daily model ([#58](https://github.com/get-select/dbt-snowflake-monitoring/pull/58)) 476 | - Improve cost per query freshness by using latest rates ([#52](https://github.com/get-select/dbt-snowflake-monitoring/pull/52)) 477 | 478 | ### Fixes 479 | 480 | - Select correct overage vs. regular rate each day ([#59](https://github.com/get-select/dbt-snowflake-monitoring/pull/59)) 481 | 482 | 483 | 484 | ## dbt-snowflake-monitoring 1.3.1 - December 31, 2022 485 | 486 | ### Features 487 | 488 | - Support dbt_utils 1.0.0 ([#48](https://github.com/get-select/dbt-snowflake-monitoring/pull/48)) 489 | 490 | ### Contributors 491 | - [@GtheSheep](https://github.com/GtheSheep) (Features) 492 | 493 | 494 | ## dbt-snowflake-monitoring 1.3.0 - December 21, 2022 495 | 496 | ### Features 497 | 498 | - Add dbt_queries model to easily understand dbt model costs with links to dbt Cloud ([#47](https://github.com/get-select/dbt-snowflake-monitoring/pull/47)) 499 | 500 | 501 | 502 | ## dbt-snowflake-monitoring 1.2.3 - December 15, 2022 503 | 504 | ### Features 505 | 506 | - Add serverless tasks history and spend ([#37](https://github.com/get-select/dbt-snowflake-monitoring/pull/37)) 507 | - Add `warehouse_cluster_status`, `warehouse_credits_map` and `warehouses_type2_dimension` models ([#40](https://github.com/get-select/dbt-snowflake-monitoring/pull/40)) 508 | 509 | ### Contributors 510 | - [@gthesheep](https://github.com/gthesheep) (Features) 511 | - [@ian-whitestone](https://github.com/ian-whitestone) (Features) 512 | 513 | ## dbt-snowflake-monitoring 1.2.2 - December 05, 2022 514 | 515 | ### Fixes 516 | 517 | - Show only complete days in daily_spend and materialize as table ([#34](https://github.com/get-select/dbt-snowflake-monitoring/pull/34)) 518 | 519 | 520 | 521 | ## dbt-snowflake-monitoring 1.2.1 - December 04, 2022 522 | 523 | ### Fixes 524 | 525 | - Fix account_locator var ([#33](https://github.com/get-select/dbt-snowflake-monitoring/pull/33)) 526 | 527 | 528 | 529 | ## dbt-snowflake-monitoring 1.2.0 - November 26, 2022 530 | 531 | ### Features 532 | 533 | - Add get_query_comment macro for model metadata in query history ([#29](https://github.com/get-select/dbt-snowflake-monitoring/pull/29)) 534 | - Support overage usage type for PAYG accounts ([#24](https://github.com/get-select/dbt-snowflake-monitoring/pull/24)) 535 | 536 | ### Fixes 537 | 538 | - Division by zero in cost_per_query model ([#27](https://github.com/get-select/dbt-snowflake-monitoring/pull/27)) 539 | 540 | ### Contributors 541 | - [@aphethean1](https://github.com/aphethean1) (Features, Fixes) 542 | 543 | 544 | ## dbt-snowflake-monitoring 1.1.0 - October 28, 2022 545 | 546 | ### Features 547 | 548 | - Add daily_spend model which aligns with monthly bill ([#13](https://github.com/get-select/dbt-snowflake-monitoring/pull/13)) 549 | 550 | 551 | ## dbt-snowflake-monitoring 1.0.0 - October 17, 2022 552 | 553 | ### Features 554 | 555 | - Add query_history_enriched model with additional dimensions and cost ([#12](https://github.com/get-select/dbt-snowflake-monitoring/pull/12)) 556 | - Add cost per query model ([#1](https://github.com/get-select/dbt-snowflake-monitoring/pull/1)) 557 | 558 | 559 | 560 | -------------------------------------------------------------------------------- /models/hourly_spend.sql: -------------------------------------------------------------------------------- 1 | -- depends_on: {{ ref('stg_metering_history') }} 2 | {{ config(materialized='table') }} 3 | 4 | with hour_spine as ( 5 | {% if execute %} 6 | {% set stg_metering_history_relation = load_relation(ref('stg_metering_history')) %} 7 | {% if stg_metering_history_relation %} 8 | {% set results = run_query("select coalesce(min(convert_timezone('UTC', start_time)), '2023-01-01 00:00:00') from " ~ ref('stg_metering_history')) %} 9 | {% set start_date = "'" ~ results.columns[0][0] ~ "'" %} 10 | {% set results = run_query("select coalesce(dateadd(hour, 1, max(convert_timezone('UTC', start_time))), '2023-01-01 01:00:00') from " ~ ref('stg_metering_history')) %} 11 | {% set end_date = "'" ~ results.columns[0][0] ~ "'" %} 12 | {% else %} 13 | {% set start_date = "'2023-01-01 00:00:00'" %} {# this is just a dummy date for initial compilations before stg_metering_history exists #} 14 | {% set end_date = "'2023-01-01 01:00:00'" %} {# this is just a dummy date for initial compilations before stg_metering_history exists #} 15 | {% endif %} 16 | {% endif %} 17 | {{ dbt_utils.date_spine( 18 | datepart="hour", 19 | start_date=start_date, 20 | end_date=end_date 21 | ) 22 | }} 23 | ), 24 | 25 | hours as ( 26 | select 27 | date_hour as hour, 28 | hour::date as date, 29 | count(hour) over (partition by date) as hours_thus_far, 30 | day(last_day(date)) as days_in_month 31 | from hour_spine 32 | ), 33 | 34 | -- GROUP BY to collapse possible overage and non-overage cost from the same service in the 35 | -- same day into a single row so this model does not emit multiple rows for the same service 36 | -- and hour 37 | usage_in_currency_daily as ( 38 | select 39 | usage_date, 40 | account_name, 41 | replace(usage_type, 'overage-', '') as usage_type, 42 | currency, 43 | sum(usage_in_currency) as usage_in_currency, 44 | sum(usage) as usage, 45 | from {{ ref('stg_usage_in_currency_daily') }} 46 | group by all 47 | ), 48 | 49 | total_itemized_storage_bytes_daily as ( 50 | select 51 | date, 52 | sum(average_database_bytes) / power(1024, 4) as database_storage_terabytes, 53 | sum(average_failsafe_bytes) / power(1024, 4) as failsafe_storage_terabytes 54 | from {{ ref('stg_database_storage_usage_history') }} 55 | group by 1 56 | ), 57 | 58 | -- There's a known difference between the total storage bytes reported in the storage_usage view 59 | -- and the sum of the itemized storage bytes reported in the database_storage_usage_history view. 60 | -- Snowflake invoicing aligns more closely with the storage_usage view, and the difference is reported 61 | -- as being due to storage relating to deleted objects, but retained due to retained clones. 62 | total_reconciled_storage_bytes_daily as ( 63 | select 64 | stg_storage_usage.date, 65 | stg_storage_usage.storage_bytes / power(1024, 4) as storage_terabytes, 66 | stg_storage_usage.failsafe_bytes / power(1024, 4) as failsafe_terabytes, 67 | storage_terabytes - total_itemized_storage_bytes_daily.database_storage_terabytes as residual_storage_terabytes, 68 | failsafe_terabytes - total_itemized_storage_bytes_daily.failsafe_storage_terabytes as residual_failsafe_terabytes 69 | from {{ ref('stg_storage_usage') }} as stg_storage_usage 70 | left join total_itemized_storage_bytes_daily on stg_storage_usage.date = total_itemized_storage_bytes_daily.date 71 | ), 72 | 73 | storage_terabytes_daily as ( 74 | select 75 | date, 76 | 'Table and Time Travel' as storage_type, 77 | database_name, 78 | sum(average_database_bytes) / power(1024, 4) as storage_terabytes 79 | from {{ ref('stg_database_storage_usage_history') }} 80 | group by 1, 2, 3 81 | union all 82 | select 83 | date, 84 | 'Failsafe' as storage_type, 85 | database_name, 86 | sum(average_failsafe_bytes) / power(1024, 4) as storage_terabytes 87 | from {{ ref('stg_database_storage_usage_history') }} 88 | group by 1, 2, 3 89 | union all 90 | select 91 | date, 92 | 'Stage' as storage_type, 93 | null as database_name, 94 | sum(average_stage_bytes) / power(1024, 4) as storage_terabytes 95 | from {{ ref('stg_stage_storage_usage_history') }} 96 | group by 1, 2, 3 97 | union all 98 | select 99 | date, 100 | 'Retained for Clones' as storage_type, 101 | null as database_name, 102 | residual_storage_terabytes as storage_terabytes 103 | from total_reconciled_storage_bytes_daily 104 | where residual_storage_terabytes > 0 105 | union all 106 | select 107 | date, 108 | 'Failsafe Retained for Clones' as storage_type, 109 | null as database_name, 110 | residual_failsafe_terabytes as storage_terabytes 111 | from total_reconciled_storage_bytes_daily 112 | where residual_failsafe_terabytes > 0 113 | ), 114 | 115 | storage_spend_hourly as ( 116 | select 117 | hours.hour, 118 | 'Storage' as service, 119 | storage_terabytes_daily.storage_type, 120 | null as warehouse_name, 121 | storage_terabytes_daily.database_name, 122 | coalesce( 123 | sum( 124 | div0( 125 | storage_terabytes_daily.storage_terabytes, 126 | hours.days_in_month * 24 127 | ) 128 | ), 129 | 0 130 | ) as usage, 131 | usage as usage_net_cloud_services, 132 | coalesce( 133 | sum( 134 | div0( 135 | storage_terabytes_daily.storage_terabytes, 136 | hours.days_in_month * 24 137 | ) * daily_rates.effective_rate 138 | ), 139 | 0 140 | ) as spend, 141 | spend as spend_net_cloud_services, 142 | any_value(daily_rates.currency) as currency, 143 | 'TB' as usage_unit, 144 | any_value(daily_rates.effective_rate) as usage_rate 145 | from hours 146 | left join storage_terabytes_daily on hours.date = convert_timezone('UTC', storage_terabytes_daily.date) 147 | left join {{ ref('daily_rates') }} as daily_rates 148 | on storage_terabytes_daily.date = daily_rates.date 149 | and daily_rates.service_type = 'STORAGE' 150 | and daily_rates.usage_type = 'storage' 151 | group by 1, 2, 3, 4, 5 152 | ), 153 | 154 | -- Hybrid Table Storage has its own service type in `usage_in_currency_daily`, 155 | -- so we also handle it separately, and not with "Storage". 156 | _hybrid_table_terabytes_daily as ( 157 | select 158 | date, 159 | null as storage_type, 160 | database_name, 161 | sum(average_hybrid_table_storage_bytes) / power(1024, 4) as storage_terabytes 162 | from {{ ref('stg_database_storage_usage_history') }} 163 | group by 1, 2, 3 164 | ), 165 | 166 | hybrid_table_storage_spend_hourly as ( 167 | select 168 | hours.hour, 169 | 'Hybrid Table Storage' as service, 170 | null as storage_type, 171 | null as warehouse_name, 172 | _hybrid_table_terabytes_daily.database_name, 173 | coalesce( 174 | sum( 175 | div0( 176 | _hybrid_table_terabytes_daily.storage_terabytes, 177 | hours.days_in_month * 24 178 | ) 179 | ), 180 | 0 181 | ) as usage, 182 | usage as usage_net_cloud_services, 183 | coalesce( 184 | sum( 185 | div0( 186 | _hybrid_table_terabytes_daily.storage_terabytes, 187 | hours.days_in_month * 24 188 | ) * daily_rates.effective_rate 189 | ), 190 | 0 191 | ) as spend, 192 | spend as spend_net_cloud_services, 193 | any_value(daily_rates.currency) as currency, 194 | 'TB' as usage_unit, 195 | any_value(daily_rates.effective_rate) as usage_rate 196 | from hours 197 | left join _hybrid_table_terabytes_daily on hours.date = convert_timezone('UTC', _hybrid_table_terabytes_daily.date) 198 | left join {{ ref('daily_rates') }} as daily_rates 199 | on _hybrid_table_terabytes_daily.date = daily_rates.date 200 | and daily_rates.service_type = 'HYBRID_TABLE_STORAGE' 201 | and daily_rates.usage_type = 'hybrid table storage' 202 | group by 1, 2, 3, 4, 5 203 | ), 204 | 205 | data_transfer_spend_hourly as ( 206 | -- Right now we don't have a way of getting this at an hourly grain 207 | -- We can get source cloud + region, target cloud + region, and bytes transferred at an hourly grain from DATA_TRANSFER_HISTORY 208 | -- But Snowflake doesn't provide data transfer rates programmatically, so we can't get the cost 209 | -- We could make a LUT from https://www.snowflake.com/legal-files/CreditConsumptionTable.pdf but it would be a lot of work to maintain and would frequently become out of date 210 | -- So for now we just use the daily reported usage and evenly distribute it across the day 211 | select 212 | hours.hour, 213 | 'Data Transfer' as service, 214 | null as storage_type, 215 | null as warehouse_name, 216 | null as database_name, 217 | coalesce(usage_in_currency_daily.usage / hours.hours_thus_far, 0) as usage, 218 | coalesce(usage_in_currency_daily.usage / hours.hours_thus_far, 0) as usage_net_cloud_services, 219 | coalesce(usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) as spend, 220 | spend as spend_net_cloud_services, 221 | usage_in_currency_daily.currency as currency, 222 | 'TB' as usage_unit, 223 | round(div0(usage_in_currency_daily.usage_in_currency, usage_in_currency_daily.usage), 2) as usage_rate 224 | from hours 225 | left join usage_in_currency_daily on 226 | usage_in_currency_daily.account_name = {{ account_name() }} 227 | and usage_in_currency_daily.usage_type = 'data transfer' 228 | and hours.hour::date = usage_in_currency_daily.usage_date 229 | ), 230 | 231 | logging_spend_hourly as ( 232 | -- More granular cost information is available in the EVENT_USAGE_HISTORY view. 233 | -- https://docs.snowflake.com/en/developer-guide/logging-tracing/logging-tracing-billing 234 | -- For now we just use the daily reported usage and evenly distribute it across the day 235 | select 236 | hours.hour, 237 | 'Logging' as service, 238 | null as storage_type, 239 | null as warehouse_name, 240 | null as database_name, 241 | coalesce(usage_in_currency_daily.usage / hours.hours_thus_far, 0) as usage, 242 | coalesce(usage_in_currency_daily.usage / hours.hours_thus_far, 0) as usage_net_cloud_services, 243 | coalesce(usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) as spend, 244 | spend as spend_net_cloud_services, 245 | usage_in_currency_daily.currency as currency, 246 | 'Credits' as usage_unit, 247 | round(div0(usage_in_currency_daily.usage_in_currency, usage_in_currency_daily.usage), 2) as usage_rate 248 | from hours 249 | left join usage_in_currency_daily on 250 | usage_in_currency_daily.account_name = {{ account_name() }} 251 | and usage_in_currency_daily.usage_type = 'logging' 252 | and hours.hour::date = usage_in_currency_daily.usage_date 253 | ), 254 | 255 | -- For now we just use the daily reported usage and evenly distribute it across the day 256 | -- More detailed information can be found on READER_ACCOUNT_USAGE.* 257 | {% set reader_usage_types = [ 258 | 'reader compute', 'reader storage', 'reader data transfer' 259 | ] %} 260 | 261 | {%- for reader_usage_type in reader_usage_types %} 262 | "{{ reader_usage_type }}_spend_hourly" as ( 263 | select 264 | hours.hour, 265 | INITCAP('{{ reader_usage_type }}') as service, 266 | null as storage_type, 267 | null as warehouse_name, 268 | null as database_name, 269 | coalesce(usage_in_currency_daily.usage / hours.hours_thus_far, 0) as usage, 270 | coalesce(usage_in_currency_daily.usage / hours.hours_thus_far, 0) as usage_net_cloud_services, 271 | coalesce(usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) as spend, 272 | spend as spend_net_cloud_services, 273 | usage_in_currency_daily.currency as currency, 274 | '{% if reader_usage_type == "reader compute" %}Credits{% else %}TB{% endif %}' as usage_unit, 275 | round(div0(usage_in_currency_daily.usage_in_currency, usage_in_currency_daily.usage), 2) as usage_rate 276 | from hours 277 | left join usage_in_currency_daily on 278 | usage_in_currency_daily.account_name = {{ account_name() }} 279 | and usage_in_currency_daily.usage_type = '{{ reader_usage_type }}' 280 | and hours.hour::date = usage_in_currency_daily.usage_date 281 | ), 282 | {% endfor %} 283 | 284 | reader_adj_for_incl_cloud_services_hourly as ( 285 | select 286 | hours.hour, 287 | INITCAP('reader adj for incl cloud services') as service, 288 | null as storage_type, 289 | null as warehouse_name, 290 | null as database_name, 291 | coalesce(usage_in_currency_daily.usage / hours.hours_thus_far, 0) as usage, 292 | 0 as usage_net_cloud_services, 293 | coalesce(usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) as spend, 294 | 0 as spend_net_cloud_services, 295 | usage_in_currency_daily.currency as currency, 296 | 'Credits' as usage_unit, 297 | round(div0(usage_in_currency_daily.usage_in_currency, usage_in_currency_daily.usage), 2) as usage_rate 298 | from hours 299 | left join usage_in_currency_daily on 300 | usage_in_currency_daily.account_name = {{ account_name() }} 301 | and usage_in_currency_daily.usage_type = 'reader adj for incl cloud services' 302 | and hours.hour::date = usage_in_currency_daily.usage_date 303 | ), 304 | 305 | reader_cloud_services_hourly as ( 306 | select 307 | hours.hour, 308 | INITCAP('reader cloud services') as service, 309 | null as storage_type, 310 | null as warehouse_name, 311 | null as database_name, 312 | coalesce(usage_in_currency_daily.usage / hours.hours_thus_far, 0) as usage, 313 | coalesce(usage_in_currency_daily.usage / hours.hours_thus_far, 0) + reader_adj_for_incl_cloud_services_hourly.usage as usage_net_cloud_services, 314 | coalesce(usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) as spend, 315 | coalesce(usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) + reader_adj_for_incl_cloud_services_hourly.spend as spend_net_cloud_services, 316 | usage_in_currency_daily.currency as currency, 317 | 'Credits' as usage_unit, 318 | round(div0(usage_in_currency_daily.usage_in_currency, usage_in_currency_daily.usage), 2) as usage_rate 319 | from hours 320 | left join usage_in_currency_daily on 321 | usage_in_currency_daily.account_name = {{ account_name() }} 322 | and usage_in_currency_daily.usage_type = 'reader cloud services' 323 | and hours.hour::date = usage_in_currency_daily.usage_date 324 | left join reader_adj_for_incl_cloud_services_hourly on 325 | hours.hour = reader_adj_for_incl_cloud_services_hourly.hour 326 | ), 327 | 328 | compute_spend_hourly as ( 329 | select 330 | hours.hour, 331 | 'Compute' as service, 332 | null as storage_type, 333 | stg_metering_history.name as warehouse_name, 334 | null as database_name, 335 | coalesce(sum(stg_metering_history.credits_used_compute),0) as usage, 336 | usage as usage_net_cloud_services, 337 | coalesce( 338 | sum( 339 | stg_metering_history.credits_used_compute * daily_rates.effective_rate 340 | ), 341 | 0 342 | ) as spend, 343 | spend as spend_net_cloud_services, 344 | any_value(daily_rates.currency) as currency, 345 | 'Credits' as usage_unit, 346 | any_value(daily_rates.effective_rate) as usage_rate 347 | from hours 348 | left join {{ ref('stg_metering_history') }} as stg_metering_history on 349 | hours.hour = convert_timezone( 350 | 'UTC', stg_metering_history.start_time 351 | ) 352 | left join {{ ref('daily_rates') }} as daily_rates 353 | on hours.hour::date = daily_rates.date 354 | and daily_rates.service_type = 'WAREHOUSE_METERING' 355 | and daily_rates.usage_type = 'compute' 356 | where 357 | stg_metering_history.service_type = 'WAREHOUSE_METERING' and stg_metering_history.name != 'CLOUD_SERVICES_ONLY' 358 | group by 1, 2, 3, 4 359 | ), 360 | 361 | serverless_task_spend_hourly as ( 362 | select 363 | hours.hour, 364 | 'Serverless Tasks' as service, 365 | null as storage_type, 366 | null as warehouse_name, 367 | stg_serverless_task_history.database_name, 368 | coalesce(sum(stg_serverless_task_history.credits_used),0) as usage, 369 | usage as usage_net_cloud_services, 370 | coalesce( 371 | sum( 372 | stg_serverless_task_history.credits_used * daily_rates.effective_rate 373 | ), 374 | 0 375 | ) as spend, 376 | spend as spend_net_cloud_services, 377 | any_value(daily_rates.currency) as currency, 378 | 'Credits' as usage_unit, 379 | any_value(daily_rates.effective_rate) as usage_rate 380 | from hours 381 | left join {{ ref('stg_serverless_task_history') }} as stg_serverless_task_history on 382 | hours.hour = date_trunc('hour', stg_serverless_task_history.start_time) 383 | left join {{ ref('daily_rates') }} as daily_rates 384 | on hours.hour::date = daily_rates.date 385 | and daily_rates.service_type = 'SERVERLESS_TASK' 386 | and daily_rates.usage_type = 'serverless tasks' 387 | group by 1, 2, 3, 4, 5 388 | ), 389 | 390 | adj_for_incl_cloud_services_hourly as ( 391 | select 392 | hours.hour, 393 | 'Adj For Incl Cloud Services' as service, 394 | null as storage_type, 395 | null as warehouse_name, 396 | null as database_name, 397 | coalesce(sum(stg_metering_daily_history.credits_adjustment_cloud_services), 0) as usage, 398 | 0 as usage_net_cloud_services, 399 | coalesce( 400 | sum( 401 | stg_metering_daily_history.credits_adjustment_cloud_services * daily_rates.effective_rate 402 | ), 403 | 0 404 | ) as spend, 405 | 0 as spend_net_cloud_services, 406 | any_value(daily_rates.currency) as currency, 407 | 'Credits' as usage_unit, 408 | any_value(daily_rates.effective_rate) as usage_rate 409 | from hours 410 | left join {{ ref('stg_metering_daily_history') }} as stg_metering_daily_history on 411 | hours.hour = stg_metering_daily_history.date 412 | left join {{ ref('daily_rates') }} as daily_rates 413 | on hours.hour::date = daily_rates.date 414 | and daily_rates.service_type = 'CLOUD_SERVICES' 415 | and daily_rates.usage_type = 'cloud services' 416 | group by 1, 2, 3, 4 417 | ), 418 | 419 | _cloud_services_usage_hourly as ( 420 | select 421 | hours.hour, 422 | hours.date, 423 | 'Cloud Services' as service, 424 | null as storage_type, 425 | case 426 | when 427 | stg_metering_history.name = 'CLOUD_SERVICES_ONLY' then 'Cloud Services Only' 428 | else stg_metering_history.name 429 | end as warehouse_name, 430 | null as database_name, 431 | coalesce( 432 | sum(stg_metering_history.credits_used_cloud_services), 0 433 | ) as credits_used_cloud_services 434 | from hours 435 | left join {{ ref('stg_metering_history') }} as stg_metering_history on 436 | hours.hour = convert_timezone( 437 | 'UTC', stg_metering_history.start_time 438 | ) 439 | and stg_metering_history.service_type = 'WAREHOUSE_METERING' 440 | group by 1, 2, 3, 4, 5 441 | ), 442 | 443 | _cloud_services_billed_daily as ( 444 | select 445 | date, 446 | sum(credits_used_cloud_services) as credits_used_cloud_services, 447 | sum( 448 | credits_used_cloud_services + credits_adjustment_cloud_services 449 | ) as credits_used_cloud_services_billable 450 | from {{ ref('stg_metering_daily_history') }} 451 | where 452 | service_type = 'WAREHOUSE_METERING' 453 | group by 1 454 | ), 455 | 456 | cloud_services_spend_hourly as ( 457 | select 458 | _cloud_services_usage_hourly.hour, 459 | _cloud_services_usage_hourly.service, 460 | _cloud_services_usage_hourly.storage_type, 461 | _cloud_services_usage_hourly.warehouse_name, 462 | _cloud_services_usage_hourly.database_name, 463 | _cloud_services_usage_hourly.credits_used_cloud_services as usage, 464 | ( 465 | div0( 466 | _cloud_services_usage_hourly.credits_used_cloud_services, 467 | _cloud_services_billed_daily.credits_used_cloud_services 468 | ) * _cloud_services_billed_daily.credits_used_cloud_services_billable 469 | ) as usage_net_cloud_services, 470 | usage * daily_rates.effective_rate as spend, 471 | usage_net_cloud_services * daily_rates.effective_rate as spend_net_cloud_services, 472 | daily_rates.currency, 473 | 'Credits' as usage_unit, 474 | daily_rates.effective_rate as usage_rate 475 | from _cloud_services_usage_hourly 476 | inner join _cloud_services_billed_daily on 477 | _cloud_services_usage_hourly.date = _cloud_services_billed_daily.date 478 | left join {{ ref('daily_rates') }} as daily_rates 479 | on _cloud_services_usage_hourly.date = daily_rates.date 480 | and daily_rates.service_type = 'CLOUD_SERVICES' 481 | and daily_rates.usage_type = 'cloud services' 482 | 483 | ), 484 | 485 | other_costs as ( 486 | select 487 | hours.hour, 488 | 489 | /* Sometimes Snowflake is inconsistent and the service names in metering_history 490 | do not match the service names in our daily_rates (coming from rate_sheet_daily), 491 | so we rename them to make it match */ 492 | case stg_metering_history.service_type 493 | when 'AUTO_CLUSTERING' then 'AUTOMATIC_CLUSTERING' 494 | when 'PIPE' then 'SNOWPIPE' 495 | else stg_metering_history.service_type 496 | end as _service_renamed, 497 | 498 | /* Convert it to a more human-readable format 499 | AUTOMATIC_CLUSTERING -> Automatic Clustering 500 | */ 501 | case _service_renamed 502 | when 'MATERIALIZED_VIEW' then 'Materialized Views' 503 | when 'AI_SERVICES' then 'AI Services' 504 | else initcap(replace(_service_renamed, '_', ' ')) 505 | end as service, 506 | 507 | /* Extract useful information from the row depending on the service type */ 508 | null as storage_type, 509 | case 510 | when stg_metering_history.service_type = 'QUERY_ACCELERATION' 511 | then stg_metering_history.name 512 | else null 513 | end as warehouse_name, 514 | null as database_name, 515 | 516 | coalesce(sum(stg_metering_history.credits_used), 0) as usage, 517 | usage as usage_net_cloud_services, 518 | coalesce( 519 | sum( 520 | stg_metering_history.credits_used * daily_rates.effective_rate 521 | ), 522 | 0 523 | ) as spend, 524 | spend as spend_net_cloud_services, 525 | any_value(daily_rates.currency) as currency, 526 | 'Credits' as usage_unit, 527 | any_value(daily_rates.effective_rate) as usage_rate 528 | 529 | from hours 530 | 531 | left join {{ ref('stg_metering_history') }} as stg_metering_history 532 | on hours.hour = convert_timezone('UTC', stg_metering_history.start_time) 533 | 534 | left join {{ ref('daily_rates') }} as daily_rates 535 | on hour::date = daily_rates.date 536 | and _service_renamed = daily_rates.service_type 537 | /* daily_rates can have multiple rows for the same service_type, 538 | with different values in usage_type (eg: usage_type = "automatic clustering" or 539 | "adjustment-automatic clustering"). We want to join only with the row where 540 | usage_type is the same as the service_type */ 541 | and lower(service) = daily_rates.usage_type 542 | 543 | -- Covered by their own CTEs due to more complex logic or better sources 544 | where stg_metering_history.service_type not in ( 545 | 'SERVERLESS_TASK', 'WAREHOUSE_METERING', 'WAREHOUSE_METERING_READER' 546 | ) 547 | 548 | group by 1, 2, 3, 4, 5 549 | ), 550 | 551 | unioned as ( 552 | select * from storage_spend_hourly 553 | union all 554 | select * from hybrid_table_storage_spend_hourly 555 | union all 556 | select * from data_transfer_spend_hourly 557 | union all 558 | select * from logging_spend_hourly 559 | union all 560 | {%- for reader_usage_type in reader_usage_types %} 561 | select * from "{{ reader_usage_type }}_spend_hourly" 562 | union all 563 | {%- endfor %} 564 | select * from reader_adj_for_incl_cloud_services_hourly 565 | union all 566 | select * from reader_cloud_services_hourly 567 | union all 568 | select * from compute_spend_hourly 569 | union all 570 | select * from adj_for_incl_cloud_services_hourly 571 | union all 572 | select * from cloud_services_spend_hourly 573 | union all 574 | select * from serverless_task_spend_hourly 575 | union all 576 | select * exclude (_service_renamed) from other_costs 577 | ) 578 | 579 | select 580 | convert_timezone('UTC', hour)::timestamp_ltz as hour, 581 | service, 582 | storage_type, 583 | warehouse_name, 584 | database_name, 585 | usage, 586 | usage_net_cloud_services, 587 | spend, 588 | spend_net_cloud_services, 589 | currency, 590 | usage_unit, 591 | usage_rate 592 | from unioned 593 | --------------------------------------------------------------------------------