├── .gitignore ├── packages.yml ├── macros ├── internal │ ├── metadata_processing │ │ ├── wrap_get_columns_in_relation.sql │ │ ├── check_required_parameters.sql │ │ ├── as_constant.sql │ │ ├── escape_column_name.sql │ │ ├── concat_ws.sql │ │ ├── expand_column_list.sql │ │ ├── alias.sql │ │ ├── alias_all.sql │ │ ├── multikey.sql │ │ ├── process_payload_column_excludes.sql │ │ ├── get_escape_characters.sql │ │ └── escape_column_names.sql │ └── helpers │ │ ├── prepend_generated_by.sql │ │ ├── repeat.sql │ │ ├── stage_processing_macros │ │ ├── extract_column_names.sql │ │ ├── print_list.sql │ │ ├── process_columns_to_select.sql │ │ ├── extract_null_column_names.sql │ │ ├── process_hash_column_excludes.sql │ │ └── process_columns_to_escape.sql │ │ ├── dateadd.sql │ │ ├── logging │ │ ├── log_relation_sources.sql │ │ ├── logging_helpers.sql │ │ └── error_messages.sql │ │ ├── is_checks.sql │ │ └── timestamp_add.sql ├── supporting │ ├── data_types │ │ ├── type_timestamp.sql │ │ ├── type_string.sql │ │ └── type_binary.sql │ ├── hash_components │ │ ├── get_hash_string_length.sql │ │ ├── null_expression.sql │ │ └── standard_column_wrapper.sql │ ├── get_query_results_as_dict.sql │ ├── ghost_records │ │ ├── null_ghost.sql │ │ ├── date_ghost.sql │ │ ├── ghost_for_type.sql │ │ ├── binary_ghost.sql │ │ └── create_ghost_record.sql │ ├── max_datetime.sql │ ├── fusion_compat.sql │ ├── casting │ │ ├── cast_binary.sql │ │ ├── cast_date.sql │ │ └── cast_datetime.sql │ ├── prefix.sql │ ├── bridge_shared.sql │ ├── as_of_date_window.sql │ └── hash.sql ├── materialisations │ ├── shared_helpers.sql │ ├── drop_temporary.sql │ ├── period_mat_helpers │ │ ├── error_messages.sql │ │ ├── get_start_stop_dates.sql │ │ ├── check_datediff.sql │ │ ├── get_period_filter_sql.sql │ │ ├── get_period_of_load.sql │ │ └── replace_placeholder_with_period_filter.sql │ ├── rank_mat_helpers │ │ ├── get_min_max_ranks.sql │ │ └── replace_placeholder_with_rank_filter.sql │ ├── incremental_pit_materialization.sql │ ├── incremental_bridge_materialization.sql │ ├── mat_is_checks.sql │ └── incremental_pit_bridge_replace.sql ├── tables │ ├── bigquery │ │ ├── ref_table.sql │ │ ├── link.sql │ │ ├── t_link.sql │ │ ├── xts.sql │ │ ├── sat.sql │ │ ├── ma_sat.sql │ │ ├── eff_sat.sql │ │ ├── bridge.sql │ │ ├── pit.sql │ │ └── hub.sql │ ├── postgres │ │ ├── ref_table.sql │ │ ├── t_link.sql │ │ ├── xts.sql │ │ ├── ma_sat.sql │ │ ├── eff_sat.sql │ │ ├── bridge.sql │ │ ├── hub.sql │ │ └── link.sql │ ├── sqlserver │ │ ├── ref_table.sql │ │ ├── t_link.sql │ │ ├── xts.sql │ │ ├── sat.sql │ │ ├── ma_sat.sql │ │ ├── pit.sql │ │ ├── eff_sat.sql │ │ ├── bridge.sql │ │ ├── hub.sql │ │ └── link.sql │ ├── databricks │ │ ├── xts.sql │ │ ├── ma_sat.sql │ │ ├── pit.sql │ │ ├── bridge.sql │ │ ├── ref_table.sql │ │ ├── t_link.sql │ │ ├── hub.sql │ │ └── link.sql │ └── snowflake │ │ ├── ref_table.sql │ │ ├── t_link.sql │ │ ├── hub.sql │ │ ├── link.sql │ │ └── xts.sql └── staging │ ├── source_columns.sql │ ├── hash_columns.sql │ ├── rank_columns.sql │ ├── null_columns.sql │ └── derive_columns.sql ├── dbt_project.yml ├── NOTICE ├── .github ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md └── workflows │ └── ado_sync.yml ├── CONTRIBUTING.md ├── CODE_OF_CONDUCT.md └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .idea/ 3 | 4 | invoke.yml 5 | -------------------------------------------------------------------------------- /packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: [">=1.0.0", "<2.0.0"] -------------------------------------------------------------------------------- /macros/internal/metadata_processing/wrap_get_columns_in_relation.sql: -------------------------------------------------------------------------------- 1 | {% macro wrap_get_columns_in_relation(node) -%} 2 | {{ return(adapter.get_columns_in_relation(node)) }} 3 | {% endmacro %} -------------------------------------------------------------------------------- /macros/internal/helpers/prepend_generated_by.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro prepend_generated_by() -%} 7 | -- Generated by AutomateDV (formerly known as dbtvault) 8 | {%- endmacro -%} -------------------------------------------------------------------------------- /dbt_project.yml: -------------------------------------------------------------------------------- 1 | name: automate_dv 2 | version: 0.11.4 3 | require-dbt-version: [ ">=1.0.0", "<3.0.0" ] 4 | config-version: 2 5 | 6 | model-paths: [ "models" ] 7 | analysis-paths: [ "analyses" ] 8 | test-paths: [ "tests" ] 9 | seed-paths: [ "seeds" ] 10 | macro-paths: [ "macros" ] 11 | docs-paths: [ "docs" ] 12 | 13 | target-path: "target" 14 | clean-targets: 15 | - "target" 16 | - "dbt_packages" 17 | -------------------------------------------------------------------------------- /macros/internal/helpers/repeat.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro repeat(str_to_repeat, desired_length=0) -%} 7 | 8 | {%- set repeated_string = str_to_repeat * desired_length -%} 9 | 10 | {%- do return(repeated_string | string) -%} 11 | 12 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/supporting/data_types/type_timestamp.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro type_timestamp() -%} 7 | {{- return(adapter.dispatch('type_timestamp', 'automate_dv')()) -}} 8 | {%- endmacro -%} 9 | 10 | {%- macro default__type_timestamp() -%} 11 | TIMESTAMP_NTZ 12 | {%- endmacro -%} 13 | 14 | {%- macro sqlserver__type_timestamp() -%} 15 | DATETIME2 16 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/materialisations/shared_helpers.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro check_placeholder(model_sql, placeholder='__PERIOD_FILTER__') -%} 7 | 8 | {%- if model_sql.find(placeholder) == -1 -%} 9 | {%- set error_message -%} 10 | Model '{{ model.unique_id }}' does not include the required string '{{ placeholder }}' in its sql 11 | {%- endset -%} 12 | 13 | {{- exceptions.raise_compiler_error(error_message) -}} 14 | {%- endif -%} 15 | 16 | {%- endmacro -%} -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | AutomateDV (f.k.a dbtvault) 2 | Copyright 2019-2025 Business Thinking Ltd (Trading as Datavault). 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. -------------------------------------------------------------------------------- /macros/tables/bigquery/ref_table.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro bigquery__ref_table(src_pk, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__ref_table(src_pk=src_pk, 9 | src_extra_columns=src_extra_columns, 10 | src_ldts=src_ldts, 11 | src_source=src_source, 12 | source_model=source_model) -}} 13 | 14 | {%- endmacro -%} 15 | -------------------------------------------------------------------------------- /macros/tables/postgres/ref_table.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro postgres__ref_table(src_pk, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__ref_table(src_pk=src_pk, 9 | src_extra_columns=src_extra_columns, 10 | src_ldts=src_ldts, 11 | src_source=src_source, 12 | source_model=source_model) -}} 13 | 14 | {%- endmacro -%} 15 | -------------------------------------------------------------------------------- /macros/tables/sqlserver/ref_table.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro sqlserver__ref_table(src_pk, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__ref_table(src_pk=src_pk, 9 | src_extra_columns=src_extra_columns, 10 | src_ldts=src_ldts, 11 | src_source=src_source, 12 | source_model=source_model) -}} 13 | 14 | {%- endmacro -%} 15 | -------------------------------------------------------------------------------- /macros/tables/bigquery/link.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro bigquery__link(src_pk, src_fk, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__link(src_pk=src_pk, 9 | src_fk=src_fk, 10 | src_extra_columns=src_extra_columns, 11 | src_ldts=src_ldts, 12 | src_source=src_source, 13 | source_model=source_model) -}} 14 | 15 | {%- endmacro -%} 16 | -------------------------------------------------------------------------------- /macros/staging/source_columns.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro source_columns(source_relation=none) -%} 7 | 8 | {%- if source_relation -%} 9 | {%- set source_model_cols = adapter.get_columns_in_relation(source_relation) -%} 10 | 11 | {%- set column_list = [] -%} 12 | 13 | {%- for source_col in source_model_cols -%} 14 | {%- do column_list.append(source_col.column) -%} 15 | {%- endfor -%} 16 | 17 | {%- do return(column_list) -%} 18 | {%- endif %} 19 | 20 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/tables/bigquery/t_link.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro bigquery__t_link(src_pk, src_fk, src_payload, src_extra_columns, src_eff, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__t_link(src_pk=src_pk, src_fk=src_fk, src_payload=src_payload, 9 | src_extra_columns=src_extra_columns, 10 | src_eff=src_eff, src_ldts=src_ldts, src_source=src_source, 11 | source_model=source_model) -}} 12 | 13 | {%- endmacro -%} 14 | -------------------------------------------------------------------------------- /macros/tables/postgres/t_link.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro postgres__t_link(src_pk, src_fk, src_payload, src_extra_columns, src_eff, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__t_link(src_pk=src_pk, src_fk=src_fk, src_payload=src_payload, 9 | src_extra_columns=src_extra_columns, 10 | src_eff=src_eff, src_ldts=src_ldts, src_source=src_source, 11 | source_model=source_model) -}} 12 | 13 | {%- endmacro -%} 14 | -------------------------------------------------------------------------------- /macros/tables/databricks/xts.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro databricks__xts(src_pk, src_satellite, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__xts(src_pk=src_pk, 9 | src_satellite=src_satellite, 10 | src_extra_columns=src_extra_columns, 11 | src_ldts=src_ldts, 12 | src_source=src_source, 13 | source_model=source_model) -}} 14 | 15 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/tables/sqlserver/t_link.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro sqlserver__t_link(src_pk, src_fk, src_payload, src_extra_columns, src_eff, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__t_link(src_pk=src_pk, src_fk=src_fk, src_payload=src_payload, 9 | src_extra_columns=src_extra_columns, 10 | src_eff=src_eff, src_ldts=src_ldts, src_source=src_source, 11 | source_model=source_model) -}} 12 | 13 | {%- endmacro -%} 14 | -------------------------------------------------------------------------------- /macros/tables/sqlserver/xts.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro sqlserver__xts(src_pk, src_satellite, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__xts(src_pk=src_pk, 9 | src_satellite=src_satellite, 10 | src_extra_columns=src_extra_columns, 11 | src_ldts=src_ldts, 12 | src_source=src_source, 13 | source_model=source_model) -}} 14 | 15 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/tables/bigquery/xts.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro bigquery__xts(src_pk, src_satellite, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__xts(src_pk=src_pk, 9 | src_satellite=src_satellite, 10 | src_extra_columns=src_extra_columns, 11 | src_ldts=src_ldts, 12 | src_source=src_source, 13 | source_model=source_model) -}} 14 | 15 | {%- endmacro -%} 16 | -------------------------------------------------------------------------------- /macros/tables/postgres/xts.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro postgres__xts(src_pk, src_satellite, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__xts(src_pk=src_pk, 9 | src_satellite=src_satellite, 10 | src_extra_columns=src_extra_columns, 11 | src_ldts=src_ldts, 12 | src_source=src_source, 13 | source_model=source_model) -}} 14 | 15 | {%- endmacro -%} 16 | -------------------------------------------------------------------------------- /macros/tables/sqlserver/sat.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro sqlserver__sat(src_pk, src_hashdiff, src_payload, src_extra_columns, src_eff, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.postgres__sat(src_pk=src_pk, src_hashdiff=src_hashdiff, 9 | src_payload=src_payload, src_extra_columns=src_extra_columns, 10 | src_eff=src_eff, src_ldts=src_ldts, 11 | src_source=src_source, source_model=source_model) -}} 12 | 13 | {%- endmacro -%} 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "[FEATURE]" 5 | labels: feature 6 | assignees: DVAlexHiggs 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /macros/internal/helpers/stage_processing_macros/extract_column_names.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro extract_column_names(columns_dict=none) -%} 7 | 8 | {%- set extracted_column_names = [] -%} 9 | 10 | {%- if columns_dict is mapping -%} 11 | {%- for key, value in columns_dict.items() -%} 12 | {%- do extracted_column_names.append(key) -%} 13 | {%- endfor -%} 14 | 15 | {%- do return(extracted_column_names) -%} 16 | {%- else -%} 17 | {%- do return([]) -%} 18 | {%- endif -%} 19 | 20 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/tables/bigquery/sat.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro bigquery__sat(src_pk, src_hashdiff, src_payload, src_extra_columns, src_eff, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__sat(src_pk=src_pk, src_hashdiff=src_hashdiff, 9 | src_payload=src_payload, 10 | src_extra_columns=src_extra_columns, 11 | src_eff=src_eff, src_ldts=src_ldts, 12 | src_source=src_source, source_model=source_model) -}} 13 | 14 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/internal/helpers/stage_processing_macros/print_list.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro print_list(list_to_print=none, indent=4, columns_to_escape=none) -%} 7 | 8 | {%- for col_name in list_to_print -%} 9 | {%- if col_name | lower in columns_to_escape | map('lower') | list -%} 10 | {{- automate_dv.escape_column_name(col_name) | indent(indent) -}}{{ ",\n " if not loop.last }} 11 | {%- else -%} 12 | {{- col_name | indent(indent) -}}{{ ",\n " if not loop.last }} 13 | {%- endif -%} 14 | {%- endfor -%} 15 | 16 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/tables/postgres/ma_sat.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro postgres__ma_sat(src_pk, src_cdk, src_hashdiff, src_payload, src_extra_columns, src_eff, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__ma_sat(src_pk=src_pk, src_cdk=src_cdk, 9 | src_hashdiff=src_hashdiff, src_payload=src_payload, 10 | src_extra_columns=src_extra_columns, src_eff=src_eff, 11 | src_ldts=src_ldts, src_source=src_source, 12 | source_model=source_model) -}} 13 | 14 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/tables/bigquery/ma_sat.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro bigquery__ma_sat(src_pk, src_cdk, src_hashdiff, src_payload, src_extra_columns, src_eff, src_ldts, src_source, source_model) %} 7 | 8 | {{- automate_dv.default__ma_sat(src_pk=src_pk, src_cdk=src_cdk, 9 | src_hashdiff=src_hashdiff, src_payload=src_payload, 10 | src_extra_columns=src_extra_columns, src_eff=src_eff, 11 | src_ldts=src_ldts, src_source=src_source, 12 | source_model=source_model) -}} 13 | 14 | {%- endmacro -%} 15 | -------------------------------------------------------------------------------- /macros/supporting/hash_components/get_hash_string_length.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro get_hash_string_length(hash) -%} 7 | 8 | {%- set enable_native_hashes = var('enable_native_hashes', false) -%} 9 | 10 | {%- if hash | lower == 'md5' -%} 11 | {%- set hash_length = 32 -%} 12 | {%- elif hash | lower == 'sha' -%} 13 | {%- set hash_length = 64 -%} 14 | {%- elif hash | lower == 'sha1' -%} 15 | {%- set hash_length = 40 -%} 16 | {%- else -%} 17 | {%- set hash_length = 32 -%} 18 | {%- endif -%} 19 | 20 | {%- do return(hash_length) -%} 21 | 22 | {%- endmacro %} -------------------------------------------------------------------------------- /macros/tables/sqlserver/ma_sat.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro sqlserver__ma_sat(src_pk, src_cdk, src_hashdiff, src_payload, src_extra_columns, src_eff, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__ma_sat(src_pk=src_pk, src_cdk=src_cdk, 9 | src_hashdiff=src_hashdiff, src_payload=src_payload, 10 | src_extra_columns=src_extra_columns, src_eff=src_eff, 11 | src_ldts=src_ldts, src_source=src_source, 12 | source_model=source_model) -}} 13 | 14 | {%- endmacro -%} 15 | -------------------------------------------------------------------------------- /macros/tables/databricks/ma_sat.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro databricks__ma_sat(src_pk, src_cdk, src_hashdiff, src_payload, src_extra_columns, src_eff, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__ma_sat(src_pk=src_pk, src_cdk=src_cdk, 9 | src_hashdiff=src_hashdiff, src_payload=src_payload, 10 | src_extra_columns=src_extra_columns, src_eff=src_eff, 11 | src_ldts=src_ldts, src_source=src_source, 12 | source_model=source_model) -}} 13 | 14 | {%- endmacro -%} 15 | -------------------------------------------------------------------------------- /macros/supporting/get_query_results_as_dict.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {% macro get_query_results_as_dict(query) %} 7 | {{ return(adapter.dispatch('get_query_results_as_dict', 'automate_dv')(query)) }} 8 | {% endmacro %} 9 | 10 | {% macro default__get_query_results_as_dict(query) %} 11 | {%- set query_results = dbt_utils.get_query_results_as_dict(query) -%} 12 | {%- set query_results_processed = {} -%} 13 | 14 | {% for k, v in query_results.items() %} 15 | {% do query_results_processed.update({k.upper(): v}) %} 16 | {% endfor %} 17 | 18 | {{ return(query_results_processed) }} 19 | {% endmacro %} 20 | -------------------------------------------------------------------------------- /macros/tables/databricks/pit.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro databricks__pit(src_pk, src_extra_columns, as_of_dates_table, satellites, stage_tables_ldts, src_ldts, source_model) -%} 7 | 8 | {{- automate_dv.default__pit(src_pk=src_pk, 9 | src_extra_columns=src_extra_columns, 10 | as_of_dates_table=as_of_dates_table, 11 | satellites=satellites, 12 | stage_tables_ldts=stage_tables_ldts, 13 | src_ldts=src_ldts, 14 | source_model=source_model) -}} 15 | 16 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/tables/sqlserver/pit.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro sqlserver__pit(src_pk, src_extra_columns, as_of_dates_table, satellites, stage_tables_ldts, src_ldts, source_model) -%} 7 | 8 | {{- automate_dv.default__pit(src_pk=src_pk, 9 | src_extra_columns=src_extra_columns, 10 | as_of_dates_table=as_of_dates_table, 11 | satellites=satellites, 12 | stage_tables_ldts=stage_tables_ldts, 13 | src_ldts=src_ldts, 14 | source_model=source_model) -}} 15 | 16 | {%- endmacro -%} 17 | -------------------------------------------------------------------------------- /macros/tables/bigquery/eff_sat.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro bigquery__eff_sat(src_pk, src_dfk, src_sfk, src_extra_columns, src_start_date, src_end_date, src_eff, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__eff_sat(src_pk=src_pk, src_dfk=src_dfk, src_sfk=src_sfk, 9 | src_extra_columns=src_extra_columns, 10 | src_start_date=src_start_date, src_end_date=src_end_date, 11 | src_eff=src_eff, src_ldts=src_ldts, src_source=src_source, 12 | source_model=source_model) -}} 13 | 14 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/tables/postgres/eff_sat.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro postgres__eff_sat(src_pk, src_dfk, src_sfk, src_extra_columns, src_start_date, src_end_date, src_eff, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__eff_sat(src_pk=src_pk, src_dfk=src_dfk, src_sfk=src_sfk, 9 | src_extra_columns=src_extra_columns, 10 | src_start_date=src_start_date, src_end_date=src_end_date, 11 | src_eff=src_eff, src_ldts=src_ldts, src_source=src_source, 12 | source_model=source_model) -}} 13 | 14 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/tables/sqlserver/eff_sat.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro sqlserver__eff_sat(src_pk, src_dfk, src_sfk, src_extra_columns, src_start_date, src_end_date, src_eff, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.default__eff_sat(src_pk=src_pk, src_dfk=src_dfk, src_sfk=src_sfk, 9 | src_extra_columns=src_extra_columns, 10 | src_start_date=src_start_date, src_end_date=src_end_date, 11 | src_eff=src_eff, src_ldts=src_ldts, src_source=src_source, 12 | source_model=source_model) -}} 13 | 14 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/internal/metadata_processing/check_required_parameters.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro check_required_parameters() -%} 7 | 8 | {%- set ns = namespace(missing_parameters=[]) -%} 9 | 10 | {%- if kwargs is not none -%} 11 | 12 | {%- for k, v in kwargs.items() %} 13 | {%- do ns.missing_parameters.append(k) if v is none -%} 14 | {%- endfor -%} 15 | 16 | {%- if ns.missing_parameters -%} 17 | {{- exceptions.raise_compiler_error("Required parameter(s) missing or none in '{}': {}".format(this, ns.missing_parameters | join(", "))) -}} 18 | {%- endif -%} 19 | {%- endif -%} 20 | 21 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/tables/postgres/bridge.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro postgres__bridge(src_pk, as_of_dates_table, bridge_walk, stage_tables_ldts, src_extra_columns, src_ldts, source_model) -%} 7 | 8 | {{- automate_dv.default__bridge(src_pk=src_pk, 9 | as_of_dates_table=as_of_dates_table, 10 | bridge_walk=bridge_walk, 11 | stage_tables_ldts=stage_tables_ldts, 12 | src_extra_columns=src_extra_columns, 13 | src_ldts=src_ldts, 14 | source_model=source_model) -}} 15 | 16 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/tables/bigquery/bridge.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro bigquery__bridge(src_pk, as_of_dates_table, bridge_walk, stage_tables_ldts, src_extra_columns, src_ldts, source_model) -%} 7 | 8 | {{- automate_dv.default__bridge(src_pk=src_pk, 9 | as_of_dates_table=as_of_dates_table, 10 | bridge_walk=bridge_walk, 11 | stage_tables_ldts=stage_tables_ldts, 12 | src_extra_columns=src_extra_columns, 13 | src_ldts=src_ldts, 14 | source_model=source_model) -}} 15 | 16 | {%- endmacro -%} 17 | -------------------------------------------------------------------------------- /macros/tables/bigquery/pit.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro bigquery__pit(src_pk, src_extra_columns, as_of_dates_table, satellites, stage_tables_ldts, src_ldts, source_model) %} 7 | 8 | {{- automate_dv.default__pit(src_pk=src_pk, 9 | src_extra_columns=src_extra_columns, 10 | as_of_dates_table=as_of_dates_table, 11 | satellites=satellites, 12 | stage_tables_ldts=stage_tables_ldts, 13 | src_ldts=src_ldts, 14 | source_model=source_model) -}} 15 | 16 | {%- endmacro -%} 17 | -------------------------------------------------------------------------------- /macros/tables/databricks/bridge.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro databricks__bridge(src_pk, src_extra_columns, as_of_dates_table, bridge_walk, stage_tables_ldts, src_ldts, source_model) -%} 7 | 8 | {{- automate_dv.default__bridge(src_pk=src_pk, 9 | src_extra_columns=src_extra_columns, 10 | src_ldts=src_ldts, 11 | as_of_dates_table=as_of_dates_table, 12 | bridge_walk=bridge_walk, 13 | stage_tables_ldts=stage_tables_ldts, 14 | source_model=source_model) -}} 15 | 16 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/tables/sqlserver/bridge.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro sqlserver__bridge(src_pk, as_of_dates_table, bridge_walk, stage_tables_ldts, src_extra_columns, src_ldts, source_model) -%} 7 | 8 | {{- automate_dv.default__bridge(src_pk=src_pk, 9 | as_of_dates_table=as_of_dates_table, 10 | bridge_walk=bridge_walk, 11 | stage_tables_ldts=stage_tables_ldts, 12 | src_extra_columns=src_extra_columns, 13 | src_ldts=src_ldts, 14 | source_model=source_model) -}} 15 | 16 | {%- endmacro -%} 17 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[BUG] " 5 | labels: bug 6 | assignees: DVAlexHiggs 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Environment** 14 | 15 | dbt version: 16 | automate_dv version: 17 | Database/Platform: 18 | 19 | **To Reproduce** 20 | Steps to reproduce the behavior: 21 | 1. Go to '...' 22 | 2. Click on '....' 23 | 3. Scroll down to '....' 24 | 4. See error 25 | 26 | **Expected behavior** 27 | A clear and concise description of what you expected to happen. 28 | 29 | **Screenshots** 30 | If applicable, add screenshots to help explain your problem. 31 | 32 | **Log files** 33 | If applicable, provide dbt log files which include the problem. 34 | 35 | **Additional context** 36 | Add any other context about the problem here. 37 | -------------------------------------------------------------------------------- /macros/supporting/ghost_records/null_ghost.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro null_ghost(data_type, alias) -%} 7 | {{ adapter.dispatch('null_ghost', 'automate_dv')(data_type=data_type, alias=alias) }} 8 | {%- endmacro -%} 9 | 10 | {%- macro default__null_ghost(data_type, alias) -%} 11 | NULL AS {{alias}} 12 | {%- endmacro -%} 13 | 14 | {% macro bigquery__null_ghost(data_type, alias) -%} 15 | CAST(NULL AS {{data_type}}) AS {{alias}} 16 | {%- endmacro -%} 17 | 18 | {%- macro postgres__null_ghost(data_type, alias) -%} 19 | {{ automate_dv.bigquery__null_ghost(data_type, alias) }} 20 | {%- endmacro -%} 21 | 22 | {%- macro sqlserver__null_ghost(data_type, alias) -%} 23 | {{ automate_dv.bigquery__null_ghost(data_type, alias) }} 24 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/materialisations/drop_temporary.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {% macro drop_temporary_special(tmp_relation) %} 7 | {# In databricks and sqlserver a temporary view/table can only be dropped by #} 8 | {# the connection or session that created it so drop it now before the commit below closes this session #} 9 | 10 | {%- set drop_query_name = 'DROP_QUERY-' ~ i -%} 11 | {% call statement(drop_query_name, fetch_result=True) -%} 12 | {% if target.type == 'databricks' %} 13 | DROP VIEW {{ tmp_relation }}; 14 | {% elif target.type == 'sqlserver' %} 15 | DROP TABLE {{ tmp_relation }}; 16 | {% elif target.type == 'postgres' %} 17 | DROP TABLE {{ tmp_relation }}; 18 | {% endif %} 19 | {%- endcall %} 20 | 21 | {% endmacro %} -------------------------------------------------------------------------------- /macros/supporting/max_datetime.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro max_datetime() -%} 7 | 8 | {%- do return(adapter.dispatch('max_datetime', 'automate_dv')()) -%} 9 | 10 | {%- endmacro %} 11 | 12 | {%- macro default__max_datetime() %} 13 | 14 | {%- do return(var('max_datetime', '9999-12-31 23:59:59.999999')) -%} 15 | 16 | {%- endmacro -%} 17 | 18 | {%- macro sqlserver__max_datetime() -%} 19 | 20 | {%- do return(var('max_datetime', '9999-12-31 23:59:59.9999999')) -%} 21 | 22 | {%- endmacro -%} 23 | 24 | {%- macro bigquery__max_datetime() -%} 25 | 26 | {%- do return(var('max_datetime', '9999-12-31 23:59:59.999999')) -%} 27 | 28 | {%- endmacro -%} 29 | 30 | {%- macro postgres__max_datetime() %} 31 | 32 | {%- do return(var('max_datetime', '9999-12-31 23:59:59.999')) -%} 33 | 34 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/internal/helpers/dateadd.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {% macro dateadd(datepart, interval, from_date_or_timestamp) %} 7 | {{ return(adapter.dispatch('dateadd', 'automate_dv')(datepart=datepart, 8 | interval=interval, 9 | from_date_or_timestamp=from_date_or_timestamp)) }} 10 | {%- endmacro -%} 11 | 12 | {% macro default__dateadd(datepart, interval, from_date_or_timestamp) %} 13 | 14 | {{ dateadd(datepart, interval, from_date_or_timestamp) }} 15 | 16 | {% endmacro %} 17 | 18 | {% macro sqlserver__dateadd(datepart, interval, from_date_or_timestamp) %} 19 | 20 | dateadd( 21 | {{ datepart }}, 22 | {{ interval }}, 23 | CAST({{ from_date_or_timestamp }} AS DATETIME2) 24 | ) 25 | 26 | {% endmacro %} -------------------------------------------------------------------------------- /macros/internal/metadata_processing/as_constant.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro as_constant(column_str=none) -%} 7 | 8 | {{- adapter.dispatch('as_constant', 'automate_dv')(column_str=column_str) -}} 9 | 10 | {%- endmacro %} 11 | 12 | {%- macro default__as_constant(column_str) -%} 13 | 14 | {%- if column_str is not none and column_str is string and column_str -%} 15 | 16 | {%- if column_str | first == "!" -%} 17 | 18 | {{- return("'" ~ column_str[1:] ~ "'") -}} 19 | 20 | {%- else -%} 21 | 22 | {{- return(column_str) -}} 23 | 24 | {%- endif -%} 25 | {%- else -%} 26 | {%- if execute -%} 27 | {{ exceptions.raise_compiler_error("Invalid columns_str object provided. Must be a string and not null.") }} 28 | {%- endif %} 29 | {%- endif -%} 30 | 31 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/internal/metadata_processing/escape_column_name.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro escape_column_name(column) -%} 7 | 8 | {{- adapter.dispatch('escape_column_name', 'automate_dv')(column=column) -}} 9 | 10 | {%- endmacro %} 11 | 12 | {%- macro default__escape_column_name(column) -%} 13 | 14 | {# Do not escape a constant (single quoted) value #} 15 | {%- if column | first == "'" and column | last == "'" -%} 16 | {%- set escaped_column_name = column -%} 17 | {%- else -%} 18 | {%- set escape_char_left, escape_char_right = automate_dv.get_escape_characters() -%} 19 | 20 | {%- set escaped_column_name = escape_char_left ~ column | replace(escape_char_left, '') | replace(escape_char_right, '') | trim ~ escape_char_right -%} 21 | {%- endif -%} 22 | 23 | {%- do return(escaped_column_name) -%} 24 | 25 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/supporting/fusion_compat.sql: -------------------------------------------------------------------------------- 1 | {% macro config_meta_get(key, default=none) %} 2 | {%- if config.get(key) != none -%} 3 | {{ return(config.get(key)) }} 4 | {%- elif config.get("meta") != none and (key in config.get("meta", {}).keys()) -%} 5 | {{ return(config.get("meta").get(key)) }} 6 | {%- else -%} 7 | {{ return(default) }} 8 | {%- endif -%} 9 | {% endmacro %} 10 | 11 | 12 | {% macro config_meta_require(key) %} 13 | {# the first case is required to avoid errors #} 14 | {%- if config == {} -%} 15 | {{ return(none) }} 16 | {%- elif config.get(key) != none -%} 17 | {{ return(config.get(key)) }} 18 | {%- elif config.get("meta") != none and (key in config.get("meta", {}).keys()) -%} 19 | {{ return(config.get("meta").get(key)) }} 20 | {%- else -%} 21 | {% do exceptions.raise_compiler_error("Configuration '" ~ key ~ "' is required but was not found under config or meta (Fusion requires custom configuration under meta)") %} 22 | {%- endif -%} 23 | {% endmacro %} -------------------------------------------------------------------------------- /macros/tables/databricks/ref_table.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro databricks__ref_table(src_pk, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_extra_columns, src_ldts, src_source]) %} 9 | 10 | WITH source_data AS ( 11 | {%- for src in source_model %} 12 | SELECT DISTINCT 13 | {{ automate_dv.prefix(source_cols, 'a') }} 14 | FROM {{ ref(src) }} AS a 15 | WHERE a.{{ src_pk }} IS NOT NULL 16 | {%- endfor %} 17 | ), 18 | 19 | records_to_insert AS ( 20 | SELECT 21 | {{ automate_dv.prefix(source_cols, 'a') }} 22 | FROM source_data AS a 23 | {%- if automate_dv.is_any_incremental() %} 24 | LEFT ANTI JOIN {{ this }} AS d 25 | ON {{ automate_dv.multikey(src_pk, prefix=['a','d'], condition='=') }} 26 | {%- endif %} 27 | ) 28 | 29 | SELECT * FROM records_to_insert 30 | 31 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/supporting/ghost_records/date_ghost.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro date_ghost(date_type, alias) -%} 7 | {{ adapter.dispatch('date_ghost', 'automate_dv')(date_type=date_type, alias=alias) }} 8 | {%- endmacro -%} 9 | 10 | {%- macro default__date_ghost(date_type, alias=none) -%} 11 | 12 | {%- if date_type == 'date' -%} 13 | {{ automate_dv.cast_date('1900-01-01', as_string=true, datetime=false, alias=alias) }} 14 | {%- else -%} 15 | {{ automate_dv.cast_date('1900-01-01 00:00:00', as_string=true, datetime=true, alias=alias, date_type=date_type) }} 16 | {%- endif -%} 17 | 18 | {%- endmacro -%} 19 | 20 | {%- macro postgres__date_ghost(date_type, alias=none) -%} 21 | 22 | {%- if date_type == 'date' -%} 23 | {{ automate_dv.cast_date('1900-01-01', as_string=true, datetime=false, alias=alias) }} 24 | {%- else -%} 25 | to_char(timestamp '1900-01-01 00:00:00', 'YYYY-MM-DD HH24:MI:SS.MS')::timestamp {%- if alias %} AS {{alias}}{%- endif -%} 26 | {%- endif -%} 27 | 28 | {%- endmacro -%} 29 | -------------------------------------------------------------------------------- /macros/supporting/data_types/type_string.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro type_string(is_hash=false, char_length=255) -%} 7 | {{- return(adapter.dispatch('type_string', 'automate_dv')(is_hash=is_hash, char_length=char_length)) -}} 8 | {%- endmacro -%} 9 | 10 | {%- macro default__type_string(is_hash, char_length) -%} 11 | VARCHAR 12 | {%- endmacro -%} 13 | 14 | {%- macro bigquery__type_string(is_hash, char_length) -%} 15 | STRING 16 | {%- endmacro -%} 17 | 18 | {%- macro sqlserver__type_string(is_hash, char_length) -%} 19 | VARCHAR 20 | {%- endmacro -%} 21 | 22 | {%- macro databricks__type_string(is_hash=false, char_length=255) -%} 23 | {%- if is_hash -%} 24 | {%- if var('hash', 'MD5') | lower == 'md5' -%} 25 | VARCHAR(16) 26 | {%- elif var('hash', 'MD5') | lower == 'sha' -%} 27 | VARCHAR(32) 28 | {%- elif var('hash', 'MD5') | lower == 'sha1' -%} 29 | VARCHAR(20) 30 | {%- endif -%} 31 | {%- else -%} 32 | VARCHAR({{ char_length }}) 33 | {%- endif -%} 34 | {%- endmacro -%} 35 | -------------------------------------------------------------------------------- /macros/internal/helpers/stage_processing_macros/process_columns_to_select.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro process_columns_to_select(columns_list=none, exclude_columns_list=none) -%} 7 | 8 | {% set columns_list = columns_list | list %} 9 | {% set exclude_columns_list = exclude_columns_list | list %} 10 | 11 | {% set columns_to_select = [] %} 12 | 13 | {% if not automate_dv.is_list(columns_list) or not automate_dv.is_list(exclude_columns_list) %} 14 | 15 | {{- exceptions.raise_compiler_error("One or both arguments are not of list type.") -}} 16 | 17 | {%- endif -%} 18 | 19 | {%- if automate_dv.is_something(columns_list) and automate_dv.is_something(exclude_columns_list) -%} 20 | 21 | {%- for col in columns_list -%} 22 | 23 | {%- if (col | upper) not in (exclude_columns_list | map('upper') | list) -%} 24 | {%- do columns_to_select.append(col) -%} 25 | {%- endif -%} 26 | 27 | {%- endfor -%} 28 | 29 | {%- endif -%} 30 | 31 | {%- do return(columns_to_select) -%} 32 | 33 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/materialisations/period_mat_helpers/error_messages.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2023 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro sqlserver_datepart_too_small_error(period) -%} 7 | 8 | {%- set error_message -%} 9 | This datepart ({{ period }}) is too small and cannot be used for this purpose in MS SQL Server, consider using a different datepart value (e.g. day). 10 | vault_insert_by materialisations are not intended for this purpose, please see https://automate-dv.readthedocs.io/en/latest/materialisations/ 11 | {%- endset -%} 12 | 13 | {{- exceptions.raise_compiler_error(error_message) -}} 14 | 15 | {%- endmacro -%} 16 | 17 | {%- macro sqlserver_max_iterations_error() -%} 18 | 19 | {%- set error_message -%} 20 | Max iterations is 100,000. Consider using a different datepart value (e.g. day) or loading data for a shorter time period. 21 | vault_insert_by materialisations are not intended for this purpose, please see https://automate-dv.readthedocs.io/en/latest/materialisations/ 22 | {%- endset -%} 23 | 24 | {{- exceptions.raise_compiler_error(error_message) -}} 25 | 26 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/internal/helpers/stage_processing_macros/extract_null_column_names.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro extract_null_column_names(columns_dict=none) -%} 7 | 8 | {%- set extracted_column_names = [] -%} 9 | 10 | {%- if columns_dict is mapping -%} 11 | {%- for key, value in columns_dict.items() -%} 12 | {%- if automate_dv.is_something(value) -%} 13 | {% if automate_dv.is_list(value) %} 14 | {% for col_name in value %} 15 | {%- do extracted_column_names.append(col_name) -%} 16 | {%- do extracted_column_names.append(col_name ~ "_ORIGINAL") -%} 17 | {% endfor %} 18 | {% else %} 19 | {%- do extracted_column_names.append(value) -%} 20 | {%- do extracted_column_names.append(value ~ "_ORIGINAL") -%} 21 | {% endif %} 22 | {%- endif -%} 23 | {%- endfor -%} 24 | 25 | {%- do return(extracted_column_names) -%} 26 | {%- else -%} 27 | {%- do return([]) -%} 28 | {%- endif -%} 29 | 30 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/supporting/casting/cast_binary.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro cast_binary(column_str, alias=none, quote=true) -%} 7 | {{ return(adapter.dispatch('cast_binary', 'automate_dv')(column_str=column_str, alias=alias, quote=quote)) }} 8 | {%- endmacro -%} 9 | 10 | 11 | {%- macro default__cast_binary(column_str, alias=none, quote=true) -%} 12 | 13 | {%- if quote -%} 14 | CAST('{{ column_str }}' AS {{ automate_dv.type_binary() }}) 15 | {%- else -%} 16 | CAST({{ column_str }} AS {{ automate_dv.type_binary() }}) 17 | {%- endif -%} 18 | 19 | {%- if alias %} AS {{ alias }} {%- endif -%} 20 | 21 | {%- endmacro -%} 22 | 23 | 24 | {%- macro sqlserver__cast_binary(column_str, alias=none, quote=true) -%} 25 | 26 | {%- if quote -%} 27 | CONVERT({{ automate_dv.type_binary() }}, '{{ column_str }}', 2) 28 | {%- else -%} 29 | CONVERT({{ automate_dv.type_binary() }}, {{ column_str }}, 2) 30 | {%- endif -%} 31 | 32 | {% if alias %} AS {{ alias }} {%- endif %} 33 | 34 | {%- endmacro -%} 35 | 36 | {%- macro bigquery__cast_binary(column_str, alias=none, quote=true) -%} 37 | 38 | {{ automate_dv.default__cast_binary(column_str=column_str, alias=alias, quote=quote) }} 39 | 40 | {%- endmacro -%} 41 | 42 | -------------------------------------------------------------------------------- /macros/internal/helpers/logging/log_relation_sources.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {% macro log_relation_sources(relation, source_count) %} 7 | {{ return(adapter.dispatch('log_relation_sources', 'automate_dv')(relation=relation, source_count=source_count)) }} 8 | {%- endmacro -%} 9 | 10 | {% macro default__log_relation_sources(relation, source_count) %} 11 | 12 | {%- if execute and automate_dv.is_something(invocation_args_dict.get('which')) and invocation_args_dict.get('which') != 'docs' -%} 13 | 14 | {%- do dbt_utils.log_info('Loading {} from {} source(s)'.format("{}.{}.{}".format(relation.database, relation.schema, relation.identifier), 15 | source_count)) -%} 16 | {%- endif -%} 17 | {% endmacro %} 18 | 19 | {% macro databricks__log_relation_sources(relation, source_count) %} 20 | 21 | {%- if execute and automate_dv.is_something(invocation_args_dict.get('which')) and invocation_args_dict.get('which') != 'docs' -%} 22 | 23 | {%- do dbt_utils.log_info('Loading {} from {} source(s)'.format("{}.{}".format(relation.schema, relation.identifier), 24 | source_count)) -%} 25 | {%- endif -%} 26 | {% endmacro %} -------------------------------------------------------------------------------- /macros/supporting/ghost_records/ghost_for_type.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro ghost_for_type(col_type, col_name) -%} 7 | 8 | {%- set hash = var('hash', 'MD5') -%} 9 | {%- set binary_types = [automate_dv.type_binary(for_dbt_compare=true)] | map('lower') | list -%} 10 | {%- set string_types = [dbt.type_string() | lower, 'varchar', 'character varying', 'char', 'string', 11 | 'text', 'ntext', 'nvarchar', 'nchar'] -%} 12 | {%- set time_types = [dbt.type_timestamp() | lower, 'date', 'datetime', 'datetime2', 'timestamp', 13 | 'timestamp_ltz', 'timestamp_ntz', 'timestamp_tz', 'timestamp with time zone', 14 | 'timestamp without time zone', 'datetimeoffset', 'smalldatetime', 'time', 15 | 'time with timezone'] -%} 16 | 17 | {%- if col_type in binary_types -%} 18 | {%- do return(automate_dv.binary_ghost(alias=col_name, hash=hash)) -%} 19 | {%- elif col_type in string_types -%} 20 | {%- do return(automate_dv.null_ghost(data_type=col_type, alias=col_name)) -%} 21 | {%- elif col_type in time_types -%} 22 | {%- do return(automate_dv.date_ghost(date_type=col_type, alias=col_name)) -%} 23 | {%- else -%} 24 | {%- do return(automate_dv.null_ghost(data_type=col_type, alias=col_name)) -%} 25 | {%- endif -%} 26 | 27 | {% endmacro %} 28 | -------------------------------------------------------------------------------- /.github/workflows/ado_sync.yml: -------------------------------------------------------------------------------- 1 | name: Sync issue to Azure DevOps work item 2 | 3 | on: 4 | issues: 5 | types: 6 | [opened, edited, deleted, closed, reopened, labeled, unlabeled, assigned] 7 | issue_comment: 8 | types: [created, edited, deleted] 9 | 10 | concurrency: 11 | group: issue-${{ github.event.issue.number }} 12 | cancel-in-progress: false 13 | 14 | jobs: 15 | alert: 16 | if: ${{ !github.event.issue.pull_request }} 17 | runs-on: ubuntu-latest 18 | steps: 19 | - uses: danhellem/github-actions-issue-to-work-item@master 20 | env: 21 | ado_token: "${{ secrets.ADO_PERSONAL_ACCESS_TOKEN }}" 22 | github_token: "${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}" 23 | ado_organization: "${{ vars.ADO_ORGANIZATION }}" 24 | ado_project: "${{ vars.ADO_PROJECT_NAME }}" 25 | ado_area_path: "${{ vars.ADO_AREA_PATH }}" 26 | ado_iteration_path: "${{ vars.ADO_ITERATION_PATH }}" 27 | ado_wit: ${{ (contains(github.event.issue.title, 'FEATURE') || contains(github.event.issue.title, 'DOCS')) && 'Product Backlog Item' || 'Bug' }} 28 | ado_new_state: "New" 29 | ado_active_state: ${{ (contains(github.event.issue.title, 'FEATURE') || contains(github.event.issue.title, 'DOCS')) && 'Active' || 'Committed' }} 30 | ado_close_state: ${{ (contains(github.event.issue.title, 'FEATURE') || contains(github.event.issue.title, 'DOCS')) && 'Closed' || 'Done' }} 31 | ado_bypassrules: true 32 | log_level: 100 33 | -------------------------------------------------------------------------------- /macros/internal/metadata_processing/concat_ws.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro concat_ws(string_list, separator="||") -%} 7 | 8 | {{- adapter.dispatch('concat_ws', 'automate_dv')(string_list=string_list, separator=separator) -}} 9 | 10 | {%- endmacro %} 11 | 12 | {%- macro default__concat_ws(string_list, separator="||") -%} 13 | 14 | CONCAT_WS('{{ separator }}', 15 | {%- for str_obj in string_list %} 16 | {{ str_obj }}{% if not loop.last %},{% endif %} 17 | {%- endfor %} 18 | ) 19 | 20 | {%- endmacro -%} 21 | 22 | {%- macro bigquery__concat_ws(string_list, separator="||") -%} 23 | 24 | CONCAT( 25 | {%- for str in string_list %} 26 | {{ str }} 27 | {%- if not loop.last %}, '{{ separator }}', {%- endif -%} 28 | {%- endfor %} 29 | ) 30 | 31 | {%- endmacro -%} 32 | 33 | {%- macro sqlserver__concat_ws(string_list, separator="||") -%} 34 | 35 | {{ automate_dv.default__concat_ws(string_list=string_list, separator=separator) }} 36 | 37 | {%- endmacro -%} 38 | 39 | {%- macro postgres__concat_ws(string_list, separator="||") -%} 40 | 41 | {{ automate_dv.default__concat_ws(string_list=string_list, separator=separator) }} 42 | 43 | {%- endmacro -%} 44 | 45 | {%- macro databricks__concat_ws(string_list, separator="||") -%} 46 | 47 | {{ automate_dv.default__concat_ws(string_list=string_list, separator=separator) }} 48 | 49 | {%- endmacro -%} 50 | -------------------------------------------------------------------------------- /macros/internal/metadata_processing/expand_column_list.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro expand_column_list(columns=none) -%} 7 | 8 | {%- if not columns -%} 9 | {%- do return([]) -%} 10 | {%- endif -%} 11 | 12 | {%- set col_list = [] -%} 13 | 14 | {%- if automate_dv.is_list(columns) -%} 15 | 16 | {%- set columns = columns | reject("none") %} 17 | 18 | {%- for col in columns -%} 19 | 20 | {%- if col is string -%} 21 | 22 | {%- do col_list.append(col) -%} 23 | 24 | {#- If list of lists -#} 25 | {%- elif automate_dv.is_list(col) -%} 26 | 27 | {%- for cols in col -%} 28 | 29 | {%- do col_list.append(cols) -%} 30 | 31 | {%- endfor -%} 32 | {%- elif col is mapping -%} 33 | 34 | {%- do col_list.append(col) -%} 35 | 36 | {%- else -%} 37 | 38 | {%- if execute -%} 39 | {{- exceptions.raise_compiler_error("Invalid columns object provided. Must be a list of lists, dictionaries or strings.") -}} 40 | {%- endif %} 41 | 42 | {%- endif -%} 43 | 44 | {%- endfor -%} 45 | {%- else -%} 46 | 47 | {%- if execute -%} 48 | {{- exceptions.raise_compiler_error("Invalid columns object provided. Must be a list.") -}} 49 | {%- endif %} 50 | 51 | {%- endif -%} 52 | 53 | {%- do return(col_list) -%} 54 | 55 | {%- endmacro -%} 56 | -------------------------------------------------------------------------------- /macros/internal/metadata_processing/alias.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro alias(alias_config=none, prefix=none) -%} 7 | 8 | {{- adapter.dispatch('alias', 'automate_dv')(alias_config=alias_config, prefix=prefix) -}} 9 | 10 | {%- endmacro %} 11 | 12 | {%- macro default__alias(alias_config=none, prefix=none) -%} 13 | 14 | {%- if alias_config is defined and alias_config is not none and alias_config -%} 15 | 16 | {%- if alias_config is mapping -%} 17 | 18 | {%- if alias_config['source_column'] and alias_config['alias'] -%} 19 | 20 | {%- if prefix -%} 21 | {{prefix}}.{{ alias_config['source_column'] }} AS {{ alias_config['alias'] }} 22 | {%- else -%} 23 | {{ alias_config['source_column'] }} AS {{ alias_config['alias'] }} 24 | {%- endif -%} 25 | 26 | {%- endif -%} 27 | 28 | {%- else -%} 29 | 30 | {%- if prefix -%} 31 | 32 | {{- automate_dv.prefix([alias_config], prefix) -}} 33 | 34 | {%- else -%} 35 | 36 | {{ alias_config }} 37 | 38 | {%- endif -%} 39 | 40 | {%- endif -%} 41 | 42 | {%- else -%} 43 | 44 | {%- if execute -%} 45 | 46 | {{ exceptions.raise_compiler_error("Invalid alias configuration:\nexpected format: {source_column: 'column', alias: 'column_alias'}\ngot: " ~ alias_config) }} 47 | 48 | {%- endif -%} 49 | 50 | {%- endif -%} 51 | 52 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/internal/metadata_processing/alias_all.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro alias_all(columns=none, prefix=none) -%} 7 | 8 | {{- adapter.dispatch('alias_all', 'automate_dv')(columns=columns, prefix=prefix) -}} 9 | 10 | {%- endmacro %} 11 | 12 | {%- macro default__alias_all(columns, prefix) -%} 13 | 14 | {%- if automate_dv.is_list(columns) -%} 15 | 16 | {%- set processed_columns = [] -%} 17 | 18 | {%- for col in columns -%} 19 | {%- if col | lower not in processed_columns | map('lower') | list -%} 20 | 21 | {{ automate_dv.alias(alias_config=col, prefix=prefix) }} 22 | {%- if not loop.last -%} , {% endif -%} 23 | 24 | {%- if col is mapping -%} 25 | {%- if col['source_column'] | lower and col['alias'] | lower -%} 26 | {%- do processed_columns.append(col['source_column']) -%} 27 | {% endif -%} 28 | {%- else -%} 29 | {%- do processed_columns.append(col) -%} 30 | {% endif -%} 31 | {% endif -%} 32 | {%- endfor -%} 33 | 34 | {%- elif columns is string -%} 35 | 36 | {{ automate_dv.alias(alias_config=columns, prefix=prefix) }} 37 | 38 | {%- else -%} 39 | 40 | {%- if execute -%} 41 | {{ exceptions.raise_compiler_error("Invalid columns object provided. Must be a list or a string.") }} 42 | {%- endif %} 43 | 44 | {%- endif %} 45 | 46 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/materialisations/rank_mat_helpers/get_min_max_ranks.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {% macro get_min_max_ranks(rank_column, rank_source_models) %} 7 | 8 | {% if rank_source_models is not none %} 9 | 10 | {% if rank_source_models is string %} 11 | {% set rank_source_models = [rank_source_models] %} 12 | {% endif %} 13 | 14 | {% set query_sql %} 15 | WITH stage AS ( 16 | {% for source_model in rank_source_models %} 17 | SELECT {{ rank_column }} FROM {{ ref(source_model) }} 18 | {% if not loop.last %} UNION ALL {% endif %} 19 | {% endfor %}) 20 | 21 | SELECT MIN({{ rank_column }}) AS MIN, MAX({{ rank_column }}) AS MAX 22 | FROM stage 23 | {% endset %} 24 | 25 | {% set min_max_dict = automate_dv.get_query_results_as_dict(query_sql) %} 26 | 27 | {% set min_rank = min_max_dict['MIN'][0] | string %} 28 | {% set max_rank = min_max_dict['MAX'][0] | string %} 29 | {% set min_max_ranks = {"min_rank": min_rank, "max_rank": max_rank} %} 30 | 31 | {% do return(min_max_ranks) %} 32 | 33 | {% else %} 34 | {%- if execute -%} 35 | {{ exceptions.raise_compiler_error("Invalid 'vault_insert_by_rank' configuration. Must provide 'rank_column', and 'rank_source_models' options.") }} 36 | {%- endif -%} 37 | {% endif %} 38 | 39 | {% endmacro %} 40 | -------------------------------------------------------------------------------- /macros/supporting/data_types/type_binary.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro type_binary(for_dbt_compare) -%} 7 | {{- return(adapter.dispatch('type_binary', 'automate_dv')(for_dbt_compare=for_dbt_compare)) -}} 8 | {%- endmacro -%} 9 | 10 | {%- macro default__type_binary(for_dbt_compare=false) -%} 11 | 12 | {%- if for_dbt_compare -%} 13 | BINARY 14 | {%- else -%} 15 | {%- set selected_hash = var('hash', 'MD5') | lower -%} 16 | 17 | {%- if selected_hash == 'md5' -%} 18 | BINARY(16) 19 | {%- elif selected_hash == 'sha' -%} 20 | BINARY(32) 21 | {%- elif selected_hash == 'sha1' -%} 22 | BINARY(20) 23 | {%- else -%} 24 | BINARY(16) 25 | {%- endif -%} 26 | {%- endif -%} 27 | {%- endmacro -%} 28 | 29 | {%- macro bigquery__type_binary(for_dbt_compare=false) -%} 30 | {%- set enable_native_hashes = var('enable_native_hashes', false) -%} 31 | 32 | {%- if not enable_native_hashes -%} 33 | STRING 34 | {%- else -%} 35 | BYTES 36 | {%- endif -%} 37 | {%- endmacro -%} 38 | 39 | {%- macro postgres__type_binary(for_dbt_compare=false) -%} 40 | BYTEA 41 | {%- endmacro -%} 42 | 43 | {%- macro databricks__type_binary(for_dbt_compare=false) -%} 44 | {%- set enable_native_hashes = var('enable_native_hashes', false) -%} 45 | 46 | {%- if not enable_native_hashes -%} 47 | STRING 48 | {%- else -%} 49 | BINARY 50 | {%- endif -%} 51 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/internal/metadata_processing/multikey.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro multikey(columns, prefix=none, condition=none, operator='AND') -%} 7 | 8 | {{- adapter.dispatch('multikey', 'automate_dv')(columns=columns, prefix=prefix, condition=condition, operator=operator) -}} 9 | 10 | {%- endmacro %} 11 | 12 | {%- macro default__multikey(columns, prefix=none, condition=none, operator='AND') -%} 13 | 14 | {%- if prefix is string -%} 15 | {%- set prefix = [prefix] -%} 16 | {%- endif -%} 17 | 18 | {%- if columns is string -%} 19 | {%- set columns = [columns] -%} 20 | {%- endif -%} 21 | 22 | {%- if condition in ['<>', '!=', '='] -%} 23 | {%- for col in columns -%} 24 | {%- if prefix -%} 25 | {{- automate_dv.prefix([col], prefix[0], alias_target='target') }} {{ condition }} {{ automate_dv.prefix([col], prefix[1]) -}} 26 | {%- endif %} 27 | {%- if not loop.last %} {{ operator }} {% endif -%} 28 | {% endfor -%} 29 | {%- else -%} 30 | {%- if automate_dv.is_list(columns) -%} 31 | {%- for col in columns -%} 32 | {{ (prefix[0] ~ '.') if prefix }}{{ col }} {{ condition if condition else '' }} 33 | {%- if not loop.last -%} {{ "\n " ~ operator }} {% endif -%} 34 | {%- endfor -%} 35 | {%- else -%} 36 | {{ prefix[0] ~ '.' if prefix }}{{ columns }} {{ condition if condition else '' }} 37 | {%- endif -%} 38 | {%- endif -%} 39 | 40 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/internal/helpers/stage_processing_macros/process_hash_column_excludes.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro process_hash_column_excludes(hash_columns=none, source_columns=none) -%} 7 | 8 | {%- set processed_hash_columns = {} -%} 9 | 10 | {%- for col, col_mapping in hash_columns.items() -%} 11 | 12 | {%- if col_mapping is mapping -%} 13 | {%- if col_mapping.exclude_columns -%} 14 | 15 | {%- if col_mapping.columns -%} 16 | 17 | {%- set columns_to_hash = automate_dv.process_columns_to_select(source_columns, col_mapping.columns) -%} 18 | 19 | {%- do hash_columns[col].pop('exclude_columns') -%} 20 | {%- do hash_columns[col].update({'columns': columns_to_hash}) -%} 21 | 22 | {%- do processed_hash_columns.update({col: hash_columns[col]}) -%} 23 | {%- else -%} 24 | 25 | {%- do hash_columns[col].pop('exclude_columns') -%} 26 | {%- do hash_columns[col].update({'columns': source_columns}) -%} 27 | 28 | {%- do processed_hash_columns.update({col: hash_columns[col]}) -%} 29 | {%- endif -%} 30 | {%- else -%} 31 | {%- do processed_hash_columns.update({col: col_mapping}) -%} 32 | {%- endif -%} 33 | {%- else -%} 34 | {%- do processed_hash_columns.update({col: col_mapping}) -%} 35 | {%- endif -%} 36 | 37 | {%- endfor -%} 38 | 39 | {%- do return(processed_hash_columns) -%} 40 | 41 | {%- endmacro -%} 42 | -------------------------------------------------------------------------------- /macros/internal/helpers/logging/logging_helpers.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | 7 | {% macro wrap_message(message, type='WARNING') %} 8 | 9 | {%- set new_message = [] -%} 10 | {%- set length_list = [] -%} 11 | 12 | {%- for ln in message.split('\n') -%} 13 | {%- do new_message.append((ln | trim)) -%} 14 | {%- do length_list.append((ln | length)) -%} 15 | {%- endfor -%} 16 | 17 | {%- set max_line_length = length_list | max -%} 18 | {%- set padding_length = (max_line_length - 7) // 2 -%} 19 | 20 | {%- set sep = automate_dv.repeat('=', padding_length) %} 21 | 22 | {%- set border = sep ~ (type | upper) ~ sep -%} 23 | 24 | {%- set wrapped_message = '\n' ~ border ~ '\n' ~ new_message | join('\n') ~ '\n' ~ border -%} 25 | 26 | {%- do return(wrapped_message) -%} 27 | 28 | {% endmacro %} 29 | 30 | 31 | {%- macro log_warning(message) -%} 32 | 33 | {%- set message = automate_dv.wrap_message(message) -%} 34 | 35 | {%- if execute and automate_dv.is_something(invocation_args_dict.get('which')) and invocation_args_dict.get('which') != 'compile' -%} 36 | {%- do exceptions.warn(message) -%} 37 | {%- endif -%} 38 | 39 | {%- endmacro -%} 40 | 41 | 42 | {%- macro log_error(message) -%} 43 | 44 | {%- set message = automate_dv.wrap_message(message, type='ERROR') -%} 45 | 46 | {%- if execute and automate_dv.is_something(invocation_args_dict.get('which')) and invocation_args_dict.get('which') != 'compile' -%} 47 | {%- do exceptions.raise_compiler_error(message) -%} 48 | {%- endif -%} 49 | 50 | {%- endmacro -%} 51 | -------------------------------------------------------------------------------- /macros/tables/databricks/t_link.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro databricks__t_link(src_pk, src_fk, src_payload, src_extra_columns, src_eff, src_ldts, src_source, source_model) -%} 7 | 8 | {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_fk, src_payload, src_extra_columns, src_eff, src_ldts, src_source]) -%} 9 | {%- set fk_cols = automate_dv.expand_column_list([src_fk]) %} 10 | 11 | WITH stage AS ( 12 | SELECT DISTINCT {{ source_cols | join(', ') }} 13 | FROM {{ ref(source_model) }} 14 | {%- if model.config.materialized == 'vault_insert_by_period' %} 15 | WHERE __PERIOD_FILTER__ 16 | AND {{ automate_dv.multikey(src_pk, condition='IS NOT NULL') }} 17 | AND {{ automate_dv.multikey(fk_cols, condition='IS NOT NULL') }} 18 | {%- elif model.config.materialized == 'vault_insert_by_rank' %} 19 | WHERE __RANK_FILTER__ 20 | AND {{ automate_dv.multikey(src_pk, condition='IS NOT NULL') }} 21 | AND {{ automate_dv.multikey(fk_cols, condition='IS NOT NULL') }} 22 | {%- else %} 23 | WHERE {{ automate_dv.multikey(src_pk, condition='IS NOT NULL') }} 24 | AND {{ automate_dv.multikey(fk_cols, condition='IS NOT NULL') }} 25 | {%- endif %} 26 | ), 27 | records_to_insert AS ( 28 | SELECT {{ automate_dv.prefix(source_cols, 'stg') }} 29 | FROM stage AS stg 30 | {% if automate_dv.is_any_incremental() -%} 31 | LEFT ANTI JOIN {{ this }} AS tgt 32 | ON {{ automate_dv.multikey(src_pk, prefix=['stg','tgt'], condition='=') }} 33 | {%- endif %} 34 | ) 35 | 36 | SELECT * FROM records_to_insert 37 | 38 | {%- endmacro -%} -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing new features 2 | 3 | Please refer to our contribution guidelines over on our [development repository](https://github.com/Datavault-UK/dbtvault-dev/blob/master/CONTRIBUTING.md) 4 | 5 | ## We'd love to hear from you 6 | 7 | AutomateDV is very much a work in progress – we’re constantly adding quality of life improvements and will be adding 8 | new table types regularly. 9 | 10 | Rest assured we’re working on future releases – [our roadmap contains information on what’s coming](https://dbtvault.readthedocs.io/en/latest/roadmap/). 11 | 12 | If you spot anything you’d like to bring to our attention, have a request for new features, have spotted an improvement we could make, 13 | or want to tell us about a typo or bug, then please don’t hesitate to let us know via [github](https://github.com/Datavault-UK/dbtvault/issues). 14 | 15 | We’d rather know you are making active use of this package than hearing nothing from all of you out there! 16 | 17 | Happy Data Vaulting! 18 | 19 | ## Issue guidelines 20 | 21 | ### If it's a bug 22 | We've tested the package rigorously, but if you think you've found a bug please provide the following 23 | at a minimum (or use the issue templates) so we can fix it as quickly as possible: 24 | 25 | - The version of dbt being used 26 | - The version of automate_dv being used. 27 | - Steps to reproduce the issue 28 | - Any error messages or dbt log files which can give more detail of the problem 29 | 30 | ### If it's a feature request 31 | We'd love to add new features to make this package even more useful for the community, 32 | please feel free to submit ideas and thoughts! 33 | 34 | ### If it's an idea, feedback or a general inquiry 35 | Create a post with as much detail as possible; We'll be happy to reply and work with you. -------------------------------------------------------------------------------- /macros/supporting/hash_components/null_expression.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro null_expression(column_str) -%} 7 | 8 | {%- if execute and not column_str -%} 9 | {%- do exceptions.raise_compiler_error("Must provide a column_str argument to null expression macro!") -%} 10 | {%- endif -%} 11 | 12 | {%- set null_placeholder_string = var('null_placeholder_string', '^^') -%} 13 | {%- set standardise = automate_dv.standard_column_wrapper() %} 14 | 15 | {{ return(adapter.dispatch('null_expression', 'automate_dv')(standardise=standardise, column_str=column_str, null_placeholder_string=null_placeholder_string)) }} 16 | {%- endmacro %} 17 | 18 | 19 | {%- macro default__null_expression(standardise, column_str, null_placeholder_string) -%} 20 | 21 | {%- set column_expression -%} 22 | IFNULL({{ standardise | replace('[EXPRESSION]', column_str) }}, '{{ null_placeholder_string}}') 23 | {%- endset -%} 24 | 25 | {% do return(column_expression) %} 26 | 27 | {%- endmacro -%} 28 | 29 | 30 | {%- macro postgres__null_expression(standardise, column_str, null_placeholder_string) -%} 31 | 32 | {%- set column_expression -%} 33 | COALESCE({{ standardise | replace('[EXPRESSION]', column_str) }}, '{{ null_placeholder_string }}') 34 | {%- endset -%} 35 | 36 | {% do return(column_expression) %} 37 | 38 | {%- endmacro -%} 39 | 40 | {%- macro sqlserver__null_expression(standardise, column_str, null_placeholder_string) -%} 41 | 42 | {%- set column_expression -%} 43 | ISNULL({{ standardise | replace('[EXPRESSION]', column_str) }}, '{{ null_placeholder_string }}') 44 | {%- endset -%} 45 | 46 | {% do return(column_expression) %} 47 | 48 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/tables/snowflake/ref_table.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro ref_table(src_pk, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.check_required_parameters(src_pk=src_pk, source_model=source_model) -}} 9 | 10 | {%- if not automate_dv.is_list(source_model) -%} 11 | {%- set source_model = [source_model] -%} 12 | {%- endif -%} 13 | 14 | {{- automate_dv.prepend_generated_by() -}} 15 | 16 | {{- adapter.dispatch('ref_table', 'automate_dv')(src_pk=src_pk, src_extra_columns=src_extra_columns, 17 | src_ldts=src_ldts, src_source=src_source, 18 | source_model=source_model) -}} 19 | 20 | {%- endmacro -%} 21 | 22 | {%- macro default__ref_table(src_pk, src_extra_columns, src_ldts, src_source, source_model) -%} 23 | 24 | {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_extra_columns, src_ldts, src_source]) %} 25 | 26 | WITH source_data AS ( 27 | {%- for src in source_model %} 28 | SELECT DISTINCT 29 | {{ automate_dv.prefix(source_cols, 'a') }} 30 | FROM {{ ref(src) }} AS a 31 | WHERE a.{{ src_pk }} IS NOT NULL 32 | {%- endfor %} 33 | ), 34 | 35 | records_to_insert AS ( 36 | SELECT 37 | {{ automate_dv.prefix(source_cols, 'a') }} 38 | FROM source_data AS a 39 | {%- if automate_dv.is_any_incremental() %} 40 | LEFT JOIN {{ this }} AS d 41 | ON {{ automate_dv.multikey(src_pk, prefix=['a','d'], condition='=') }} 42 | WHERE {{ automate_dv.multikey(src_pk, prefix='d', condition='IS NULL') }} 43 | {%- endif %} 44 | ) 45 | 46 | SELECT * FROM records_to_insert 47 | 48 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/staging/hash_columns.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro hash_columns(columns=none, columns_to_escape=none) -%} 7 | 8 | {{- adapter.dispatch('hash_columns', 'automate_dv')(columns=columns, columns_to_escape=columns_to_escape) -}} 9 | 10 | {%- endmacro %} 11 | 12 | {%- macro default__hash_columns(columns=none, columns_to_escape=none) -%} 13 | 14 | {%- if columns is mapping and columns is not none -%} 15 | 16 | {%- for col in columns -%} 17 | 18 | {%- if columns[col] is mapping and columns[col].is_hashdiff -%} 19 | 20 | {{- automate_dv.hash(columns=columns[col]['columns'], 21 | alias=col, 22 | is_hashdiff=columns[col]['is_hashdiff'], 23 | columns_to_escape=columns_to_escape) -}} 24 | 25 | {%- elif columns[col] is not mapping -%} 26 | 27 | {{- automate_dv.hash(columns=columns[col], 28 | alias=col, 29 | is_hashdiff=false, 30 | columns_to_escape=columns_to_escape) -}} 31 | 32 | {%- elif columns[col] is mapping and not columns[col].is_hashdiff -%} 33 | 34 | {%- if execute -%} 35 | {%- do exceptions.warn("[" ~ this ~ "] Warning: You provided a list of columns under a 'columns' key, but did not provide the 'is_hashdiff' flag. Use list syntax for PKs.") -%} 36 | {% endif %} 37 | 38 | {{- automate_dv.hash(columns=columns[col]['columns'], alias=col, columns_to_escape=columns_to_escape) -}} 39 | 40 | {%- endif -%} 41 | 42 | {{- ",\n\n" if not loop.last -}} 43 | {%- endfor -%} 44 | 45 | {%- endif %} 46 | {%- endmacro -%} 47 | -------------------------------------------------------------------------------- /macros/internal/helpers/is_checks.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro is_list(obj, empty_is_false=false) -%} 7 | 8 | {%- if obj is iterable and obj is not string and obj is not mapping -%} 9 | {%- if obj is none and obj is undefined and not obj and empty_is_false -%} 10 | {%- do return(false) -%} 11 | {%- endif -%} 12 | 13 | {%- do return(true) -%} 14 | {%- else -%} 15 | {%- do return(false) -%} 16 | {%- endif -%} 17 | 18 | {%- endmacro -%} 19 | 20 | 21 | 22 | {%- macro is_nothing(obj) -%} 23 | 24 | {%- if obj is none or obj is undefined or not obj or automate_dv.is_list(obj, empty_is_false=true) -%} 25 | {%- do return(true) -%} 26 | {%- else -%} 27 | {%- do return(false) -%} 28 | {%- endif -%} 29 | 30 | {%- endmacro -%} 31 | 32 | 33 | 34 | {%- macro is_something(obj) -%} 35 | 36 | {%- if obj is not none and obj is defined and obj -%} 37 | {#- if an empty list, do not consider the object something -#} 38 | {% if automate_dv.is_list(empty_is_false=true) %} 39 | {%- do return(true) -%} 40 | {%- else -%} 41 | {%- do return(false) -%} 42 | {%- endif -%} 43 | {%- else -%} 44 | {%- do return(false) -%} 45 | {%- endif -%} 46 | 47 | {%- endmacro -%} 48 | 49 | 50 | 51 | {%- macro is_expression(obj) -%} 52 | 53 | {%- if obj is string -%} 54 | {%- if (obj | first == "'" and obj | last == "'") or ("(" in obj and ")" in obj) or "::" in obj -%} 55 | {%- do return(true) -%} 56 | {%- else -%} 57 | {%- do return(false) -%} 58 | {%- endif -%} 59 | {%- else -%} 60 | {%- do return(false) -%} 61 | {%- endif -%} 62 | 63 | {%- endmacro -%} 64 | -------------------------------------------------------------------------------- /macros/internal/helpers/stage_processing_macros/process_columns_to_escape.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro process_columns_to_escape(derived_columns_list=none) -%} 7 | 8 | {%- if derived_columns_list -%} 9 | 10 | {%- set ns = namespace(columns_to_escape=[]) -%} 11 | {%- set escape_char_left, escape_char_right = automate_dv.get_escape_characters() -%} 12 | {%- set quote_pattern = '\{}([a-zA-Z\s]+)\{}'.format(escape_char_left, escape_char_right) -%} 13 | {%- set re = modules.re -%} 14 | 15 | {%- for col_name, col_def in derived_columns_list.items() -%} 16 | 17 | {%- if col_def is mapping -%} 18 | {%- if col_def['escape'] == true -%} 19 | {%- if automate_dv.is_list(col_def['source_column']) -%} 20 | {%- set ns.columns_to_escape = ns.columns_to_escape + col_def['source_column'] -%} 21 | {%- else -%} 22 | {%- set ns.columns_to_escape = ns.columns_to_escape + [col_def['source_column']] -%} 23 | {%- endif -%} 24 | {%- endif -%} 25 | {%- elif col_def is string -%} 26 | 27 | {#- Find a quoted string in the column definition so that we can escape it everywhere else -#} 28 | {% set is_match = re.findall(quote_pattern, col_def, re.IGNORECASE) %} 29 | 30 | {%- if is_match -%} 31 | {%- set ns.columns_to_escape = ns.columns_to_escape + is_match -%} 32 | {%- endif -%} 33 | {%- endif -%} 34 | {%- endfor -%} 35 | 36 | {%- do return(ns.columns_to_escape | unique | list) -%} 37 | {%- else -%} 38 | {%- do return([]) -%} 39 | {%- endif -%} 40 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/materialisations/rank_mat_helpers/replace_placeholder_with_rank_filter.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro replace_placeholder_with_rank_filter(core_sql, rank_column, rank_iteration) -%} 7 | 8 | {% set macro = adapter.dispatch('replace_placeholder_with_rank_filter', 9 | 'automate_dv')(core_sql=core_sql, 10 | rank_column=rank_column, 11 | rank_iteration=rank_iteration) %} 12 | {% do return(macro) %} 13 | {%- endmacro %} 14 | 15 | 16 | 17 | {% macro default__replace_placeholder_with_rank_filter(core_sql, rank_column, rank_iteration) %} 18 | 19 | {%- set rank_filter -%} 20 | {{ rank_column }}:: INTEGER = {{ rank_iteration }}::INTEGER 21 | {%- endset -%} 22 | 23 | {%- set filtered_sql = core_sql | replace("__RANK_FILTER__", rank_filter) -%} 24 | 25 | {% do return(filtered_sql) %} 26 | {% endmacro %} 27 | 28 | 29 | 30 | 31 | {% macro sqlserver__replace_placeholder_with_rank_filter(core_sql, rank_column, rank_iteration) %} 32 | 33 | {%- set rank_filter -%} 34 | CAST({{ rank_column }} AS INT) = CAST({{ rank_iteration }} AS INT) 35 | {%- endset -%} 36 | 37 | {%- set filtered_sql = core_sql | replace("__RANK_FILTER__", rank_filter) -%} 38 | 39 | {% do return(filtered_sql) %} 40 | {% endmacro %} 41 | 42 | 43 | 44 | 45 | {% macro bigquery__replace_placeholder_with_rank_filter(core_sql, rank_column, rank_iteration) %} 46 | {%- set rank_filter -%} 47 | CAST({{ rank_column }} AS INTEGER) = CAST({{ rank_iteration }} AS INTEGER) 48 | {%- endset -%} 49 | 50 | {%- set filtered_sql = core_sql | replace("__RANK_FILTER__", rank_filter) -%} 51 | 52 | {% do return(filtered_sql) %} 53 | {% endmacro %} -------------------------------------------------------------------------------- /macros/internal/metadata_processing/process_payload_column_excludes.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro process_payload_column_excludes(src_pk, src_hashdiff, src_payload, src_extra_columns, src_cdk, 7 | src_eff, src_ldts, src_source, source_model) -%} 8 | 9 | {%- if src_payload is not mapping -%} 10 | {%- do return(src_payload) -%} 11 | {%- endif -%} 12 | 13 | {%- set source_model_cols = automate_dv.wrap_get_columns_in_relation(ref(source_model)) -%} 14 | {%- set columns_in_metadata = automate_dv.expand_column_list( 15 | columns=[src_pk, src_hashdiff, src_cdk, 16 | src_payload, src_extra_columns, 17 | src_eff, src_ldts, src_source]) | map('lower') | list -%} 18 | 19 | {%- set payload_cols = [] -%} 20 | {%- for col in source_model_cols -%} 21 | {%- if col.column | lower not in columns_in_metadata -%} 22 | {%- do payload_cols.append(col.column) -%} 23 | {%- endif -%} 24 | {%- endfor -%} 25 | 26 | {%- if 'exclude_columns' in src_payload.keys() -%} 27 | {%- set table_excludes_columns = src_payload.exclude_columns -%} 28 | 29 | {%- if table_excludes_columns -%} 30 | 31 | {%- set excluded_payload = [] -%} 32 | {%- set exclude_columns_list = src_payload.columns | map('lower') | list -%} 33 | 34 | {%- for col in payload_cols -%} 35 | {%- if col | lower not in exclude_columns_list -%} 36 | {%- do excluded_payload.append(col) -%} 37 | {%- endif -%} 38 | {%- endfor -%} 39 | {%- endif -%} 40 | {%- endif -%} 41 | 42 | {%- do return(excluded_payload) -%} 43 | 44 | {%- endmacro -%} 45 | -------------------------------------------------------------------------------- /macros/materialisations/period_mat_helpers/get_start_stop_dates.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {% macro get_start_stop_dates(timestamp_field, date_source_models) %} 7 | 8 | {% if automate_dv.config_meta_get('start_date', default=none) is not none %} 9 | {%- set start_date = automate_dv.config_meta_get('start_date') -%} 10 | {%- set stop_date = automate_dv.config_meta_get('stop_date', default=none) -%} 11 | 12 | {% do return({'start_date': start_date,'stop_date': stop_date}) %} 13 | 14 | {% elif date_source_models is not none %} 15 | {% if date_source_models is string %} 16 | {% set date_source_models = [date_source_models] %} 17 | {% endif %} 18 | {% set query_sql %} 19 | WITH stage AS ( 20 | {% for source_model in date_source_models %} 21 | SELECT 22 | {{ timestamp_field }} 23 | FROM {{ ref(source_model) }} 24 | {% if not loop.last %} UNION ALL {% endif %} 25 | {% endfor %}) 26 | 27 | SELECT MIN({{ timestamp_field }}) AS MIN, MAX({{ timestamp_field }}) AS MAX 28 | FROM stage 29 | {% endset %} 30 | 31 | {% set min_max_dict = automate_dv.get_query_results_as_dict(query_sql) %} 32 | 33 | {% set start_date = min_max_dict['MIN'][0] | string %} 34 | {% set stop_date = min_max_dict['MAX'][0] | string %} 35 | {% set min_max_dates = {"start_date": start_date, "stop_date": stop_date} %} 36 | 37 | {% do return(min_max_dates) %} 38 | 39 | {% else %} 40 | {%- if execute -%} 41 | {{ exceptions.raise_compiler_error("Invalid 'vault_insert_by_period' configuration. Must provide 'start_date' and 'stop_date', just 'stop_date', and/or 'date_source_models' options.") }} 42 | {%- endif -%} 43 | {% endif %} 44 | 45 | {% endmacro %} -------------------------------------------------------------------------------- /macros/materialisations/period_mat_helpers/check_datediff.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro check_num_periods(start_date, stop_date, period) -%} 7 | 8 | {% set num_periods = adapter.dispatch('check_num_periods', 9 | 'automate_dv')(start_date=start_date, 10 | stop_date=stop_date, 11 | period=period) %} 12 | 13 | {%- if num_periods > 100000 -%} 14 | {{ automate_dv.max_iterations_error() }} 15 | {%- endif -%} 16 | 17 | {% do return(num_periods) %} 18 | 19 | {%- endmacro %} 20 | 21 | {% macro default__check_num_periods(start_date, stop_date, period) %} 22 | 23 | {% set num_periods_check_sql %} 24 | SELECT {{ datediff('start_timestamp', 'stop_timestamp', period) }} AS NUM_PERIODS 25 | FROM 26 | (SELECT CAST('{{ start_date }}' AS {{ dbt.type_timestamp() }}) AS start_timestamp, 27 | CAST(NULLIF('{{ stop_date | lower }}', 'none') AS {{ dbt.type_timestamp() }}) AS stop_timestamp) AS SUBQUERY_ALIAS 28 | {% endset %} 29 | {% set num_periods_dict = automate_dv.get_query_results_as_dict(num_periods_check_sql) %} 30 | {% set num_periods = num_periods_dict['NUM_PERIODS'][0] | int %} 31 | 32 | {% do return(num_periods) %} 33 | 34 | {% endmacro %} 35 | 36 | {% macro sqlserver__check_num_periods(start_date, stop_date, period) %} 37 | 38 | {% set num_periods_check_sql %} 39 | SELECT DATEDIFF_BIG({{ period }}, CAST('{{ start_date }}' AS DATETIME2), 40 | CAST(NULLIF('{{ stop_date | lower }}', 'none') AS DATETIME2)) AS NUM_PERIODS 41 | {% endset %} 42 | {% set num_periods_dict = automate_dv.get_query_results_as_dict(num_periods_check_sql) %} 43 | {% set num_periods = num_periods_dict['NUM_PERIODS'][0] | int %} 44 | 45 | {% do return(num_periods) %} 46 | 47 | {% endmacro %} -------------------------------------------------------------------------------- /macros/supporting/casting/cast_date.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro cast_date(column_str, as_string=false, datetime=false, alias=none, date_type=none) -%} 7 | {%- if datetime -%} 8 | {{- automate_dv.cast_datetime(column_str=column_str, as_string=as_string, alias=alias, date_type=date_type) -}} 9 | {%- else -%} 10 | {{ return(adapter.dispatch('cast_date', 'automate_dv')(column_str=column_str, as_string=as_string, alias=alias)) }} 11 | {%- endif -%} 12 | {%- endmacro -%} 13 | 14 | {%- macro snowflake__cast_date(column_str, as_string=false, alias=none) -%} 15 | 16 | {%- if not as_string -%} 17 | TO_DATE({{ column_str }}) 18 | {%- else -%} 19 | TO_DATE('{{ column_str }}') 20 | {%- endif -%} 21 | 22 | {%- if alias %} AS {{ alias }} {%- endif %} 23 | 24 | {%- endmacro -%} 25 | 26 | 27 | {%- macro sqlserver__cast_date(column_str, as_string=false, alias=none) -%} 28 | 29 | {%- if not as_string -%} 30 | CONVERT(DATE, {{ column_str }}) 31 | {%- else -%} 32 | CONVERT(DATE, '{{ column_str }}') 33 | {%- endif -%} 34 | 35 | {%- if alias %} AS {{ alias }} {%- endif %} 36 | 37 | 38 | {%- endmacro -%} 39 | 40 | 41 | {%- macro bigquery__cast_date(column_str, as_string=false, alias=none) -%} 42 | 43 | {%- if not as_string -%} 44 | DATE({{ column_str }}) 45 | {%- else -%} 46 | DATE('{{ column_str }}') 47 | {%- endif -%} 48 | 49 | {%- if alias %} AS {{ alias }} {%- endif %} 50 | 51 | {%- endmacro -%} 52 | 53 | 54 | {%- macro databricks__cast_date(column_str, as_string=false, alias=none) -%} 55 | 56 | {{ automate_dv.snowflake__cast_date(column_str=column_str, as_string=as_string, alias=alias)}} 57 | 58 | {%- endmacro -%} 59 | 60 | 61 | {%- macro postgres__cast_date(column_str, as_string=false, alias=none) -%} 62 | 63 | {%- if as_string -%} 64 | TO_DATE('{{ column_str }}', 'YYY-MM-DD') 65 | {%- else -%} 66 | TO_DATE({{ column_str }}::VARCHAR, 'YYY-MM-DD') 67 | {%- endif -%} 68 | 69 | {%- if alias %} AS {{ alias }} {%- endif %} 70 | 71 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/supporting/casting/cast_datetime.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro cast_datetime(column_str, as_string=false, alias=none, date_type=none) -%} 7 | 8 | {%- if as_string -%} 9 | {%- set column_str -%} '{{ column_str }}' {%- endset -%} 10 | {%- endif -%} 11 | 12 | {%- set date_type = date_type | lower -%} 13 | 14 | {{ return(adapter.dispatch('cast_datetime', 'automate_dv')(column_str=column_str, as_string=as_string, alias=alias, date_type=date_type)) }} 15 | {%- endmacro -%} 16 | 17 | {%- macro snowflake__cast_datetime(column_str, as_string=false, alias=none, date_type=none) -%} 18 | 19 | {%- if date_type == 'timestamp_tz' -%} 20 | TO_TIMESTAMP_TZ({{ column_str }}) 21 | {%- elif date_type == 'timestamp_ltz' -%} 22 | TO_TIMESTAMP_LTZ({{ column_str }}) 23 | {%- elif date_type == 'timestamp_ntz' -%} 24 | TO_TIMESTAMP_NTZ({{ column_str }}) 25 | {%- else -%} 26 | TO_TIMESTAMP({{ column_str }}) 27 | {%- endif -%} 28 | 29 | {%- if alias %} AS {{ alias }} {%- endif %} 30 | 31 | {%- endmacro -%} 32 | 33 | 34 | {%- macro sqlserver__cast_datetime(column_str, as_string=false, alias=none, date_type=none) -%} 35 | 36 | CONVERT(DATETIME2, {{ column_str }}) 37 | 38 | {%- if alias %} AS {{ alias }} {%- endif %} 39 | 40 | {%- endmacro -%} 41 | 42 | 43 | {%- macro bigquery__cast_datetime(column_str, as_string=false, alias=none, date_type=none) -%} 44 | 45 | {%- if date_type == 'timestamp' -%} 46 | PARSE_TIMESTAMP('%F %H:%M:%E6S', {{ column_str }}) 47 | {%- else -%} 48 | PARSE_DATETIME('%F %H:%M:%E6S', {{ column_str }}) 49 | {%- endif -%} 50 | 51 | {%- if alias %} AS {{ alias }} {%- endif %} 52 | 53 | {%- endmacro -%} 54 | 55 | 56 | {%- macro databricks__cast_datetime(column_str, as_string=false, alias=none, date_type=none) -%} 57 | 58 | {{ automate_dv.snowflake__cast_datetime(column_str=column_str, as_string=as_string, alias=alias, date_type=date_type)}} 59 | 60 | {%- endmacro -%} 61 | 62 | 63 | {%- macro postgres__cast_datetime(column_str, as_string=false, alias=none, date_type=none) -%} 64 | 65 | to_char(timestamp {{ column_str }}, 'YYYY-MM-DD HH24:MI:SS.MS')::timestamp 66 | 67 | {%- if alias %} AS {{ alias }} {%- endif %} 68 | 69 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/internal/helpers/timestamp_add.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {% macro timestamp_add(datepart, interval, from_date_or_timestamp) %} 7 | {{ return(adapter.dispatch('timestamp_add', 'automate_dv')(datepart=datepart, interval=interval, 8 | from_date_or_timestamp=from_date_or_timestamp)) }} 9 | {%- endmacro -%} 10 | 11 | {%- macro default__timestamp_add(datepart, interval, from_date_or_timestamp) -%} 12 | 13 | {%- if datepart is in ['day', 'week', 'month', 'quarter', 'year'] -%} 14 | {{ automate_dv.dateadd('millisecond', 86399999, from_date_or_timestamp) }} 15 | {%- elif datepart == 'microsecond' -%} 16 | {{ automate_dv.dateadd('microsecond', 1, from_date_or_timestamp) }} 17 | {%- elif datepart == 'millisecond' -%} 18 | {{ automate_dv.dateadd('microsecond', 999, from_date_or_timestamp) }} 19 | {%- elif datepart == 'second' -%} 20 | {{ automate_dv.dateadd('millisecond', 999, from_date_or_timestamp) }} 21 | {%- elif datepart == 'minute' -%} 22 | {{ automate_dv.dateadd('millisecond', 5999, from_date_or_timestamp) }} 23 | {%- elif datepart == 'hour' -%} 24 | {{ automate_dv.dateadd('millisecond', 3599999, from_date_or_timestamp) }} 25 | {%- endif -%} 26 | 27 | {%- endmacro -%} 28 | 29 | {% macro bigquery__timestamp_add(datepart, interval, from_date_or_timestamp) %} 30 | 31 | {%- if datepart is in ['day', 'week', 'month', 'quarter', 'year'] -%} 32 | {{ automate_dv.dateadd('millisecond', 86399999, from_date_or_timestamp) }} 33 | {%- elif datepart == 'microsecond' -%} 34 | TIMESTAMP_ADD(CAST( {{from_date_or_timestamp}} AS TIMESTAMP), INTERVAL 1 microsecond) 35 | {%- elif datepart == 'millisecond' -%} 36 | TIMESTAMP_ADD(CAST( {{from_date_or_timestamp}} AS TIMESTAMP), INTERVAL 999 microsecond) 37 | {%- elif datepart == 'second' -%} 38 | TIMESTAMP_ADD(CAST( {{from_date_or_timestamp}} AS TIMESTAMP), INTERVAL 999 millisecond) 39 | {%- elif datepart == 'minute' -%} 40 | TIMESTAMP_ADD(CAST( {{from_date_or_timestamp}} AS TIMESTAMP), INTERVAL 5999 millisecond) 41 | {%- elif datepart == 'hour' -%} 42 | TIMESTAMP_ADD(CAST( {{from_date_or_timestamp}} AS TIMESTAMP), INTERVAL 3599999 millisecond) 43 | {%- endif -%} 44 | 45 | {% endmacro %} 46 | 47 | 48 | -------------------------------------------------------------------------------- /macros/supporting/ghost_records/binary_ghost.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro binary_ghost(alias, hash) -%} 7 | {%- set hash = hash | lower -%} 8 | 9 | {{ adapter.dispatch('binary_ghost', 'automate_dv')(alias=alias, hash=hash) }} 10 | {%- endmacro -%} 11 | 12 | {%- macro default__binary_ghost(alias, hash) -%} 13 | 14 | {%- if hash == 'md5' -%} 15 | {%- set zero_string_size = 32 %} 16 | {%- elif hash == 'sha' -%} 17 | {%- set zero_string_size = 64 %} 18 | {%- elif hash == 'sha1' -%} 19 | {%- set zero_string_size = 40 %} 20 | {%- else -%} 21 | {%- set zero_string_size = 32 %} 22 | {%- endif -%} 23 | 24 | {%- set zero_string = automate_dv.repeat('0', zero_string_size) -%} 25 | 26 | {{- automate_dv.cast_binary(column_str=zero_string, alias=alias, quote=true) -}} 27 | 28 | {%- endmacro -%} 29 | 30 | {%- macro bigquery__binary_ghost(alias, hash) -%} 31 | 32 | {%- if hash == 'md5' -%} 33 | {%- set zero_string_size = 32 %} 34 | {%- elif hash == 'sha' -%} 35 | {%- set zero_string_size = 64 %} 36 | {%- elif hash == 'sha1' -%} 37 | {%- set zero_string_size = 40 %} 38 | {%- else -%} 39 | {%- set zero_string_size = 32 %} 40 | {%- endif -%} 41 | 42 | {%- set enable_native_hashes = var('enable_native_hashes', false) -%} 43 | {%- set zero_string = automate_dv.repeat('0', zero_string_size) -%} 44 | 45 | {%- if enable_native_hashes -%} 46 | {%- set column_str = "FROM_HEX('{}')".format(zero_string) -%} 47 | {%- else -%} 48 | {%- set column_str = zero_string -%} 49 | {%- endif -%} 50 | 51 | {{- automate_dv.cast_binary(column_str=column_str, alias=alias, quote=false if enable_native_hashes else true) -}} 52 | 53 | {%- endmacro -%} 54 | 55 | {%- macro sqlserver__binary_ghost(alias, hash) -%} 56 | {%- if hash == 'md5' -%} 57 | {%- set zero_string_size = 16 %} 58 | {%- elif hash == 'sha' -%} 59 | {%- set zero_string_size = 32 %} 60 | {%- elif hash == 'sha1' -%} 61 | {%- set zero_string_size = 20 %} 62 | {%- else -%} 63 | {%- set zero_string_size = 16 %} 64 | {%- endif -%} 65 | 66 | CAST(REPLICATE(CAST(CAST('0' AS tinyint) AS BINARY({{ zero_string_size }})), {{ zero_string_size }}) AS BINARY({{ zero_string_size }})) 67 | 68 | {%- if alias %} AS {{ alias }} {%- endif -%} 69 | {%- endmacro -%} 70 | -------------------------------------------------------------------------------- /macros/internal/metadata_processing/get_escape_characters.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {% macro get_escape_characters() -%} 7 | 8 | {%- set default_escape_char_left, default_escape_char_right = adapter.dispatch('get_escape_characters', 'automate_dv')() -%} 9 | 10 | {%- if ((var('escape_char_left', default_escape_char_left) == '') and (var('escape_char_right', default_escape_char_right) == '')) -%} 11 | {%- set warning_message = 'Invalid escape_char_left and escape_char_right value provided. ' + 12 | 'Using platform defaults ({}{})'.format(default_escape_char_left, default_escape_char_right) -%} 13 | {%- set escape_chars = (default_escape_char_left, default_escape_char_right) -%} 14 | 15 | {%- elif var('escape_char_left', default_escape_char_left) == '' -%} 16 | {%- set warning_message = 'Invalid escape_char_left value provided. Using platform default ({})'.format(default_escape_char_left) -%} 17 | {%- set escape_chars = (default_escape_char_left, var('escape_char_right', default_escape_char_right)) -%} 18 | 19 | {%- elif var('escape_char_right', default_escape_char_right) == '' -%} 20 | {%- set warning_message = 'Invalid escape_char_right value provided. Using platform default ({})'.format(default_escape_char_right) -%} 21 | {%- set escape_chars = (var('escape_char_left', default_escape_char_left), default_escape_char_right) -%} 22 | 23 | {%- else -%} 24 | {%- set escape_chars = (var('escape_char_left', default_escape_char_left), var('escape_char_right', default_escape_char_right)) -%} 25 | {%- endif -%} 26 | 27 | {%- if execute and warning_message -%} 28 | {%- do exceptions.warn(warning_message) -%} 29 | {%- endif -%} 30 | 31 | {%- do return(escape_chars) -%} 32 | 33 | {%- endmacro %} 34 | 35 | {%- macro snowflake__get_escape_characters() %} 36 | {%- do return (('"', '"')) -%} 37 | {%- endmacro %} 38 | 39 | {%- macro bigquery__get_escape_characters() %} 40 | {%- do return (('`', '`')) -%} 41 | {%- endmacro %} 42 | 43 | {%- macro sqlserver__get_escape_characters() %} 44 | {%- do return (('"', '"')) -%} 45 | {%- endmacro %} 46 | 47 | {%- macro databricks__get_escape_characters() %} 48 | {%- do return (('`', '`')) -%} 49 | {%- endmacro %} 50 | 51 | {%- macro postgres__get_escape_characters() %} 52 | {%- do return (('"', '"')) -%} 53 | {%- endmacro %} 54 | -------------------------------------------------------------------------------- /macros/materialisations/incremental_pit_materialization.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- materialization pit_incremental, default -%} 7 | 8 | {%- set full_refresh_mode = should_full_refresh() -%} 9 | 10 | {% if target.type == "sqlserver" %} 11 | {%- set target_relation = this.incorporate(type='table') -%} 12 | {% else %} 13 | {%- set target_relation = this -%} 14 | {% endif %} 15 | {%- set existing_relation = load_relation(this) -%} 16 | {%- set tmp_relation = make_temp_relation(target_relation) -%} 17 | 18 | {{ run_hooks(pre_hooks, inside_transaction=False) }} 19 | 20 | -- `BEGIN` happens here: 21 | {{ run_hooks(pre_hooks, inside_transaction=True) }} 22 | 23 | {%- set to_drop = [] -%} 24 | {%- if existing_relation is none -%} 25 | {%- set build_sql = create_table_as(False, target_relation, sql) -%} 26 | {%- elif existing_relation.is_view or full_refresh_mode -%} 27 | {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #} 28 | {%- set backup_identifier = existing_relation.identifier ~ "__dbt_backup" -%} 29 | {%- set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) -%} 30 | {%- do adapter.drop_relation(backup_relation) -%} 31 | 32 | {%- do adapter.rename_relation(target_relation, backup_relation) -%} 33 | {%- set build_sql = create_table_as(False, target_relation, sql) -%} 34 | {%- do to_drop.append(backup_relation) -%} 35 | {%- else -%} 36 | 37 | {%- set tmp_relation = make_temp_relation(target_relation) -%} 38 | {%- do run_query(create_table_as(True, tmp_relation, sql)) -%} 39 | {%- do adapter.expand_target_column_types( 40 | from_relation=tmp_relation, 41 | to_relation=target_relation) -%} 42 | {%- set build_sql = automate_dv.incremental_pit_replace(tmp_relation, target_relation) -%} 43 | {% if target.type == "sqlserver" %} 44 | {%- do to_drop.append(tmp_relation) -%} 45 | {% endif %} 46 | {%- endif -%} 47 | 48 | {%- call statement("main") -%} 49 | {{ build_sql }} 50 | {%- endcall -%} 51 | 52 | {{ run_hooks(post_hooks, inside_transaction=True) }} 53 | 54 | -- `COMMIT` happens here 55 | {%- do adapter.commit() -%} 56 | 57 | {%- for rel in to_drop -%} 58 | {%- do adapter.drop_relation(rel) -%} 59 | {%- endfor -%} 60 | 61 | {{ run_hooks(post_hooks, inside_transaction=False) }} 62 | 63 | {{ return({'relations': [target_relation]}) }} 64 | 65 | {%- endmaterialization -%} -------------------------------------------------------------------------------- /macros/materialisations/incremental_bridge_materialization.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- materialization bridge_incremental, default -%} 7 | 8 | {%- set full_refresh_mode = should_full_refresh() -%} 9 | 10 | {% if target.type == "sqlserver" %} 11 | {%- set target_relation = this.incorporate(type='table') -%} 12 | {% else %} 13 | {%- set target_relation = this -%} 14 | {% endif %} 15 | {%- set existing_relation = load_relation(this) -%} 16 | {%- set tmp_relation = make_temp_relation(target_relation) -%} 17 | 18 | {{ run_hooks(pre_hooks, inside_transaction=False) }} 19 | 20 | -- `BEGIN` happens here: 21 | {{ run_hooks(pre_hooks, inside_transaction=True) }} 22 | 23 | {%- set to_drop = [] -%} 24 | {%- if existing_relation is none -%} 25 | {%- set build_sql = create_table_as(False, target_relation, sql) -%} 26 | {%- elif existing_relation.is_view or full_refresh_mode -%} 27 | {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #} 28 | {%- set backup_identifier = existing_relation.identifier ~ "__dbt_backup" -%} 29 | {%- set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) -%} 30 | {%- do adapter.drop_relation(backup_relation) -%} 31 | 32 | {%- do adapter.rename_relation(target_relation, backup_relation) -%} 33 | {%- set build_sql = create_table_as(False, target_relation, sql) -%} 34 | {%- do to_drop.append(backup_relation) -%} 35 | {%- else -%} 36 | 37 | {%- set tmp_relation = make_temp_relation(target_relation) -%} 38 | {%- do run_query(create_table_as(True, tmp_relation, sql)) -%} 39 | {%- do adapter.expand_target_column_types( 40 | from_relation=tmp_relation, 41 | to_relation=target_relation) -%} 42 | {%- set build_sql = automate_dv.incremental_bridge_replace(tmp_relation, target_relation) -%} 43 | {% if target.type == "sqlserver" %} 44 | {%- do to_drop.append(tmp_relation) -%} 45 | {% endif %} 46 | {%- endif -%} 47 | 48 | {%- call statement("main") -%} 49 | {{ build_sql }} 50 | {%- endcall -%} 51 | 52 | {{ run_hooks(post_hooks, inside_transaction=True) }} 53 | 54 | -- `COMMIT` happens here 55 | {%- do adapter.commit() -%} 56 | 57 | {%- for rel in to_drop -%} 58 | {%- do adapter.drop_relation(rel) -%} 59 | {%- endfor -%} 60 | 61 | {{ run_hooks(post_hooks, inside_transaction=False) }} 62 | 63 | {{ return({'relations': [target_relation]}) }} 64 | 65 | {%- endmaterialization -%} -------------------------------------------------------------------------------- /macros/tables/snowflake/t_link.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro t_link(src_pk, src_fk, src_payload, src_extra_columns, src_eff, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.check_required_parameters(src_pk=src_pk, src_fk=src_fk, src_eff=src_eff, 9 | src_ldts=src_ldts, src_source=src_source, 10 | source_model=source_model) -}} 11 | 12 | {{ automate_dv.prepend_generated_by() }} 13 | 14 | {{ adapter.dispatch('t_link', 'automate_dv')(src_pk=src_pk, src_fk=src_fk, src_payload=src_payload, 15 | src_extra_columns=src_extra_columns, 16 | src_eff=src_eff, src_ldts=src_ldts, src_source=src_source, 17 | source_model=source_model) -}} 18 | 19 | {%- endmacro %} 20 | 21 | {%- macro default__t_link(src_pk, src_fk, src_payload, src_extra_columns, src_eff, src_ldts, src_source, source_model) -%} 22 | 23 | {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_fk, src_payload, src_extra_columns, src_eff, src_ldts, src_source]) -%} 24 | {%- set fk_cols = automate_dv.expand_column_list([src_fk]) %} 25 | 26 | WITH stage AS ( 27 | SELECT {{ source_cols | join(', ') }} 28 | FROM {{ ref(source_model) }} 29 | {%- if model.config.materialized == 'vault_insert_by_period' %} 30 | WHERE __PERIOD_FILTER__ 31 | AND {{ automate_dv.multikey(src_pk, condition='IS NOT NULL') }} 32 | AND {{ automate_dv.multikey(fk_cols, condition='IS NOT NULL') }} 33 | {%- elif model.config.materialized == 'vault_insert_by_rank' %} 34 | WHERE __RANK_FILTER__ 35 | AND {{ automate_dv.multikey(src_pk, condition='IS NOT NULL') }} 36 | AND {{ automate_dv.multikey(fk_cols, condition='IS NOT NULL') }} 37 | {%- else %} 38 | WHERE {{ automate_dv.multikey(src_pk, condition='IS NOT NULL') }} 39 | AND {{ automate_dv.multikey(fk_cols, condition='IS NOT NULL') }} 40 | {%- endif %} 41 | ), 42 | records_to_insert AS ( 43 | SELECT DISTINCT {{ automate_dv.prefix(source_cols, 'stg') }} 44 | FROM stage AS stg 45 | {% if automate_dv.is_any_incremental() -%} 46 | LEFT JOIN {{ this }} AS tgt 47 | ON {{ automate_dv.multikey(src_pk, prefix=['stg','tgt'], condition='=') }} 48 | WHERE {{ automate_dv.multikey(src_pk, prefix='tgt', condition='IS NULL') }} 49 | {%- endif %} 50 | ) 51 | 52 | SELECT * FROM records_to_insert 53 | 54 | {%- endmacro -%} 55 | -------------------------------------------------------------------------------- /macros/materialisations/mat_is_checks.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro is_any_incremental() -%} 7 | {%- if automate_dv.is_vault_insert_by_period() or automate_dv.is_vault_insert_by_rank() or automate_dv.is_pit_incremental() or automate_dv.is_bridge_incremental() or is_incremental() -%} 8 | {%- do return(true) -%} 9 | {%- else -%} 10 | {%- do return(false) -%} 11 | {%- endif -%} 12 | {%- endmacro -%} 13 | 14 | 15 | 16 | {% macro is_vault_insert_by_period() %} 17 | {% if not execute %} 18 | {{ return(False) }} 19 | {% else %} 20 | {% set relation = adapter.get_relation(this.database, this.schema, this.table) %} 21 | 22 | {{ return(relation is not none 23 | and relation.type == 'table' 24 | and model.config.materialized == 'vault_insert_by_period' 25 | and not should_full_refresh()) }} 26 | {% endif %} 27 | {% endmacro %} 28 | 29 | 30 | 31 | {% macro is_vault_insert_by_rank() %} 32 | {#-- do not run introspective queries in parsing #} 33 | {% if not execute %} 34 | {{ return(False) }} 35 | {% else %} 36 | {% set relation = adapter.get_relation(this.database, this.schema, this.table) %} 37 | 38 | {{ return(relation is not none 39 | and relation.type == 'table' 40 | and model.config.materialized == 'vault_insert_by_rank' 41 | and not should_full_refresh()) }} 42 | {% endif %} 43 | {% endmacro %} 44 | 45 | 46 | 47 | {% macro is_bridge_incremental() %} 48 | {#-- do not run introspective queries in parsing #} 49 | {% if not execute %} 50 | {{ return(False) }} 51 | {% else %} 52 | {% set relation = adapter.get_relation(this.database, this.schema, this.table) %} 53 | 54 | {{ return(relation is not none 55 | and relation.type == 'table' 56 | and model.config.materialized == 'bridge_incremental' 57 | and not should_full_refresh()) }} 58 | {% endif %} 59 | {% endmacro %} 60 | 61 | 62 | 63 | {% macro is_pit_incremental() %} 64 | {#-- do not run introspective queries in parsing #} 65 | {% if not execute %} 66 | {{ return(False) }} 67 | {% else %} 68 | {% set relation = adapter.get_relation(this.database, this.schema, this.table) %} 69 | 70 | {{ return(relation is not none 71 | and relation.type == 'table' 72 | and model.config.materialized == 'pit_incremental' 73 | and not should_full_refresh()) }} 74 | {% endif %} 75 | {% endmacro %} -------------------------------------------------------------------------------- /macros/supporting/hash_components/standard_column_wrapper.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro standard_column_wrapper() -%} 7 | 8 | {%- set hash_content_casing = var('hash_content_casing', 'upper') -%} 9 | {%- set available_case_configs = ['upper', 'disabled'] -%} 10 | 11 | {%- if execute and (hash_content_casing | lower) not in available_case_configs -%} 12 | {%- do exceptions.raise_compiler_error("Must provide a valid casing config for hash_content_casing. 13 | '{}' was provided. Can be one of {} (case insensitive)".format( 14 | hash_content_casing, 15 | available_case_configs | join(','))) -%} 16 | {%- endif -%} 17 | 18 | {{ return(adapter.dispatch('standard_column_wrapper', 'automate_dv')(hash_content_casing=hash_content_casing | lower)) }} 19 | {%- endmacro %} 20 | 21 | 22 | {%- macro default__standard_column_wrapper(hash_content_casing) -%} 23 | 24 | {%- if hash_content_casing == 'upper' -%} 25 | {%- set standardise -%} 26 | NULLIF(UPPER(TRIM(CAST([EXPRESSION] AS {{ automate_dv.type_string() }}))), '') 27 | {%- endset -%} 28 | {%- else -%} 29 | {%- set standardise -%} 30 | NULLIF(TRIM(CAST([EXPRESSION] AS {{ automate_dv.type_string() }})), '') 31 | {%- endset -%} 32 | {%- endif -%} 33 | 34 | {% do return(standardise) -%} 35 | 36 | {%- endmacro -%} 37 | 38 | 39 | {%- macro databricks__standard_column_wrapper(hash_content_casing) -%} 40 | 41 | {%- if hash_content_casing == 'upper' -%} 42 | {%- set standardise -%} 43 | NULLIF(UPPER(TRIM(CAST([EXPRESSION] AS {{ automate_dv.type_string(is_hash=true) }}))), '') 44 | {%- endset -%} 45 | {%- else -%} 46 | {%- set standardise -%} 47 | NULLIF(TRIM(CAST([EXPRESSION] AS {{ automate_dv.type_string(is_hash=true) }})), '') 48 | {%- endset -%} 49 | {%- endif -%} 50 | 51 | {% do return(standardise) -%} 52 | 53 | {%- endmacro -%} 54 | 55 | 56 | {%- macro sqlserver__standard_column_wrapper(hash_content_casing) -%} 57 | 58 | {%- if hash_content_casing == 'upper' -%} 59 | {%- set standardise -%} 60 | NULLIF(UPPER(TRIM(CAST([EXPRESSION] AS {{ automate_dv.type_string() }}(MAX)))), '') 61 | {%- endset -%} 62 | {%- else -%} 63 | {%- set standardise -%} 64 | NULLIF(TRIM(CAST([EXPRESSION] AS {{ automate_dv.type_string() }}(MAX))), '') 65 | {%- endset -%} 66 | {%- endif -%} 67 | 68 | {% do return(standardise) -%} 69 | 70 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/supporting/prefix.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro prefix(columns, prefix_str, alias_target) -%} 7 | 8 | {{- adapter.dispatch('prefix', 'automate_dv')(columns=columns, 9 | prefix_str=prefix_str, 10 | alias_target=alias_target) -}} 11 | 12 | {%- endmacro -%} 13 | 14 | {%- macro default__prefix(columns=none, prefix_str=none, alias_target='source') -%} 15 | 16 | {%- set processed_columns = [] -%} 17 | 18 | {%- if columns and prefix_str -%} 19 | 20 | {%- for col in columns -%} 21 | 22 | {%- if col | lower not in processed_columns | map('lower') | list -%} 23 | 24 | {%- if col is mapping -%} 25 | 26 | {%- if alias_target == 'source' -%} 27 | 28 | {{- automate_dv.prefix([col['source_column']], prefix_str) -}} 29 | 30 | {%- do processed_columns.append(col['source_column']) -%} 31 | 32 | {%- elif alias_target == 'target' -%} 33 | 34 | {{- automate_dv.prefix([col['alias']], prefix_str) -}} 35 | 36 | {%- do processed_columns.append(col['alias']) -%} 37 | 38 | {%- else -%} 39 | 40 | {{- automate_dv.prefix([col['source_column']], prefix_str) -}} 41 | 42 | {%- do processed_columns.append(col['source_column']) -%} 43 | 44 | {%- endif -%} 45 | 46 | {%- if not loop.last -%} , {% endif %} 47 | 48 | {%- else -%} 49 | 50 | {%- if col is iterable and col is not string -%} 51 | 52 | {{- automate_dv.prefix(col, prefix_str) -}} 53 | 54 | {%- do processed_columns.append(col) -%} 55 | 56 | {%- elif col is not none -%} 57 | 58 | {{- prefix_str}}.{{col.strip() -}} 59 | 60 | {%- do processed_columns.append(col) -%} 61 | {% else %} 62 | 63 | {%- if execute -%} 64 | {{- exceptions.raise_compiler_error("Unexpected or missing configuration for '" ~ this ~ "' Unable to prefix columns.") -}} 65 | {%- endif -%} 66 | {%- endif -%} 67 | 68 | {{- ', ' if not loop.last -}} 69 | 70 | {%- endif -%} 71 | {%- endif -%} 72 | 73 | {%- endfor -%} 74 | 75 | {%- else -%} 76 | 77 | {%- if execute -%} 78 | {{- exceptions.raise_compiler_error("Invalid parameters provided to prefix macro. Expected: (columns [list/string], prefix_str [string]) got: (" ~ columns ~ ", " ~ prefix_str ~ ")") -}} 79 | {%- endif -%} 80 | {%- endif -%} 81 | 82 | {%- endmacro -%} 83 | -------------------------------------------------------------------------------- /macros/materialisations/incremental_pit_bridge_replace.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {% macro incremental_pit_replace(tmp_relation, target_relation, statement_name="main") %} 7 | 8 | {{ adapter.dispatch('incremental_pit_replace', 'automate_dv')(tmp_relation=tmp_relation, 9 | target_relation=target_relation, 10 | statement_name="main") }} 11 | 12 | {%- endmacro -%} 13 | 14 | {% macro default__incremental_pit_replace(tmp_relation, target_relation, statement_name="main") %} 15 | {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%} 16 | {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%} 17 | 18 | TRUNCATE TABLE {{ target_relation }}; 19 | 20 | INSERT INTO {{ target_relation }} ({{ dest_cols_csv }}) 21 | ( 22 | SELECT {{ dest_cols_csv }} 23 | FROM {{ tmp_relation }} 24 | ); 25 | {%- endmacro %} 26 | 27 | 28 | {% macro databricks__incremental_pit_replace(tmp_relation, target_relation, statement_name="main") %} 29 | {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%} 30 | {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%} 31 | 32 | INSERT OVERWRITE {{ target_relation }} ({{ dest_cols_csv }}) 33 | SELECT {{ dest_cols_csv }} 34 | FROM {{ tmp_relation }}; 35 | 36 | {%- endmacro %} 37 | 38 | 39 | 40 | {% macro incremental_bridge_replace(tmp_relation, target_relation, statement_name="main") %} 41 | 42 | {{ adapter.dispatch('incremental_bridge_replace', 'automate_dv')(tmp_relation=tmp_relation, 43 | target_relation=target_relation, 44 | statement_name="main") }} 45 | 46 | {%- endmacro -%} 47 | 48 | 49 | {% macro default__incremental_bridge_replace(tmp_relation, target_relation, statement_name="main") %} 50 | {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%} 51 | {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%} 52 | 53 | TRUNCATE TABLE {{ target_relation }}; 54 | 55 | INSERT INTO {{ target_relation }} ({{ dest_cols_csv }}) 56 | ( 57 | SELECT {{ dest_cols_csv }} 58 | FROM {{ tmp_relation }} 59 | ); 60 | {%- endmacro %} 61 | 62 | 63 | {% macro databricks__incremental_bridge_replace(tmp_relation, target_relation, statement_name="main") %} 64 | {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%} 65 | {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%} 66 | 67 | INSERT OVERWRITE {{ target_relation }} ({{ dest_cols_csv }}) 68 | SELECT {{ dest_cols_csv }} 69 | FROM {{ tmp_relation }} 70 | ; 71 | {%- endmacro %} 72 | 73 | 74 | 75 | -------------------------------------------------------------------------------- /macros/staging/rank_columns.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro rank_columns(columns=none) -%} 7 | 8 | {{- adapter.dispatch('rank_columns', 'automate_dv')(columns=columns) -}} 9 | 10 | {%- endmacro %} 11 | 12 | {%- macro default__rank_columns(columns=none) -%} 13 | 14 | {%- if columns is mapping and columns is not none -%} 15 | 16 | {%- for col in columns -%} 17 | 18 | {%- if columns[col] is mapping and columns[col].partition_by and columns[col].order_by -%} 19 | 20 | {%- set order_by = columns[col].order_by -%} 21 | {%- set partition_by = columns[col].partition_by -%} 22 | {%- set dense_rank = columns[col].dense_rank -%} 23 | 24 | {%- if automate_dv.is_nothing(dense_rank) %} 25 | {%- set rank_type = "RANK()" -%} 26 | {%- elif dense_rank is true -%} 27 | {%- set rank_type = "DENSE_RANK()" -%} 28 | {%- else -%} 29 | {%- if execute -%} 30 | {%- do exceptions.raise_compiler_error('If dense_rank is provided, it must be true or false, not {}'.format(dense_rank)) -%} 31 | {% endif %} 32 | {%- endif -%} 33 | 34 | {%- if automate_dv.is_list(order_by) -%} 35 | 36 | {%- set order_by_str_lst = [] -%} 37 | 38 | {% for order_by_col in order_by %} 39 | 40 | {%- if order_by_col is mapping %} 41 | {%- set column_name, direction = order_by_col.items()|first -%} 42 | {%- set order_by_str = "{} {}".format(column_name, direction) | trim -%} 43 | {%- else -%} 44 | {%- set order_by_str = order_by_col -%} 45 | {%- endif -%} 46 | 47 | {%- do order_by_str_lst.append(order_by_str) -%} 48 | {%- endfor -%} 49 | 50 | {%- set order_by_str = order_by_str_lst | join(", ") -%} 51 | 52 | {%- else -%} 53 | 54 | {%- if order_by is mapping %} 55 | {%- set column_name, direction = order_by.items()|first -%} 56 | {%- else -%} 57 | {%- set column_name = order_by -%} 58 | {%- set direction = '' -%} 59 | {%- endif -%} 60 | 61 | {%- set order_by_str = "{} {}".format(column_name, direction) | trim -%} 62 | {%- endif -%} 63 | 64 | {%- if automate_dv.is_list(partition_by) -%} 65 | {%- set partition_by_str = partition_by | join(", ") -%} 66 | {%- else -%} 67 | {%- set partition_by_str = partition_by -%} 68 | {%- endif -%} 69 | 70 | {{- "{} OVER (PARTITION BY {} ORDER BY {}) AS {}".format(rank_type, partition_by_str, order_by_str, col) | indent(4) -}} 71 | 72 | {%- endif -%} 73 | 74 | {{- ",\n" if not loop.last -}} 75 | {%- endfor -%} 76 | 77 | {%- endif %} 78 | {%- endmacro -%} 79 | -------------------------------------------------------------------------------- /macros/supporting/bridge_shared.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro bridge_overlap_and_new_rows(src_pk, bridge_walk, source_model, new_as_of_dates_cte) -%} 7 | 8 | SELECT 9 | {{ automate_dv.prefix([src_pk], 'a') }}, 10 | b.AS_OF_DATE, 11 | {%- for bridge_step in bridge_walk.keys() -%} 12 | {%- set link_table = bridge_walk[bridge_step]['link_table'] -%} 13 | {%- set eff_sat_table = bridge_walk[bridge_step]['eff_sat_table'] -%} 14 | 15 | {%- set link_pk = bridge_walk[bridge_step]['link_pk'] -%} 16 | 17 | {%- set bridge_link_pk = bridge_walk[bridge_step]['bridge_link_pk'] -%} 18 | {%- set bridge_end_date = bridge_walk[bridge_step]['bridge_end_date'] -%} 19 | {%- set bridge_load_date = bridge_walk[bridge_step]['bridge_load_date'] -%} 20 | 21 | {%- set eff_sat_end_date = bridge_walk[bridge_step]['eff_sat_end_date'] -%} 22 | {%- set eff_sat_load_date = bridge_walk[bridge_step]['eff_sat_load_date'] %} 23 | 24 | {{- '\n ' }} {{ link_table | lower }}.{{ link_pk }} AS {{ bridge_link_pk }}, 25 | {{- '\n ' }} {{ eff_sat_table | lower }}.{{ eff_sat_end_date }} AS {{ bridge_end_date }}, 26 | {{- '\n ' }} {{ eff_sat_table | lower }}.{{ eff_sat_load_date }} AS {{ bridge_load_date }} 27 | 28 | {%- if not loop.last %}, {%- endif -%} 29 | 30 | {% endfor %} 31 | 32 | FROM {{ source_model }} AS a 33 | INNER JOIN {{ new_as_of_dates_cte }} AS b 34 | ON (1=1) 35 | 36 | {%- set loop_vars = namespace(last_link = '', last_link_fk = '') %} 37 | {%- for bridge_step in bridge_walk.keys() -%} 38 | 39 | {%- set current_link = bridge_walk[bridge_step]['link_table'] -%} 40 | {%- set current_eff_sat = bridge_walk[bridge_step]['eff_sat_table'] -%} 41 | 42 | {%- set link_pk = bridge_walk[bridge_step]['link_pk'] -%} 43 | {%- set link_fk1 = bridge_walk[bridge_step]['link_fk1'] -%} 44 | {%- set link_fk2 = bridge_walk[bridge_step]['link_fk2'] -%} 45 | 46 | {%- set eff_sat_pk = bridge_walk[bridge_step]['eff_sat_pk'] -%} 47 | {%- set eff_sat_load_date = bridge_walk[bridge_step]['eff_sat_load_date'] -%} 48 | 49 | {%- if loop.first %} 50 | LEFT JOIN {{ ref(current_link) }} AS {{ current_link | lower }} 51 | ON {{ automate_dv.multikey(src_pk, prefix=['a', current_link | lower], condition='=') }} 52 | {%- else %} 53 | LEFT JOIN {{ ref(current_link) }} AS {{ current_link | lower }} 54 | ON {{ loop_vars.last_link }}.{{ loop_vars.last_link_fk2 }} = {{ current_link | lower }}.{{ link_fk1 }} 55 | {%- endif %} 56 | INNER JOIN {{ ref(current_eff_sat) }} AS {{ current_eff_sat | lower }} 57 | ON {{ current_eff_sat | lower }}.{{ eff_sat_pk }} = {{ current_link | lower }}.{{ link_pk }} 58 | AND {{ current_eff_sat | lower }}.{{ eff_sat_load_date }} <= b.AS_OF_DATE 59 | {%- set loop_vars.last_link = current_link | lower -%} 60 | {%- set loop_vars.last_link_fk2 = link_fk2 -%} 61 | {% endfor %} 62 | 63 | {%- endmacro -%} 64 | -------------------------------------------------------------------------------- /macros/tables/databricks/hub.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro databricks__hub(src_pk, src_nk, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_nk, src_extra_columns, src_ldts, src_source]) -%} 9 | 10 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 11 | {%- set source_cols_with_rank = source_cols + [automate_dv.config_meta_get('rank_column')] -%} 12 | {%- endif %} 13 | 14 | {{ 'WITH ' -}} 15 | 16 | {%- set stage_count = source_model | length -%} 17 | 18 | {%- set ns = namespace(last_cte= "") -%} 19 | 20 | {%- for src in source_model -%} 21 | 22 | {%- set source_number = loop.index | string -%} 23 | 24 | row_rank_{{ source_number }} AS ( 25 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 26 | SELECT {{ automate_dv.prefix(source_cols_with_rank, 'rr') }} 27 | {%- else %} 28 | SELECT {{ automate_dv.prefix(source_cols, 'rr') }} 29 | {%- endif %} 30 | FROM {{ ref(src) }} AS rr 31 | WHERE {{ automate_dv.multikey(src_pk, prefix='rr', condition='IS NOT NULL') }} 32 | QUALIFY ROW_NUMBER() OVER( 33 | PARTITION BY {{ automate_dv.prefix([src_pk], 'rr') }} 34 | ORDER BY {{ automate_dv.prefix([src_ldts], 'rr') }} 35 | ) = 1 36 | {%- set ns.last_cte = "row_rank_{}".format(source_number) %} 37 | ),{{ "\n" if not loop.last }} 38 | {% endfor -%} 39 | {% if stage_count > 1 %} 40 | stage_union AS ( 41 | {%- for src in source_model %} 42 | SELECT * FROM row_rank_{{ loop.index | string }} 43 | {%- if not loop.last %} 44 | UNION ALL 45 | {%- endif %} 46 | {%- endfor %} 47 | {%- set ns.last_cte = "stage_union" %} 48 | ), 49 | {%- endif -%} 50 | 51 | {%- if model.config.materialized == 'vault_insert_by_period' %} 52 | stage_mat_filter AS ( 53 | SELECT * 54 | FROM {{ ns.last_cte }} 55 | WHERE __PERIOD_FILTER__ 56 | {%- set ns.last_cte = "stage_mat_filter" %} 57 | ), 58 | {%- elif model.config.materialized == 'vault_insert_by_rank' %} 59 | stage_mat_filter AS ( 60 | SELECT * 61 | FROM {{ ns.last_cte }} 62 | WHERE __RANK_FILTER__ 63 | {%- set ns.last_cte = "stage_mat_filter" %} 64 | ), 65 | {%- endif -%} 66 | 67 | {%- if stage_count > 1 %} 68 | 69 | row_rank_union AS ( 70 | SELECT ru.* 71 | FROM {{ ns.last_cte }} AS ru 72 | WHERE {{ automate_dv.multikey(src_pk, prefix='ru', condition='IS NOT NULL') }} 73 | QUALIFY ROW_NUMBER() OVER( 74 | PARTITION BY {{ automate_dv.prefix([src_pk], 'ru') }} 75 | ORDER BY {{ automate_dv.prefix([src_ldts], 'ru') }}, {{ automate_dv.prefix([src_source], 'ru') }} ASC 76 | ) = 1 77 | {%- set ns.last_cte = "row_rank_union" %} 78 | ), 79 | {% endif %} 80 | records_to_insert AS ( 81 | SELECT {{ automate_dv.prefix(source_cols, 'a', alias_target='target') }} 82 | FROM {{ ns.last_cte }} AS a 83 | {%- if automate_dv.is_any_incremental() %} 84 | LEFT ANTI JOIN {{ this }} AS d 85 | ON {{ automate_dv.multikey(src_pk, prefix=['a','d'], condition='=') }} 86 | {%- endif %} 87 | ) 88 | 89 | SELECT * FROM records_to_insert 90 | 91 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/tables/bigquery/hub.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro bigquery__hub(src_pk, src_nk, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_nk, src_extra_columns, src_ldts, src_source]) -%} 9 | 10 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 11 | {%- set source_cols_with_rank = source_cols + [automate_dv.config_meta_get('rank_column')] -%} 12 | {%- endif %} 13 | 14 | {{ 'WITH ' -}} 15 | 16 | {%- set stage_count = source_model | length -%} 17 | 18 | {%- set ns = namespace(last_cte= "") -%} 19 | 20 | {%- for src in source_model -%} 21 | 22 | {%- set source_number = loop.index | string -%} 23 | 24 | row_rank_{{ source_number }} AS ( 25 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 26 | SELECT {{ automate_dv.prefix(source_cols_with_rank, 'rr') }}, 27 | {%- else %} 28 | SELECT {{ automate_dv.prefix(source_cols, 'rr') }}, 29 | {%- endif %} 30 | FROM {{ ref(src) }} AS rr 31 | WHERE {{ automate_dv.multikey(src_pk, prefix='rr', condition='IS NOT NULL') }} 32 | QUALIFY ROW_NUMBER() OVER( 33 | PARTITION BY {{ automate_dv.prefix([src_pk], 'rr') }} 34 | ORDER BY {{ automate_dv.prefix([src_ldts], 'rr') }} 35 | ) = 1 36 | {%- set ns.last_cte = "row_rank_{}".format(source_number) %} 37 | ), 38 | 39 | {% endfor -%} 40 | {% if stage_count > 1 %} 41 | stage_union AS ( 42 | {%- for src in source_model %} 43 | SELECT * FROM row_rank_{{ loop.index | string }} 44 | {%- if not loop.last %} 45 | UNION ALL 46 | {%- endif %} 47 | {%- endfor %} 48 | {%- set ns.last_cte = "stage_union" %} 49 | ), 50 | {%- endif -%} 51 | {%- if model.config.materialized == 'vault_insert_by_period' %} 52 | stage_mat_filter AS ( 53 | SELECT * 54 | FROM {{ ns.last_cte }} 55 | WHERE __PERIOD_FILTER__ 56 | {%- set ns.last_cte = "stage_mat_filter" %} 57 | ), 58 | {%- elif model.config.materialized == 'vault_insert_by_rank' %} 59 | stage_mat_filter AS ( 60 | SELECT * 61 | FROM {{ ns.last_cte }} 62 | WHERE __RANK_FILTER__ 63 | {%- set ns.last_cte = "stage_mat_filter" %} 64 | ), 65 | {%- endif -%} 66 | {%- if stage_count > 1 %} 67 | 68 | row_rank_union AS ( 69 | SELECT ru.* 70 | FROM {{ ns.last_cte }} AS ru 71 | WHERE {{ automate_dv.multikey(src_pk, prefix='ru', condition='IS NOT NULL') }} 72 | QUALIFY ROW_NUMBER() OVER( 73 | PARTITION BY {{ automate_dv.prefix([src_pk], 'ru') }} 74 | ORDER BY {{ automate_dv.prefix([src_ldts], 'ru') }}, {{ automate_dv.prefix([src_source], 'ru') }} ASC 75 | ) = 1 76 | {%- set ns.last_cte = "row_rank_union" %} 77 | ), 78 | 79 | {% endif %} 80 | records_to_insert AS ( 81 | SELECT {{ automate_dv.prefix(source_cols, 'a', alias_target='target') }} 82 | FROM {{ ns.last_cte }} AS a 83 | {%- if automate_dv.is_any_incremental() %} 84 | LEFT JOIN {{ this }} AS d 85 | ON {{ automate_dv.multikey(src_pk, prefix=['a','d'], condition='=') }} 86 | WHERE {{ automate_dv.multikey(src_pk, prefix='d', condition='IS NULL') }} 87 | {%- endif %} 88 | ) 89 | 90 | SELECT * FROM records_to_insert 91 | 92 | {%- endmacro -%} 93 | -------------------------------------------------------------------------------- /macros/internal/metadata_processing/escape_column_names.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro escape_column_names(columns=none) -%} 7 | 8 | {%- if automate_dv.is_list(columns) -%} 9 | {%- set columns = automate_dv.expand_column_list(columns) -%} 10 | {%- endif -%} 11 | 12 | {%- if automate_dv.is_something(columns) -%} 13 | 14 | {%- set col_string = '' -%} 15 | {%- set col_list = [] -%} 16 | {%- set col_mapping = {} -%} 17 | 18 | {%- if columns is string -%} 19 | 20 | {%- set col_string = automate_dv.escape_column_name(columns) -%} 21 | 22 | {%- elif automate_dv.is_list(columns) -%} 23 | 24 | {%- for col in columns -%} 25 | 26 | {%- if col is string -%} 27 | 28 | {%- set escaped_col = automate_dv.escape_column_name(col) -%} 29 | 30 | {%- do col_list.append(escaped_col) -%} 31 | 32 | {%- else -%} 33 | 34 | {%- if execute -%} 35 | {{- exceptions.raise_compiler_error("Invalid column name(s) provided. Must be a string.") -}} 36 | {%- endif -%} 37 | 38 | {%- endif -%} 39 | 40 | {%- endfor -%} 41 | 42 | {%- elif columns is mapping -%} 43 | 44 | {%- if columns['source_column'] and columns['alias'] -%} 45 | 46 | {%- set escaped_source_col = automate_dv.escape_column_name(columns['source_column']) -%} 47 | {%- set escaped_alias_col = automate_dv.escape_column_name(columns['alias']) -%} 48 | {%- set col_mapping = {"source_column": escaped_source_col, "alias": escaped_alias_col} -%} 49 | 50 | {%- else -%} 51 | 52 | {%- if execute -%} 53 | {{- exceptions.raise_compiler_error("Invalid column name(s) provided. Must be a string, a list of strings, or a dictionary of hashdiff metadata.") -}} 54 | {%- endif %} 55 | 56 | {%- endif -%} 57 | 58 | {%- else -%} 59 | 60 | {%- if execute -%} 61 | {{- exceptions.raise_compiler_error("Invalid column name(s) provided. Must be a string, a list of strings, or a dictionary of hashdiff metadata.") -}} 62 | {%- endif %} 63 | 64 | {%- endif -%} 65 | 66 | {%- elif columns == '' -%} 67 | 68 | {%- if execute -%} 69 | {{- exceptions.raise_compiler_error("Expected a column name or a list of column names, got an empty string") -}} 70 | {%- endif -%} 71 | 72 | {%- endif -%} 73 | 74 | {%- if columns is none -%} 75 | 76 | {%- do return(none) -%} 77 | 78 | {%- elif columns == [] -%} 79 | 80 | {%- do return([]) -%} 81 | 82 | {%- elif columns == {} -%} 83 | 84 | {%- do return({}) -%} 85 | 86 | {%- elif columns is string -%} 87 | 88 | {%- do return(col_string) -%} 89 | 90 | {%- elif automate_dv.is_list(columns) -%} 91 | 92 | {%- do return(col_list) -%} 93 | 94 | {%- elif columns is mapping -%} 95 | 96 | {%- do return(col_mapping) -%} 97 | 98 | {%- endif -%} 99 | 100 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/staging/null_columns.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro null_columns(source_relation=none, columns=none) -%} 7 | 8 | {{- adapter.dispatch('null_columns', 'automate_dv')(source_relation=source_relation, columns=columns) -}} 9 | 10 | {%- endmacro %} 11 | 12 | {%- macro default__null_columns(source_relation=none, columns=none) -%} 13 | 14 | {%- if columns is mapping and columns is not none -%} 15 | 16 | {%- set ns = namespace() -%} 17 | 18 | {%- for col in columns -%} 19 | {%- if col.lower() == 'required' -%} 20 | {% if automate_dv.is_something(columns[col]) %} 21 | {%- if columns[col] is string -%} 22 | {%- set ns.required = [columns[col]] -%} 23 | {%- elif automate_dv.is_list(columns[col]) -%} 24 | {%- set ns.required = columns[col] -%} 25 | {%- endif -%} 26 | {%- endif -%} 27 | {%- endif -%} 28 | {%- if col.lower() == 'optional' -%} 29 | {% if automate_dv.is_something(columns[col]) %} 30 | {%- if columns[col] is string -%} 31 | {%- set ns.optional = [columns[col]] -%} 32 | {%- elif automate_dv.is_list(columns[col]) -%} 33 | {%- set ns.optional = columns[col] -%} 34 | {%- endif -%} 35 | {%- endif -%} 36 | {%- endif -%} 37 | {%- endfor -%} 38 | 39 | {%- set required_value = var('null_key_required', '-1') -%} 40 | {%- set optional_value = var('null_key_optional', '-2') -%} 41 | 42 | {%- if automate_dv.is_something(ns.required) -%} 43 | {%- filter indent(width=0) -%} 44 | {%- for col_name in ns.required -%} 45 | {{ automate_dv.null_column_sql(col_name, required_value) }}{{ ",\n" if not loop.last }}{{ ",\n" if loop.last and automate_dv.is_something(ns.optional) else "" }} 46 | {%- endfor -%} 47 | {%- endfilter -%} 48 | {%- endif -%} 49 | 50 | {%- if automate_dv.is_something(ns.optional) -%} 51 | {%- filter indent(width=0) -%} 52 | {%- for col_name in ns.optional -%} 53 | {{ automate_dv.null_column_sql(col_name, optional_value) }}{{ ",\n" if not loop.last }} 54 | {%- endfor -%} 55 | {%- endfilter -%} 56 | {%- endif -%} 57 | 58 | {%- endif -%} 59 | 60 | {%- endmacro -%} 61 | 62 | 63 | {%- macro null_column_sql(col_name, default_value) -%} 64 | 65 | {{- adapter.dispatch('null_column_sql', 'automate_dv')(col_name=col_name, default_value=default_value) -}} 66 | 67 | {%- endmacro -%} 68 | 69 | {%- macro default__null_column_sql(col_name, default_value) -%} 70 | 71 | {{ col_name }} AS {{ col_name ~ "_ORIGINAL" }}, 72 | IFNULL({{ col_name }}, '{{ default_value }}') AS {{ col_name }} 73 | 74 | {%- endmacro -%} 75 | 76 | {%- macro sqlserver__null_column_sql(col_name, default_value) -%} 77 | 78 | {{ col_name }} AS {{ col_name ~ "_ORIGINAL" }}, 79 | ISNULL({{ col_name }}, '{{ default_value }}') AS {{ col_name }} 80 | 81 | {%- endmacro -%} 82 | 83 | {%- macro postgres__null_column_sql(col_name, default_value) -%} 84 | 85 | {{ col_name }} AS {{ col_name ~ "_ORIGINAL" }}, 86 | COALESCE({{ col_name }}, '{{ default_value }}') AS {{ col_name }} 87 | 88 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/tables/databricks/link.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro databricks__link(src_pk, src_fk, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_fk, src_extra_columns, src_ldts, src_source]) -%} 9 | {%- set fk_cols = automate_dv.expand_column_list([src_fk]) -%} 10 | 11 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 12 | {%- set source_cols_with_rank = source_cols + [automate_dv.config_meta_get('rank_column')] -%} 13 | {%- endif %} 14 | 15 | {{ 'WITH ' -}} 16 | 17 | {%- set stage_count = source_model | length -%} 18 | 19 | {%- set ns = namespace(last_cte= "") -%} 20 | 21 | {%- for src in source_model -%} 22 | 23 | {%- set source_number = loop.index | string -%} 24 | 25 | row_rank_{{ source_number }} AS ( 26 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 27 | SELECT {{ automate_dv.prefix(source_cols_with_rank, 'rr') }} 28 | {%- else %} 29 | SELECT {{ automate_dv.prefix(source_cols, 'rr') }} 30 | {%- endif %} 31 | FROM {{ ref(src) }} AS rr 32 | {%- if stage_count == 1 %} 33 | WHERE {{ automate_dv.multikey(src_pk, prefix='rr', condition='IS NOT NULL') }} 34 | AND {{ automate_dv.multikey(fk_cols, prefix='rr', condition='IS NOT NULL') }} 35 | {%- endif %} 36 | QUALIFY ROW_NUMBER() OVER( 37 | PARTITION BY {{ automate_dv.prefix([src_pk], 'rr') }} 38 | ORDER BY {{ automate_dv.prefix([src_ldts], 'rr') }} 39 | ) = 1 40 | {%- set ns.last_cte = "row_rank_{}".format(source_number) %} 41 | ),{{ "\n" if not loop.last }} 42 | {% endfor -%} 43 | 44 | {% if stage_count > 1 %} 45 | stage_union AS ( 46 | {%- for src in source_model %} 47 | SELECT * FROM row_rank_{{ loop.index | string }} 48 | {%- if not loop.last %} 49 | UNION ALL 50 | {%- endif %} 51 | {%- endfor %} 52 | {%- set ns.last_cte = "stage_union" %} 53 | ), 54 | {%- endif -%} 55 | {%- if model.config.materialized == 'vault_insert_by_period' %} 56 | stage_mat_filter AS ( 57 | SELECT * 58 | FROM {{ ns.last_cte }} 59 | WHERE __PERIOD_FILTER__ 60 | {%- set ns.last_cte = "stage_mat_filter" %} 61 | ), 62 | {%- elif model.config.materialized == 'vault_insert_by_rank' %} 63 | stage_mat_filter AS ( 64 | SELECT * 65 | FROM {{ ns.last_cte }} 66 | WHERE __RANK_FILTER__ 67 | {%- set ns.last_cte = "stage_mat_filter" %} 68 | ), 69 | {% endif %} 70 | {%- if stage_count > 1 %} 71 | 72 | row_rank_union AS ( 73 | SELECT ru.* 74 | FROM {{ ns.last_cte }} AS ru 75 | WHERE {{ automate_dv.multikey(src_pk, prefix='ru', condition='IS NOT NULL') }} 76 | AND {{ automate_dv.multikey(fk_cols, prefix='ru', condition='IS NOT NULL') }} 77 | QUALIFY ROW_NUMBER() OVER( 78 | PARTITION BY {{ automate_dv.prefix([src_pk], 'ru') }} 79 | ORDER BY {{ automate_dv.prefix([src_ldts], 'ru') }}, {{ automate_dv.prefix([src_source], 'ru') }} ASC 80 | ) = 1 81 | {%- set ns.last_cte = "row_rank_union" %} 82 | ), 83 | {% endif %} 84 | records_to_insert AS ( 85 | SELECT {{ automate_dv.prefix(source_cols, 'a', alias_target='target') }} 86 | FROM {{ ns.last_cte }} AS a 87 | {%- if automate_dv.is_any_incremental() %} 88 | LEFT ANTI JOIN {{ this }} AS d 89 | ON {{ automate_dv.multikey(src_pk, prefix=['a','d'], condition='=') }} 90 | {%- endif %} 91 | ) 92 | 93 | SELECT * FROM records_to_insert 94 | 95 | {%- endmacro -%} -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at enquiries@data-vault.co.uk. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /macros/tables/postgres/hub.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro postgres__hub(src_pk, src_nk, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_nk, src_extra_columns, src_ldts, src_source]) -%} 9 | 10 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 11 | {%- set source_cols_with_rank = source_cols + [automate_dv.config_meta_get('rank_column')] -%} 12 | {%- endif %} 13 | 14 | {{ 'WITH ' -}} 15 | 16 | {%- if not (source_model is iterable and source_model is not string) -%} 17 | {%- set source_model = [source_model] -%} 18 | {%- endif -%} 19 | 20 | {%- set ns = namespace(last_cte= "") -%} 21 | 22 | {%- for src in source_model -%} 23 | 24 | {%- set source_number = loop.index | string -%} 25 | 26 | row_rank_{{ source_number }} AS ( 27 | {#- PostgreSQL has DISTINCT ON which should be more performant than the 28 | strategy used by Snowflake ROW_NUMBER() OVER( PARTITION BY ... 29 | -#} 30 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 31 | SELECT DISTINCT ON ({{ automate_dv.prefix([src_pk], 'rr') }}) {{ automate_dv.prefix(source_cols_with_rank, 'rr') }} 32 | {%- else %} 33 | SELECT DISTINCT ON ({{ automate_dv.prefix([src_pk], 'rr') }}) {{ automate_dv.prefix(source_cols, 'rr') }} 34 | {%- endif %} 35 | FROM {{ ref(src) }} AS rr 36 | WHERE {{ automate_dv.multikey(src_pk, prefix='rr', condition='IS NOT NULL') }} 37 | ORDER BY {{ automate_dv.prefix([src_pk], 'rr') }}, {{ automate_dv.prefix([src_ldts], 'rr') }} 38 | {%- set ns.last_cte = "row_rank_{}".format(source_number) %} 39 | ),{{ "\n" if not loop.last }} 40 | {% endfor -%} 41 | {% if source_model | length > 1 %} 42 | stage_union AS ( 43 | {%- for src in source_model %} 44 | SELECT * FROM row_rank_{{ loop.index | string }} 45 | {%- if not loop.last %} 46 | UNION ALL 47 | {%- endif %} 48 | {%- endfor %} 49 | {%- set ns.last_cte = "stage_union" %} 50 | ), 51 | {%- endif -%} 52 | {%- if model.config.materialized == 'vault_insert_by_period' %} 53 | stage_mat_filter AS ( 54 | SELECT * 55 | FROM {{ ns.last_cte }} 56 | WHERE __PERIOD_FILTER__ 57 | {%- set ns.last_cte = "stage_mat_filter" %} 58 | ), 59 | {%- elif model.config.materialized == 'vault_insert_by_rank' %} 60 | stage_mat_filter AS ( 61 | SELECT * 62 | FROM {{ ns.last_cte }} 63 | WHERE __RANK_FILTER__ 64 | {%- set ns.last_cte = "stage_mat_filter" %} 65 | ), 66 | {%- endif -%} 67 | {%- if source_model | length > 1 %} 68 | 69 | row_rank_union AS ( 70 | {#- PostgreSQL has DISTINCT ON which should be more performant than the 71 | strategy used by Snowflake ROW_NUMBER() OVER( PARTITION BY ... 72 | -#} 73 | SELECT DISTINCT ON ({{ automate_dv.prefix([src_pk], 'ru') }}) ru.* 74 | FROM {{ ns.last_cte }} AS ru 75 | WHERE {{ automate_dv.multikey(src_pk, prefix='ru', condition='IS NOT NULL') }} 76 | ORDER BY {{ automate_dv.prefix([src_pk], 'ru') }}, {{ automate_dv.prefix([src_ldts], 'ru') }}, {{ automate_dv.prefix([src_source], 'ru') }} ASC 77 | {%- set ns.last_cte = "row_rank_union" %} 78 | ), 79 | {% endif %} 80 | records_to_insert AS ( 81 | SELECT {{ automate_dv.prefix(source_cols, 'a', alias_target='target') }} 82 | FROM {{ ns.last_cte }} AS a 83 | {%- if automate_dv.is_any_incremental() %} 84 | LEFT JOIN {{ this }} AS d 85 | ON {{ automate_dv.multikey(src_pk, prefix=['a','d'], condition='=') }} 86 | WHERE {{ automate_dv.multikey(src_pk, prefix='d', condition='IS NULL') }} 87 | {%- endif %} 88 | ) 89 | 90 | SELECT * FROM records_to_insert 91 | 92 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/materialisations/period_mat_helpers/get_period_filter_sql.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro get_period_filter_sql(target_cols_csv, base_sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%} 7 | 8 | {% set macro = adapter.dispatch('get_period_filter_sql', 9 | 'automate_dv')(target_cols_csv=target_cols_csv, 10 | base_sql=base_sql, 11 | timestamp_field=timestamp_field, 12 | period=period, 13 | start_timestamp=start_timestamp, 14 | stop_timestamp=stop_timestamp, 15 | offset=offset) %} 16 | {% do return(macro) %} 17 | {%- endmacro %} 18 | 19 | 20 | 21 | 22 | {% macro default__get_period_filter_sql(target_cols_csv, base_sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%} 23 | {%- set filtered_sql = {'sql': base_sql} -%} 24 | 25 | {%- do filtered_sql.update({'sql': automate_dv.replace_placeholder_with_period_filter(core_sql=filtered_sql.sql, 26 | timestamp_field=timestamp_field, 27 | start_timestamp=start_timestamp, 28 | stop_timestamp=stop_timestamp, 29 | offset=offset, period=period)}) -%} 30 | select {{ target_cols_csv }} from ({{ filtered_sql.sql }}) 31 | {%- endmacro %} 32 | 33 | 34 | 35 | 36 | {% macro sqlserver__get_period_filter_sql(target_cols_csv, base_sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%} 37 | {%- set filtered_sql = {'sql': base_sql} -%} 38 | 39 | {%- do filtered_sql.update({'sql': automate_dv.replace_placeholder_with_period_filter(core_sql=filtered_sql.sql, 40 | timestamp_field=timestamp_field, 41 | start_timestamp=start_timestamp, 42 | stop_timestamp=stop_timestamp, 43 | offset=offset, period=period)}) -%} 44 | {# MSSQL does not allow CTEs in a subquery #} 45 | {{ filtered_sql.sql }} 46 | {%- endmacro %} 47 | 48 | 49 | 50 | {% macro postgres__get_period_filter_sql(target_cols_csv, base_sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%} 51 | 52 | {%- set filtered_sql = {'sql': base_sql} -%} 53 | 54 | {%- do filtered_sql.update({'sql': automate_dv.replace_placeholder_with_period_filter(core_sql=filtered_sql.sql, 55 | timestamp_field=timestamp_field, 56 | start_timestamp=start_timestamp, 57 | stop_timestamp=stop_timestamp, 58 | offset=offset, period=period)}) -%} 59 | select {{ target_cols_csv }} from ({{ filtered_sql.sql }}) 60 | {%- endmacro %} 61 | -------------------------------------------------------------------------------- /macros/materialisations/period_mat_helpers/get_period_of_load.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro get_period_of_load(period, offset, start_timestamp) -%} 7 | 8 | {% set macro = adapter.dispatch('get_period_of_load', 9 | 'automate_dv')(period=period, 10 | offset=offset, 11 | start_timestamp=start_timestamp) %} 12 | 13 | {% do return(macro) %} 14 | {%- endmacro %} 15 | 16 | 17 | {%- macro default__get_period_of_load(period, offset, start_timestamp) -%} 18 | 19 | {% set period_of_load_sql -%} 20 | SELECT DATE_TRUNC('{{ period }}', DATEADD({{ period }}, {{ offset }}, TO_TIMESTAMP('{{ start_timestamp }}'))) AS period_of_load 21 | {%- endset %} 22 | 23 | {% set period_of_load_dict = automate_dv.get_query_results_as_dict(period_of_load_sql) %} 24 | 25 | {% set period_of_load = period_of_load_dict['PERIOD_OF_LOAD'][0] | string %} 26 | 27 | {% do return(period_of_load) %} 28 | {%- endmacro -%} 29 | 30 | 31 | {%- macro bigquery__get_period_of_load(period, offset, start_timestamp) -%} 32 | 33 | {% set period_of_load_sql -%} 34 | {%- if period is in ['millisecond', 'microsecond', 'second', 'minute', 'hour'] -%} 35 | SELECT TIMESTAMP_TRUNC(TIMESTAMP_ADD( TIMESTAMP('{{ start_timestamp }}'), INTERVAL {{ offset }} {{ period }}), {{ period }} ) AS PERIOD_OF_LOAD 36 | {%- elif period is in ['day', 'week', 'month', 'quarter', 'year'] -%} 37 | SELECT DATE_TRUNC(DATE_ADD( DATE('{{ start_timestamp }}'), INTERVAL {{ offset }} {{ period }}), {{ period }} ) AS PERIOD_OF_LOAD 38 | {%- else -%} 39 | SELECT DATE_TRUNC(DATE_ADD( DATE('{{ start_timestamp }}'), INTERVAL {{ offset }} {{ period }}), {{ period }} ) AS PERIOD_OF_LOAD 40 | {%- endif -%} 41 | {%- endset %} 42 | 43 | {% set period_of_load_dict = automate_dv.get_query_results_as_dict(period_of_load_sql) %} 44 | 45 | {% set period_of_load = period_of_load_dict['PERIOD_OF_LOAD'][0] | string %} 46 | 47 | {% do return(period_of_load) %} 48 | {%- endmacro -%} 49 | 50 | 51 | {%- macro sqlserver__get_period_of_load(period, offset, start_timestamp) -%} 52 | {# MSSQL cannot CAST datetime2 strings with more than 7 decimal places #} 53 | {% set start_timestamp_mssql = start_timestamp[0:23] %} 54 | 55 | {% set period_of_load_sql -%} 56 | SELECT DATEADD({{ period }}, DATEDIFF({{period}}, 0, DATEADD({{ period }}, {{ offset }}, CAST('{{ start_timestamp_mssql }}' AS DATETIME2))), 0) AS period_of_load 57 | {%- endset %} 58 | 59 | {% set period_of_load_dict = automate_dv.get_query_results_as_dict(period_of_load_sql) %} 60 | 61 | {% set period_of_load = period_of_load_dict['PERIOD_OF_LOAD'][0] | string %} 62 | 63 | {% do return(period_of_load) %} 64 | {%- endmacro -%} 65 | 66 | 67 | {%- macro databricks__get_period_of_load(period, offset, start_timestamp) -%} 68 | {% do return(automate_dv.default__get_period_of_load(period=period, offset=offset, start_timestamp=start_timestamp)) %} 69 | {%- endmacro -%} 70 | 71 | 72 | {%- macro postgres__get_period_of_load(period, offset, start_timestamp) -%} 73 | {# Postgres uses different DateTime arithmetic #} 74 | {% set period_of_load_sql -%} 75 | SELECT DATE_TRUNC('{{ period }}', TIMESTAMP '{{ start_timestamp }}' + INTERVAL '{{ offset }} {{ period }}') AS period_of_load 76 | {%- endset %} 77 | 78 | {% set period_of_load_dict = automate_dv.get_query_results_as_dict(period_of_load_sql) %} 79 | 80 | {% set period_of_load = period_of_load_dict['PERIOD_OF_LOAD'][0] | string %} 81 | 82 | {% do return(period_of_load) %} 83 | {%- endmacro -%} 84 | -------------------------------------------------------------------------------- /macros/tables/sqlserver/hub.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro sqlserver__hub(src_pk, src_nk, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_nk, src_extra_columns, src_ldts, src_source]) -%} 9 | 10 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 11 | {%- set source_cols_with_rank = source_cols + [automate_dv.config_meta_get('rank_column')] -%} 12 | {%- endif %} 13 | 14 | {{ 'WITH ' -}} 15 | 16 | {%- set stage_count = source_model | length -%} 17 | 18 | {%- set ns = namespace(last_cte= "") -%} 19 | 20 | {%- for src in source_model -%} 21 | 22 | {%- set source_number = loop.index | string -%} 23 | 24 | row_rank_{{ source_number }} AS ( 25 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 26 | SELECT {{ source_cols_with_rank | join(', ') }} 27 | {%- else %} 28 | SELECT {{ source_cols | join(', ') }} 29 | {%- endif %} 30 | FROM ( 31 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 32 | SELECT {{ automate_dv.prefix(source_cols_with_rank, 'rr') }}, 33 | {%- else %} 34 | SELECT {{ automate_dv.prefix(source_cols, 'rr') }}, 35 | {%- endif %} 36 | ROW_NUMBER() OVER( 37 | PARTITION BY {{ automate_dv.prefix([src_pk], 'rr') }} 38 | ORDER BY {{ automate_dv.prefix([src_ldts], 'rr') }} 39 | ) AS row_number 40 | FROM {{ ref(src) }} AS rr 41 | WHERE {{ automate_dv.multikey(src_pk, prefix='rr', condition='IS NOT NULL') }} 42 | ) h 43 | WHERE h.row_number = 1 44 | {%- set ns.last_cte = "row_rank_{}".format(source_number) %} 45 | ),{{ "\n" if not loop.last }} 46 | {% endfor -%} 47 | {% if stage_count > 1 %} 48 | stage_union AS ( 49 | {%- for src in source_model %} 50 | SELECT * FROM row_rank_{{ loop.index | string }} 51 | {%- if not loop.last %} 52 | UNION ALL 53 | {%- endif %} 54 | {%- endfor %} 55 | {%- set ns.last_cte = "stage_union" %} 56 | ), 57 | {%- endif -%} 58 | 59 | {%- if model.config.materialized == 'vault_insert_by_period' %} 60 | stage_mat_filter AS ( 61 | SELECT * 62 | FROM {{ ns.last_cte }} 63 | WHERE __PERIOD_FILTER__ 64 | {%- set ns.last_cte = "stage_mat_filter" %} 65 | ), 66 | {%- elif model.config.materialized == 'vault_insert_by_rank' %} 67 | stage_mat_filter AS ( 68 | SELECT * 69 | FROM {{ ns.last_cte }} 70 | WHERE __RANK_FILTER__ 71 | {%- set ns.last_cte = "stage_mat_filter" %} 72 | ), 73 | {%- endif -%} 74 | 75 | {%- if stage_count > 1 %} 76 | 77 | row_rank_union AS ( 78 | SELECT * 79 | FROM ( 80 | SELECT ru.*, 81 | ROW_NUMBER() OVER( 82 | PARTITION BY {{ automate_dv.prefix([src_pk], 'ru') }} 83 | ORDER BY {{ automate_dv.prefix([src_ldts], 'ru') }}, {{ automate_dv.prefix([src_source], 'ru') }} ASC 84 | ) AS row_rank_number 85 | FROM {{ ns.last_cte }} AS ru 86 | WHERE {{ automate_dv.multikey(src_pk, prefix='ru', condition='IS NOT NULL') }} 87 | ) h 88 | WHERE h.row_rank_number = 1 89 | {%- set ns.last_cte = "row_rank_union" %} 90 | ), 91 | {% endif %} 92 | records_to_insert AS ( 93 | SELECT {{ automate_dv.prefix(source_cols, 'a', alias_target='target') }} 94 | FROM {{ ns.last_cte }} AS a 95 | {%- if automate_dv.is_any_incremental() %} 96 | LEFT JOIN {{ this }} AS d 97 | ON {{ automate_dv.multikey(src_pk, prefix=['a','d'], condition='=') }} 98 | WHERE {{ automate_dv.multikey(src_pk, prefix='d', condition='IS NULL') }} 99 | {%- endif %} 100 | ) 101 | 102 | SELECT * FROM records_to_insert 103 | 104 | {%- endmacro -%} 105 | -------------------------------------------------------------------------------- /macros/tables/sqlserver/link.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro sqlserver__link(src_pk, src_fk, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_fk, src_extra_columns, src_ldts, src_source]) -%} 9 | {%- set fk_cols = automate_dv.expand_column_list([src_fk]) -%} 10 | 11 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 12 | {%- set source_cols_with_rank = source_cols + [automate_dv.config_meta_get('rank_column')] -%} 13 | {%- endif %} 14 | 15 | {{ 'WITH ' -}} 16 | 17 | {%- set stage_count = source_model | length -%} 18 | 19 | {%- set ns = namespace(last_cte= "") -%} 20 | 21 | {%- for src in source_model -%} 22 | 23 | {%- set source_number = loop.index | string -%} 24 | 25 | row_rank_{{ source_number }} AS ( 26 | SELECT * 27 | FROM 28 | ( 29 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 30 | SELECT {{ automate_dv.prefix(source_cols_with_rank, 'rr') }}, 31 | {%- else %} 32 | SELECT {{ automate_dv.prefix(source_cols, 'rr') }}, 33 | {%- endif %} 34 | ROW_NUMBER() OVER( 35 | PARTITION BY {{ automate_dv.prefix([src_pk], 'rr') }} 36 | ORDER BY {{ automate_dv.prefix([src_ldts], 'rr') }} 37 | ) AS row_number 38 | FROM {{ ref(src) }} AS rr 39 | {%- if stage_count == 1 %} 40 | WHERE {{ automate_dv.multikey(src_pk, prefix='rr', condition='IS NOT NULL') }} 41 | AND {{ automate_dv.multikey(fk_cols, prefix='rr', condition='IS NOT NULL') }} 42 | {%- endif %} 43 | ) l 44 | WHERE l.row_number = 1 45 | {%- set ns.last_cte = "row_rank_{}".format(source_number) %} 46 | ),{{ "\n" if not loop.last }} 47 | {% endfor -%} 48 | 49 | {% if stage_count > 1 %} 50 | stage_union AS ( 51 | {%- for src in source_model %} 52 | SELECT * FROM row_rank_{{ loop.index | string }} 53 | {%- if not loop.last %} 54 | UNION ALL 55 | {%- endif %} 56 | {%- endfor %} 57 | {%- set ns.last_cte = "stage_union" %} 58 | ), 59 | {%- endif -%} 60 | {%- if model.config.materialized == 'vault_insert_by_period' %} 61 | stage_mat_filter AS ( 62 | SELECT * 63 | FROM {{ ns.last_cte }} 64 | WHERE __PERIOD_FILTER__ 65 | {%- set ns.last_cte = "stage_mat_filter" %} 66 | ), 67 | {%- elif model.config.materialized == 'vault_insert_by_rank' %} 68 | stage_mat_filter AS ( 69 | SELECT * 70 | FROM {{ ns.last_cte }} 71 | WHERE __RANK_FILTER__ 72 | {%- set ns.last_cte = "stage_mat_filter" %} 73 | ), 74 | {% endif %} 75 | {%- if stage_count > 1 %} 76 | 77 | row_rank_union AS ( 78 | SELECT * 79 | FROM 80 | ( 81 | SELECT ru.*, 82 | ROW_NUMBER() OVER( 83 | PARTITION BY {{ automate_dv.prefix([src_pk], 'ru') }} 84 | ORDER BY {{ automate_dv.prefix([src_ldts], 'ru') }}, {{ automate_dv.prefix([src_source], 'ru') }} ASC 85 | ) AS row_rank_number 86 | FROM {{ ns.last_cte }} AS ru 87 | WHERE {{ automate_dv.multikey(src_pk, prefix='ru', condition='IS NOT NULL') }} 88 | AND {{ automate_dv.multikey(fk_cols, prefix='ru', condition='IS NOT NULL') }} 89 | ) r 90 | WHERE r.row_rank_number = 1 91 | {%- set ns.last_cte = "row_rank_union" %} 92 | ), 93 | {% endif %} 94 | records_to_insert AS ( 95 | SELECT {{ automate_dv.prefix(source_cols, 'a', alias_target='target') }} 96 | FROM {{ ns.last_cte }} AS a 97 | {%- if automate_dv.is_any_incremental() %} 98 | LEFT JOIN {{ this }} AS d 99 | ON {{ automate_dv.multikey(src_pk, prefix=['a','d'], condition='=') }} 100 | WHERE {{ automate_dv.multikey(src_pk, prefix='d', condition='IS NULL') }} 101 | {%- endif %} 102 | ) 103 | 104 | SELECT * FROM records_to_insert 105 | 106 | {%- endmacro -%} 107 | -------------------------------------------------------------------------------- /macros/tables/postgres/link.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro postgres__link(src_pk, src_fk, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_fk, src_extra_columns, src_ldts, src_source]) -%} 9 | {%- set fk_cols = automate_dv.expand_column_list([src_fk]) -%} 10 | 11 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 12 | {%- set source_cols_with_rank = source_cols + [automate_dv.config_meta_get('rank_column')] -%} 13 | {%- endif %} 14 | 15 | {{ 'WITH ' -}} 16 | 17 | {%- if not (source_model is iterable and source_model is not string) -%} 18 | {%- set source_model = [source_model] -%} 19 | {%- endif -%} 20 | 21 | {%- set ns = namespace(last_cte= "") -%} 22 | 23 | {%- for src in source_model -%} 24 | 25 | {%- set source_number = loop.index | string -%} 26 | 27 | row_rank_{{ source_number }} AS ( 28 | SELECT * FROM ( 29 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 30 | SELECT {{ automate_dv.prefix(source_cols_with_rank, 'rr') }}, 31 | {%- else %} 32 | SELECT {{ automate_dv.prefix(source_cols, 'rr') }}, 33 | {%- endif %} 34 | ROW_NUMBER() OVER( 35 | PARTITION BY {{ automate_dv.prefix([src_pk], 'rr') }} 36 | ORDER BY {{ automate_dv.prefix([src_ldts], 'rr') }} 37 | ) AS row_number 38 | FROM {{ ref(src) }} AS rr 39 | {%- if source_model | length == 1 %} 40 | WHERE {{ automate_dv.multikey(src_pk, prefix='rr', condition='IS NOT NULL') }} 41 | AND {{ automate_dv.multikey(fk_cols, prefix='rr', condition='IS NOT NULL') }} 42 | {%- endif %} 43 | ) as l 44 | WHERE row_number = 1 45 | {%- set ns.last_cte = "row_rank_{}".format(source_number) %} 46 | ),{{ "\n" if not loop.last }} 47 | {% endfor -%} 48 | {% if source_model | length > 1 %} 49 | stage_union AS ( 50 | {%- for src in source_model %} 51 | SELECT * FROM row_rank_{{ loop.index | string }} 52 | {%- if not loop.last %} 53 | UNION ALL 54 | {%- endif %} 55 | {%- endfor %} 56 | {%- set ns.last_cte = "stage_union" %} 57 | ), 58 | {%- endif -%} 59 | {%- if model.config.materialized == 'vault_insert_by_period' %} 60 | stage_mat_filter AS ( 61 | SELECT * 62 | FROM {{ ns.last_cte }} 63 | WHERE __PERIOD_FILTER__ 64 | {%- set ns.last_cte = "stage_mat_filter" %} 65 | ), 66 | {%- elif model.config.materialized == 'vault_insert_by_rank' %} 67 | stage_mat_filter AS ( 68 | SELECT * 69 | FROM {{ ns.last_cte }} 70 | WHERE __RANK_FILTER__ 71 | {%- set ns.last_cte = "stage_mat_filter" %} 72 | ), 73 | {% endif %} 74 | {%- if source_model | length > 1 %} 75 | 76 | row_rank_union AS ( 77 | SELECT * FROM ( 78 | SELECT ru.*, 79 | ROW_NUMBER() OVER( 80 | PARTITION BY {{ automate_dv.prefix([src_pk], 'ru') }} 81 | ORDER BY {{ automate_dv.prefix([src_ldts], 'ru') }}, {{ automate_dv.prefix([src_source], 'ru') }} ASC 82 | ) AS row_rank_number 83 | FROM {{ ns.last_cte }} AS ru 84 | WHERE {{ automate_dv.multikey(src_pk, prefix='ru', condition='IS NOT NULL') }} 85 | AND {{ automate_dv.multikey(fk_cols, prefix='ru', condition='IS NOT NULL') }} 86 | ) AS a 87 | WHERE row_rank_number = 1 88 | {%- set ns.last_cte = "row_rank_union" %} 89 | ), 90 | {% endif %} 91 | records_to_insert AS ( 92 | SELECT {{ automate_dv.prefix(source_cols, 'a', alias_target='target') }} 93 | FROM {{ ns.last_cte }} AS a 94 | {%- if automate_dv.is_any_incremental() %} 95 | LEFT JOIN {{ this }} AS d 96 | ON {{ automate_dv.multikey(src_pk, prefix=['a','d'], condition='=') }} 97 | WHERE {{ automate_dv.multikey(src_pk, prefix='d', condition='IS NULL') }} 98 | {%- endif %} 99 | ) 100 | 101 | SELECT * FROM records_to_insert 102 | 103 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/internal/helpers/logging/error_messages.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | 7 | {%- macro datepart_too_small_error(period) -%} 8 | 9 | {%- set message -%} 10 | This datepart ({{ period }}) is too small and not recommended, consider using a different datepart value (e.g. day) or rank column. 11 | 'vault_insert_by_x' materialisations are intended for experimental or testing purposes only. They are not intended for use in production. 12 | 13 | Please see: https://automate-dv.readthedocs.io/en/latest/materialisations/ 14 | {%- endset -%} 15 | 16 | {%- do automate_dv.log_error(message) -%} 17 | {%- endmacro -%} 18 | 19 | 20 | {%- macro datepart_not_recommended_warning(period) -%} 21 | 22 | {%- set message -%} 23 | This datepart ({{ period }}) is too small and not recommended, consider using a different datepart value (e.g. day) or rank column. 24 | 'vault_insert_by_x' materialisations are intended for experimental or testing purposes only. They are not intended for use in production. 25 | 26 | Please see: https://automate-dv.readthedocs.io/en/latest/materialisations/ 27 | {%- endset -%} 28 | 29 | {%- do automate_dv.log_warning(message) -%} 30 | {%- endmacro -%} 31 | 32 | 33 | {%- macro max_iterations_error() -%} 34 | 35 | {%- set message -%} 36 | Max iterations is 100,000. Consider using a different datepart value (e.g. day), rank column or loading data for a shorter time period. 37 | 'vault_insert_by_x' materialisations are intended for experimental or testing purposes only. They are not intended for use in production. 38 | 39 | Please see: https://automate-dv.readthedocs.io/en/latest/materialisations/ 40 | {%- endset %} 41 | 42 | {%- do automate_dv.log_warning(message) -%} 43 | {%- endmacro -%} 44 | 45 | 46 | {%- macro experimental_not_recommended_warning(func_name) -%} 47 | 48 | {%- set message -%} 49 | This functionality ({{ func_name }}) is intended for experimental or testing purposes only. 50 | Its behavior, reliability, and performance have not been thoroughly vetted for production environments. 51 | Using this functionality in a live production setting may result in unpredictable outcomes, data loss, or system instability. 52 | {%- endset -%} 53 | 54 | {%- do automate_dv.log_warning(message) -%} 55 | {%- endmacro -%} 56 | 57 | 58 | {%- macro currently_disabled_error(func_name) -%} 59 | 60 | {%- set message -%} 61 | This functionality ({{ func_name }}) is currently disabled for dbt-sqlserver 1.7.x, 62 | please revert to dbt-sqlserver 1.4.3 and AutomateDV 0.10.1 to use {{ func_name }}. 63 | 64 | This is due to a suspected bug with the SQLServer Adapter in the 1.7.x version. 65 | We are actively working to get this fixed. Thank you for your understanding. 66 | {%- endset -%} 67 | 68 | {%- do automate_dv.log_error(message) -%} 69 | {%- endmacro -%} 70 | 71 | 72 | {%- macro materialisation_deprecation_warning() -%} 73 | 74 | {%- set message -%} 75 | DEPRECATED: Since AutomateDV v0.11.0, vault_insert_by_x materialisations are now deprecated. 76 | These materialisation were initially designed to provide an option for rapid iterative development of 77 | incremental loading patterns in local environments for development and testing, allowing users to bypass 78 | the need for a comprehensive PSA or delta-loading solution. They are being deprecated to encourage the use 79 | of more robust solutions. 80 | {%- endset -%} 81 | 82 | {%- do automate_dv.log_warning(message) -%} 83 | {%- endmacro -%} 84 | 85 | 86 | {%- macro pit_bridge_deprecation_warning() -%} 87 | 88 | {%- set message -%} 89 | DEPRECATED: Since AutomateDV v0.11.0, the pit() and bridge() macros are now deprecated. 90 | This is because they are not currently fit-for-purpose and need significant usability 91 | and peformance improvements, as well as a design overhaul. 92 | Improved implementations will be released in a future version of AutomateDV. 93 | {%- endset -%} 94 | 95 | {%- do automate_dv.log_warning(message) -%} 96 | {%- endmacro -%} 97 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | AutomateDV 3 | AutomateDV 4 | 5 | [![Documentation Status](https://img.shields.io/badge/docs-stable-blue)](https://automate-dv.readthedocs.io/en/stable/?badge=stable) 6 | [![Slack](https://img.shields.io/badge/Slack-Join-yellow?style=flat&logo=slack)](https://join.slack.com/t/dbtvault/shared_invite/enQtODY5MTY3OTIyMzg2LWJlZDMyNzM4YzAzYjgzYTY0MTMzNTNjN2EyZDRjOTljYjY0NDYyYzEwMTlhODMzNGY3MmU2ODNhYWUxYmM2NjA) 7 |
8 |
9 | 10 | [![dbt Versions](https://img.shields.io/badge/compatible%20dbt%20versions-%3E=1.9.x%20%3C=3.0.0-orange?logo=dbt)](https://automate-dv.readthedocs.io/en/latest/versions/) 11 | 12 |
13 | 14 | [Changelog and past doc versions](https://automate-dv.readthedocs.io/en/latest/changelog/) 15 | 16 | # AutomateDV by [Datavault](https://www.data-vault.com) 17 | 18 | Build your own Data Vault data warehouse! AutomateDV is a free to use dbt package that generates & executes the ETL you need to run a Data Vault 2.0 Data Warehouse on your data platform. 19 | 20 | What does AutomateDV offer? 21 | - productivity gains, fewer errors 22 | - multi-threaded execution of the generated SQL 23 | - your data modeller can generate most of the ETL code directly from their mapping metadata 24 | - your ETL developers can focus on the 5% of the SQL code that is different 25 | - dbt generates documentation and data flow diagrams 26 | 27 | powered by [dbt](https://www.getdbt.com/), a registered trademark of [dbt Labs](https://www.getdbt.com/dbt-labs/about-us/) 28 | 29 | ## Worked example project 30 | 31 | Learn quickly with our worked example: 32 | 33 | - [Read the docs](https://automate-dv.readthedocs.io/en/latest/worked_example/) 34 | 35 | - [Project Repository](https://github.com/Datavault-UK/automate-dv-demo) 36 | 37 | ## Supported platforms: 38 | 39 | [Platform support matrix](https://automate-dv.readthedocs.io/en/latest/platform_support/) 40 | 41 | ## Installation 42 | 43 | Check [dbt Hub](https://hub.getdbt.com/Datavault-UK/automate_dv/latest/) for the latest installation instructions, 44 | or [read the docs](https://docs.getdbt.com/docs/build/packages) for more information on installing packages. 45 | 46 | ## Usage 47 | 48 | 1. Create a model for your table. 49 | 2. Provide metadata 50 | 3. Call the appropriate template macro 51 | 52 | ```bash 53 | # Configure model 54 | {{- config(...) -}} 55 | 56 | # Provide metadata 57 | {%- set src_pk = ... -%} 58 | ... 59 | 60 | # Call the macro 61 | {{ automate_dv.hub(src_pk, src_nk, src_ldts, 62 | src_source, source_model) }} 63 | ``` 64 | 65 | ## Join our Slack Channel 66 | 67 | Talk to our developers and other members of our growing community, get support and discuss anything related to AutomateDV or Data Vault 2.0 68 | 69 | [![Join our Slack](https://img.shields.io/badge/Slack-Join-yellow?style=flat&logo=slack)](https://join.slack.com/t/dbtvault/shared_invite/enQtODY5MTY3OTIyMzg2LWJlZDMyNzM4YzAzYjgzYTY0MTMzNTNjN2EyZDRjOTljYjY0NDYyYzEwMTlhODMzNGY3MmU2ODNhYWUxYmM2NjA) 70 | 71 | 72 | ## Social 73 | 74 | [![Twitter Follow](https://img.shields.io/badge/Twitter-1DA1F2?style=for-the-badge&logo=twitter&logoColor=white)](https://twitter.com/Automate_DV) 75 | 76 | [![LinkedIn](https://img.shields.io/badge/LinkedIn-0077B5?style=for-the-badge&logo=linkedin&logoColor=white)](https://www.linkedin.com/showcase/automate-dv/) 77 | 78 | [![Youtube](https://img.shields.io/badge/YouTube-FF0000?style=for-the-badge&logo=youtube&logoColor=white)](https://www.youtube.com/@AutomateDV) 79 | 80 | 81 | ## Awards 82 | 83 |

84 | 85 | innovation awards 87 | 88 |

89 | 90 | ## Contributing 91 | [View our contribution guidelines](CONTRIBUTING.md) 92 | 93 | ## License 94 | [Apache 2.0](LICENSE) 95 | -------------------------------------------------------------------------------- /macros/tables/snowflake/hub.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro hub(src_pk, src_nk, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.check_required_parameters(src_pk=src_pk, src_nk=src_nk, 9 | src_ldts=src_ldts, src_source=src_source, 10 | source_model=source_model) -}} 11 | 12 | {%- if not automate_dv.is_list(source_model) -%} 13 | {%- set source_model = [source_model] -%} 14 | {%- endif -%} 15 | 16 | {{ automate_dv.log_relation_sources(this, source_model | length) }} 17 | 18 | {{- automate_dv.prepend_generated_by() }} 19 | 20 | {{ adapter.dispatch('hub', 'automate_dv')(src_pk=src_pk, src_nk=src_nk, 21 | src_extra_columns=src_extra_columns, 22 | src_ldts=src_ldts, src_source=src_source, 23 | source_model=source_model) -}} 24 | 25 | {%- endmacro -%} 26 | 27 | {%- macro default__hub(src_pk, src_nk, src_extra_columns, src_ldts, src_source, source_model) -%} 28 | 29 | {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_nk, src_extra_columns, src_ldts, src_source]) -%} 30 | 31 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 32 | {%- set source_cols_with_rank = source_cols + [automate_dv.config_meta_get('rank_column')] -%} 33 | {%- endif %} 34 | 35 | {{ 'WITH ' -}} 36 | 37 | {%- set stage_count = source_model | length -%} 38 | 39 | {%- set ns = namespace(last_cte= "") -%} 40 | 41 | {%- for src in source_model -%} 42 | 43 | {%- set source_number = loop.index | string -%} 44 | 45 | row_rank_{{ source_number }} AS ( 46 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 47 | SELECT {{ automate_dv.prefix(source_cols_with_rank, 'rr') }} 48 | {%- else %} 49 | SELECT {{ automate_dv.prefix(source_cols, 'rr') }} 50 | {%- endif %} 51 | FROM {{ ref(src) }} AS rr 52 | WHERE {{ automate_dv.multikey(src_pk, prefix='rr', condition='IS NOT NULL') }} 53 | QUALIFY ROW_NUMBER() OVER( 54 | PARTITION BY {{ automate_dv.prefix([src_pk], 'rr') }} 55 | ORDER BY {{ automate_dv.prefix([src_ldts], 'rr') }} 56 | ) = 1 57 | {%- set ns.last_cte = "row_rank_{}".format(source_number) %} 58 | ),{{ "\n" if not loop.last }} 59 | {% endfor -%} 60 | {% if stage_count > 1 %} 61 | stage_union AS ( 62 | {%- for src in source_model %} 63 | SELECT * FROM row_rank_{{ loop.index | string }} 64 | {%- if not loop.last %} 65 | UNION ALL 66 | {%- endif %} 67 | {%- endfor %} 68 | {%- set ns.last_cte = "stage_union" %} 69 | ), 70 | {%- endif -%} 71 | 72 | {%- if model.config.materialized == 'vault_insert_by_period' %} 73 | stage_mat_filter AS ( 74 | SELECT * 75 | FROM {{ ns.last_cte }} 76 | WHERE __PERIOD_FILTER__ 77 | {%- set ns.last_cte = "stage_mat_filter" %} 78 | ), 79 | {%- elif model.config.materialized == 'vault_insert_by_rank' %} 80 | stage_mat_filter AS ( 81 | SELECT * 82 | FROM {{ ns.last_cte }} 83 | WHERE __RANK_FILTER__ 84 | {%- set ns.last_cte = "stage_mat_filter" %} 85 | ), 86 | {%- endif -%} 87 | 88 | {%- if stage_count > 1 %} 89 | 90 | row_rank_union AS ( 91 | SELECT ru.* 92 | FROM {{ ns.last_cte }} AS ru 93 | WHERE {{ automate_dv.multikey(src_pk, prefix='ru', condition='IS NOT NULL') }} 94 | QUALIFY ROW_NUMBER() OVER( 95 | PARTITION BY {{ automate_dv.prefix([src_pk], 'ru') }} 96 | ORDER BY {{ automate_dv.prefix([src_ldts], 'ru') }}, {{ automate_dv.prefix([src_source], 'ru') }} ASC 97 | ) = 1 98 | {%- set ns.last_cte = "row_rank_union" %} 99 | ), 100 | {% endif %} 101 | records_to_insert AS ( 102 | SELECT {{ automate_dv.prefix(source_cols, 'a', alias_target='target') }} 103 | FROM {{ ns.last_cte }} AS a 104 | {%- if automate_dv.is_any_incremental() %} 105 | LEFT JOIN {{ this }} AS d 106 | ON {{ automate_dv.multikey(src_pk, prefix=['a','d'], condition='=') }} 107 | WHERE {{ automate_dv.multikey(src_pk, prefix='d', condition='IS NULL') }} 108 | {%- endif %} 109 | ) 110 | 111 | SELECT * FROM records_to_insert 112 | 113 | {%- endmacro -%} 114 | -------------------------------------------------------------------------------- /macros/supporting/as_of_date_window.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro as_of_date_window(src_pk, src_ldts, stage_tables_ldts, source_model) -%} 7 | 8 | last_safe_load_datetime AS ( 9 | SELECT MIN(LOAD_DATETIME) AS LAST_SAFE_LOAD_DATETIME 10 | FROM ( 11 | 12 | {% for stg in stage_tables_ldts -%} 13 | {%- set stage_ldts = stage_tables_ldts[stg] -%} 14 | SELECT MIN({{ stage_ldts }}) AS LOAD_DATETIME FROM {{ ref(stg) }} 15 | {% if not loop.last %} UNION ALL {% endif %} 16 | {% endfor -%} 17 | 18 | ) AS l 19 | ), 20 | 21 | as_of_grain_old_entries AS ( 22 | SELECT DISTINCT AS_OF_DATE 23 | FROM {{ this }} 24 | ), 25 | 26 | as_of_grain_lost_entries AS ( 27 | SELECT a.AS_OF_DATE 28 | FROM as_of_grain_old_entries AS a 29 | LEFT OUTER JOIN as_of_dates AS b 30 | ON a.AS_OF_DATE = b.AS_OF_DATE 31 | WHERE b.AS_OF_DATE IS NULL 32 | ), 33 | 34 | as_of_grain_new_entries AS ( 35 | SELECT a.AS_OF_DATE 36 | FROM as_of_dates AS a 37 | LEFT OUTER JOIN as_of_grain_old_entries AS b 38 | ON a.AS_OF_DATE = b.AS_OF_DATE 39 | WHERE b.AS_OF_DATE IS NULL 40 | ), 41 | 42 | min_date AS ( 43 | SELECT MIN(AS_OF_DATE) AS MIN_DATE 44 | FROM as_of_dates 45 | ), 46 | 47 | backfill_as_of AS ( 48 | SELECT AS_OF_DATE 49 | FROM as_of_dates AS a 50 | 51 | {% if target.type == "bigquery" -%} 52 | INNER JOIN last_safe_load_datetime as l 53 | ON a.AS_OF_DATE < l.LAST_SAFE_LOAD_DATETIME 54 | {% else %} 55 | WHERE a.AS_OF_DATE < (SELECT LAST_SAFE_LOAD_DATETIME FROM last_safe_load_datetime) 56 | {%- endif %} 57 | ), 58 | 59 | new_rows_pks AS ( 60 | SELECT {{ automate_dv.prefix([src_pk], 'h') }} 61 | FROM {{ source_model }} AS h 62 | 63 | {% if target.type == "bigquery" -%} 64 | INNER JOIN last_safe_load_datetime as l 65 | ON h.{{ src_ldts }} >= l.LAST_SAFE_LOAD_DATETIME 66 | {% else %} 67 | WHERE h.{{ src_ldts }} >= (SELECT LAST_SAFE_LOAD_DATETIME FROM last_safe_load_datetime) 68 | {%- endif %} 69 | ), 70 | 71 | new_rows_as_of AS ( 72 | SELECT AS_OF_DATE 73 | FROM as_of_dates AS a 74 | {% if target.type == "bigquery" -%} 75 | INNER JOIN last_safe_load_datetime as l 76 | ON a.AS_OF_DATE >= l.LAST_SAFE_LOAD_DATETIME 77 | UNION DISTINCT 78 | {% else %} 79 | WHERE a.AS_OF_DATE >= (SELECT LAST_SAFE_LOAD_DATETIME FROM last_safe_load_datetime) 80 | UNION 81 | {%- endif %} 82 | SELECT AS_OF_DATE 83 | FROM as_of_grain_new_entries 84 | ), 85 | 86 | overlap_pks AS ( 87 | SELECT a.* 88 | FROM {{ this }} AS a 89 | INNER JOIN {{ source_model }} as b 90 | ON {{ automate_dv.multikey(src_pk, prefix=['a','b'], condition='=') }} 91 | {% if target.type == "bigquery" -%} 92 | INNER JOIN min_date 93 | ON 1 = 1 94 | INNER JOIN last_safe_load_datetime 95 | ON 1 = 1 96 | LEFT OUTER JOIN as_of_grain_lost_entries 97 | ON a.AS_OF_DATE = as_of_grain_lost_entries.AS_OF_DATE 98 | WHERE a.AS_OF_DATE >= min_date.MIN_DATE 99 | AND a.AS_OF_DATE < last_safe_load_datetime.LAST_SAFE_LOAD_DATETIME 100 | AND as_of_grain_lost_entries.AS_OF_DATE IS NULL 101 | {% else %} 102 | WHERE a.AS_OF_DATE >= (SELECT MIN_DATE FROM min_date) 103 | AND a.AS_OF_DATE < (SELECT LAST_SAFE_LOAD_DATETIME FROM last_safe_load_datetime) 104 | AND a.AS_OF_DATE NOT IN (SELECT AS_OF_DATE FROM as_of_grain_lost_entries) 105 | {%- endif %} 106 | ), 107 | 108 | overlap_as_of AS ( 109 | SELECT p.AS_OF_DATE 110 | FROM as_of_dates AS p 111 | {% if target.type == "bigquery" -%} 112 | INNER JOIN min_date 113 | ON 1 = 1 114 | INNER JOIN last_safe_load_datetime 115 | ON 1 = 1 116 | LEFT OUTER JOIN as_of_grain_lost_entries 117 | ON p.AS_OF_DATE = as_of_grain_lost_entries.AS_OF_DATE 118 | WHERE p.AS_OF_DATE >= min_date.MIN_DATE 119 | AND p.AS_OF_DATE < last_safe_load_datetime.LAST_SAFE_LOAD_DATETIME 120 | AND as_of_grain_lost_entries.AS_OF_DATE IS NULL 121 | {% else %} 122 | WHERE p.AS_OF_DATE >= (SELECT MIN_DATE FROM min_date) 123 | AND p.AS_OF_DATE < (SELECT LAST_SAFE_LOAD_DATETIME FROM last_safe_load_datetime) 124 | AND p.AS_OF_DATE NOT IN (SELECT AS_OF_DATE FROM as_of_grain_lost_entries) 125 | {% endif %} 126 | ) 127 | 128 | {%- endmacro -%} 129 | -------------------------------------------------------------------------------- /macros/supporting/ghost_records/create_ghost_record.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro create_ghost_record(src_pk, src_hashdiff, src_payload, src_extra_columns, src_eff, src_ldts, src_source, source_model) -%} 7 | 8 | {{- adapter.dispatch('create_ghost_record', 'automate_dv')(src_pk=src_pk, src_hashdiff=src_hashdiff, 9 | src_payload=src_payload, src_extra_columns=src_extra_columns, 10 | src_eff=src_eff, src_ldts=src_ldts, 11 | src_source=src_source, source_model=source_model) -}} 12 | 13 | {%- endmacro -%} 14 | 15 | {%- macro default__create_ghost_record(src_pk, src_hashdiff, src_payload, src_extra_columns, src_eff, src_ldts, src_source, source_model) -%} 16 | 17 | {%- set hash = var('hash', 'MD5') -%} 18 | {%- set source_str = var('system_record_value', 'AUTOMATE_DV_SYSTEM') -%} 19 | {%- set enable_native_hashes = var('enable_native_hashes', false) -%} 20 | 21 | {%- set columns_in_source = adapter.get_columns_in_relation(ref(source_model)) -%} 22 | {%- set src_hashdiff_name = src_hashdiff['source_column'] | default(src_hashdiff) -%} 23 | {%- set col_definitions = [] -%} 24 | 25 | {%- set binary_columns = automate_dv.expand_column_list([src_pk, src_hashdiff_name]) | map('lower') | list -%} 26 | {%- set null_columns = automate_dv.expand_column_list([src_payload, src_extra_columns]) | map('lower') | list -%} 27 | {%- set time_columns = automate_dv.expand_column_list([src_ldts, src_eff]) | map('lower') | list -%} 28 | 29 | {%- set all_columns = automate_dv.expand_column_list([src_pk, src_hashdiff_name, src_payload, src_extra_columns, 30 | src_eff, src_ldts, src_source]) | list -%} 31 | 32 | {%- if target.type == 'bigquery' and not enable_native_hashes -%} 33 | {%- set warning_message -%} 34 | WARNING: In AutomateDV v0.10.2 and earlier, BigQuery used the STRING data type for hashes. 35 | If native hashes are disabled for BigQuery, all columns in the src_pk and src_hashdiff 36 | parameters will use a string of zeros (0000...) instead of the correct hash data type. 37 | To resolve this, enable native hashes at your earliest convenience. 38 | {%- endset -%} 39 | {%- do automate_dv.log_warning(warning_message) -%} 40 | {%- endif -%} 41 | 42 | {%- set filtered_source_columns = [] -%} 43 | {%- for column in columns_in_source -%} 44 | {%- if column.column | lower in all_columns | map('lower') -%} 45 | {%- do filtered_source_columns.append(column) -%} 46 | {%- endif -%} 47 | {%- endfor -%} 48 | 49 | {%- set all_columns = all_columns | map('lower') | list -%} 50 | 51 | {%- for col in filtered_source_columns -%} 52 | 53 | {%- set col_name = col.column -%} 54 | {%- set col_compare = col_name | lower -%} 55 | {%- set col_type = col.dtype | lower -%} 56 | {%- set source_system_str = var('system_record_value', 'AUTOMATE_DV_SYSTEM') -%} 57 | 58 | {# If src_pk col, use binary ghost unless composite #} 59 | {%- if col_compare in binary_columns -%} 60 | {%- if target.type == 'bigquery' and not enable_native_hashes -%} 61 | {%- do col_definitions.append(automate_dv.binary_ghost(alias=col_name, hash=hash)) -%} 62 | {%- else -%} 63 | {%- do col_definitions.append(automate_dv.ghost_for_type(col_type, col_name)) -%} 64 | {%- endif -%} 65 | {# If record source col, replace with system value #} 66 | {%- elif col_compare == (src_source | lower) -%} 67 | {%- set col_sql -%} 68 | CAST('{{ source_system_str }}' AS {{ col.dtype }}) AS {{ src_source }} 69 | {%- endset -%} 70 | {%- do col_definitions.append(col_sql) -%} 71 | {# If column in payload, make its ghost representation NULL #} 72 | {%- elif col_compare in null_columns -%} 73 | {%- do col_definitions.append(automate_dv.null_ghost(data_type=col_type, alias=col_name)) -%} 74 | {# Handle anything else as its correct ghost representation #} 75 | {%- else -%} 76 | {%- do col_definitions.append(automate_dv.ghost_for_type(col_type, col_name)) -%} 77 | {%- endif -%} 78 | 79 | {%- endfor -%} 80 | 81 | SELECT 82 | {% for column_def in col_definitions -%} 83 | {{ column_def }} 84 | {%- if not loop.last -%}, 85 | {% endif %} 86 | {%- endfor -%} 87 | 88 | {%- endmacro -%} 89 | 90 | -------------------------------------------------------------------------------- /macros/tables/snowflake/link.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro link(src_pk, src_fk, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.check_required_parameters(src_pk=src_pk, src_fk=src_fk, 9 | src_ldts=src_ldts, src_source=src_source, 10 | source_model=source_model) -}} 11 | 12 | {%- if not automate_dv.is_list(source_model) -%} 13 | {%- set source_model = [source_model] -%} 14 | {%- endif -%} 15 | 16 | {{ automate_dv.log_relation_sources(this, source_model | length) }} 17 | 18 | {{- automate_dv.prepend_generated_by() }} 19 | 20 | {{ adapter.dispatch('link', 'automate_dv')(src_pk=src_pk, src_fk=src_fk, 21 | src_extra_columns=src_extra_columns, 22 | src_ldts=src_ldts, src_source=src_source, 23 | source_model=source_model) -}} 24 | 25 | {%- endmacro -%} 26 | 27 | {%- macro default__link(src_pk, src_fk, src_extra_columns, src_ldts, src_source, source_model) -%} 28 | 29 | {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_fk, src_extra_columns, src_ldts, src_source]) -%} 30 | {%- set fk_cols = automate_dv.expand_column_list([src_fk]) -%} 31 | 32 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 33 | {%- set source_cols_with_rank = source_cols + [automate_dv.config_meta_get('rank_column')] -%} 34 | {%- endif %} 35 | 36 | {{ 'WITH ' -}} 37 | 38 | {%- set stage_count = source_model | length -%} 39 | 40 | {%- set ns = namespace(last_cte= "") -%} 41 | 42 | {%- for src in source_model -%} 43 | 44 | {%- set source_number = loop.index | string -%} 45 | 46 | row_rank_{{ source_number }} AS ( 47 | {%- if model.config.materialized == 'vault_insert_by_rank' %} 48 | SELECT {{ automate_dv.prefix(source_cols_with_rank, 'rr') }} 49 | {%- else %} 50 | SELECT {{ automate_dv.prefix(source_cols, 'rr') }} 51 | {%- endif %} 52 | FROM {{ ref(src) }} AS rr 53 | {%- if stage_count == 1 %} 54 | WHERE {{ automate_dv.multikey(src_pk, prefix='rr', condition='IS NOT NULL') }} 55 | AND {{ automate_dv.multikey(fk_cols, prefix='rr', condition='IS NOT NULL') }} 56 | {%- endif %} 57 | QUALIFY ROW_NUMBER() OVER( 58 | PARTITION BY {{ automate_dv.prefix([src_pk], 'rr') }} 59 | ORDER BY {{ automate_dv.prefix([src_ldts], 'rr') }} 60 | ) = 1 61 | {%- set ns.last_cte = "row_rank_{}".format(source_number) %} 62 | ),{{ "\n" if not loop.last }} 63 | {% endfor -%} 64 | 65 | {% if stage_count > 1 %} 66 | stage_union AS ( 67 | {%- for src in source_model %} 68 | SELECT * FROM row_rank_{{ loop.index | string }} 69 | {%- if not loop.last %} 70 | UNION ALL 71 | {%- endif %} 72 | {%- endfor %} 73 | {%- set ns.last_cte = "stage_union" %} 74 | ), 75 | {%- endif -%} 76 | {%- if model.config.materialized == 'vault_insert_by_period' %} 77 | stage_mat_filter AS ( 78 | SELECT * 79 | FROM {{ ns.last_cte }} 80 | WHERE __PERIOD_FILTER__ 81 | {%- set ns.last_cte = "stage_mat_filter" %} 82 | ), 83 | {%- elif model.config.materialized == 'vault_insert_by_rank' %} 84 | stage_mat_filter AS ( 85 | SELECT * 86 | FROM {{ ns.last_cte }} 87 | WHERE __RANK_FILTER__ 88 | {%- set ns.last_cte = "stage_mat_filter" %} 89 | ), 90 | {% endif %} 91 | {%- if stage_count > 1 %} 92 | 93 | row_rank_union AS ( 94 | SELECT ru.* 95 | FROM {{ ns.last_cte }} AS ru 96 | WHERE {{ automate_dv.multikey(src_pk, prefix='ru', condition='IS NOT NULL') }} 97 | AND {{ automate_dv.multikey(fk_cols, prefix='ru', condition='IS NOT NULL') }} 98 | QUALIFY ROW_NUMBER() OVER( 99 | PARTITION BY {{ automate_dv.prefix([src_pk], 'ru') }} 100 | ORDER BY {{ automate_dv.prefix([src_ldts], 'ru') }}, {{ automate_dv.prefix([src_source], 'ru') }} ASC 101 | ) = 1 102 | {%- set ns.last_cte = "row_rank_union" %} 103 | ), 104 | {% endif %} 105 | records_to_insert AS ( 106 | SELECT {{ automate_dv.prefix(source_cols, 'a', alias_target='target') }} 107 | FROM {{ ns.last_cte }} AS a 108 | {%- if automate_dv.is_any_incremental() %} 109 | LEFT JOIN {{ this }} AS d 110 | ON {{ automate_dv.multikey(src_pk, prefix=['a','d'], condition='=') }} 111 | WHERE {{ automate_dv.multikey(src_pk, prefix='d', condition='IS NULL') }} 112 | {%- endif %} 113 | ) 114 | 115 | SELECT * FROM records_to_insert 116 | 117 | {%- endmacro -%} 118 | -------------------------------------------------------------------------------- /macros/supporting/hash.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro hash(columns=none, alias=none, is_hashdiff=false, columns_to_escape=none) -%} 7 | 8 | {%- if is_hashdiff is none -%} 9 | {%- set is_hashdiff = false -%} 10 | {%- endif -%} 11 | 12 | {{- adapter.dispatch('hash', 'automate_dv')(columns=columns, alias=alias, 13 | is_hashdiff=is_hashdiff, columns_to_escape=columns_to_escape) -}} 14 | 15 | {%- endmacro %} 16 | 17 | {%- macro default__hash(columns, alias, is_hashdiff, columns_to_escape) -%} 18 | 19 | {%- set hash = var('hash', 'md5') -%} 20 | {%- set concat_string = var('concat_string', '||') -%} 21 | {%- set null_placeholder_string = var('null_placeholder_string', '^^') -%} 22 | 23 | {%- set hash_alg = automate_dv.select_hash_alg(hash) -%} 24 | 25 | {%- set standardise = automate_dv.standard_column_wrapper() %} 26 | 27 | {#- Alpha sort columns before hashing if a hashdiff -#} 28 | {%- if is_hashdiff and automate_dv.is_list(columns) -%} 29 | {%- set columns = columns|sort -%} 30 | {%- endif -%} 31 | 32 | {#- If single column to hash -#} 33 | {%- if columns is string -%} 34 | {%- set column_str = automate_dv.as_constant(columns) -%} 35 | 36 | {%- if automate_dv.is_something(columns_to_escape) -%} 37 | {%- if column_str in columns_to_escape -%} 38 | {%- set column_str = automate_dv.escape_column_name(column_str) -%} 39 | {%- endif -%} 40 | {%- endif -%} 41 | 42 | {{ hash_alg | replace('[HASH_STRING_PLACEHOLDER]', standardise | replace('[EXPRESSION]', column_str)) }} AS {{ alias | indent(4) }} 43 | 44 | {#- Else a list of columns to hash -#} 45 | {%- else -%} 46 | 47 | {%- set all_null = [] -%} 48 | {%- set processed_columns = [] -%} 49 | 50 | {%- for column in columns -%} 51 | {%- if automate_dv.is_something(columns_to_escape) -%} 52 | {%- if column in columns_to_escape -%} 53 | {%- set column = automate_dv.escape_column_name(column) -%} 54 | {%- endif -%} 55 | {%- endif -%} 56 | 57 | {%- set column_str = automate_dv.as_constant(column) -%} 58 | 59 | {%- set column_expression = automate_dv.null_expression(column_str) -%} 60 | 61 | {%- do all_null.append(null_placeholder_string) -%} 62 | {%- do processed_columns.append(column_expression) -%} 63 | 64 | {% endfor -%} 65 | 66 | {% if not is_hashdiff -%} 67 | 68 | {%- set concat_sql -%} 69 | NULLIF({{ automate_dv.concat_ws(processed_columns, separator=concat_string) -}} {{ ', ' -}} 70 | '{{ all_null | join(concat_string) }}') 71 | {%- endset -%} 72 | 73 | {%- set hashed_column -%} 74 | {{ hash_alg | replace('[HASH_STRING_PLACEHOLDER]', concat_sql) }} AS {{ alias }} 75 | {%- endset -%} 76 | 77 | {%- else -%} 78 | {% if automate_dv.is_list(processed_columns) and processed_columns | length > 1 %} 79 | {%- set hashed_column -%} 80 | {{ hash_alg | replace('[HASH_STRING_PLACEHOLDER]', automate_dv.concat_ws(processed_columns, separator=concat_string)) }} AS {{ alias }} 81 | {%- endset -%} 82 | {%- else -%} 83 | {%- set hashed_column -%} 84 | {{ hash_alg | replace('[HASH_STRING_PLACEHOLDER]', processed_columns[0]) }} AS {{ alias }} 85 | {%- endset -%} 86 | {%- endif -%} 87 | {%- endif -%} 88 | 89 | {{ hashed_column }} 90 | 91 | {%- endif -%} 92 | 93 | {%- endmacro -%} 94 | 95 | 96 | {%- macro bigquery__hash(columns, alias, is_hashdiff, columns_to_escape) -%} 97 | 98 | {{ automate_dv.default__hash(columns=columns, alias=alias, is_hashdiff=is_hashdiff, columns_to_escape=columns_to_escape) }} 99 | 100 | {%- endmacro -%} 101 | 102 | 103 | {%- macro sqlserver__hash(columns, alias, is_hashdiff, columns_to_escape) -%} 104 | 105 | {{ automate_dv.default__hash(columns=columns, alias=alias, is_hashdiff=is_hashdiff, columns_to_escape=columns_to_escape) }} 106 | 107 | {%- endmacro -%} 108 | 109 | 110 | {%- macro postgres__hash(columns, alias, is_hashdiff, columns_to_escape) -%} 111 | 112 | {{ automate_dv.default__hash(columns=columns, alias=alias, is_hashdiff=is_hashdiff, columns_to_escape=columns_to_escape) }} 113 | 114 | {%- endmacro -%} 115 | 116 | 117 | {%- macro databricks__hash(columns, alias, is_hashdiff, columns_to_escape) -%} 118 | 119 | {{ automate_dv.default__hash(columns=columns, alias=alias, is_hashdiff=is_hashdiff, columns_to_escape=columns_to_escape) }} 120 | 121 | {%- endmacro -%} -------------------------------------------------------------------------------- /macros/tables/snowflake/xts.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro xts(src_pk, src_satellite, src_extra_columns, src_ldts, src_source, source_model) -%} 7 | 8 | {{- automate_dv.check_required_parameters(src_pk=src_pk, src_satellite=src_satellite, 9 | src_ldts=src_ldts, src_source=src_source, 10 | source_model=source_model) -}} 11 | 12 | {%- if not automate_dv.is_list(source_model) -%} 13 | {%- set source_model = [source_model] -%} 14 | {%- endif -%} 15 | 16 | {{ automate_dv.prepend_generated_by() }} 17 | 18 | {{ adapter.dispatch('xts', 'automate_dv')(src_pk=src_pk, 19 | src_satellite=src_satellite, 20 | src_extra_columns=src_extra_columns, 21 | src_ldts=src_ldts, 22 | src_source=src_source, 23 | source_model=source_model) -}} 24 | {%- endmacro -%} 25 | 26 | {%- macro default__xts(src_pk, src_satellite, src_extra_columns, src_ldts, src_source, source_model) -%} 27 | 28 | {%- set hashdiff_col_name_alias = 'HASHDIFF' -%} 29 | {%- set satellite_name_col_name_alias = 'SATELLITE_NAME' %} 30 | {%- set satellite_count = src_satellite.keys() | list | length %} 31 | {%- set stage_count = source_model | length %} 32 | 33 | {%- if execute -%} 34 | {%- do dbt_utils.log_info('Loading {} from {} source(s) and {} satellite(s)'.format("{}.{}.{}".format(this.database, this.schema, this.identifier), 35 | stage_count, satellite_count)) -%} 36 | {%- endif %} 37 | 38 | {%- set ns = namespace(last_cte= "") %} 39 | 40 | {{ 'WITH ' }} 41 | {%- for src in source_model -%} 42 | {%- for satellite in src_satellite.items() -%} 43 | {%- set satellite_name = (satellite[1]['sat_name'].values() | list)[0] -%} 44 | {%- set hashdiff = (satellite[1]['hashdiff'].values() | list)[0] %} 45 | {%- set cte_name = "satellite_{}_from_{}".format(satellite_name, src) | lower %} 46 | 47 | {{ cte_name }} AS ( 48 | SELECT {{ automate_dv.prefix([src_pk], 's') }}, 49 | s.{{ hashdiff }} AS {{ hashdiff_col_name_alias }}, 50 | s.{{ satellite_name }} AS {{ satellite_name_col_name_alias }}, 51 | {%- if automate_dv.is_something(src_extra_columns) -%} 52 | {{ automate_dv.prefix([src_extra_columns], 's') }}, 53 | {%- endif %} 54 | s.{{ src_ldts }}, 55 | s.{{ src_source }} 56 | FROM {{ ref(src) }} AS s 57 | WHERE {{ automate_dv.multikey(src_pk, prefix='s', condition='IS NOT NULL') }} 58 | ), 59 | 60 | {%- set ns.last_cte = cte_name %} 61 | {%- endfor %} 62 | {%- endfor %} 63 | 64 | {%- if stage_count > 1 or satellite_count > 1 %} 65 | 66 | union_satellites AS ( 67 | {%- for src in source_model %} 68 | {%- for satellite in src_satellite.items() %} 69 | {%- set satellite_name = (satellite[1]['sat_name'].values() | list)[0] %} 70 | {%- set cte_name = "satellite_{}_from_{}".format(satellite_name, src) | lower %} 71 | SELECT * FROM {{ cte_name }} 72 | {%- if not loop.last %} 73 | UNION ALL 74 | {%- endif %} 75 | {%- endfor %} 76 | {%- if not loop.last %} 77 | UNION ALL 78 | {%- endif %} 79 | {%- endfor %} 80 | ), 81 | {%- set ns.last_cte = "union_satellites" -%} 82 | {%- endif %} 83 | 84 | records_to_insert AS ( 85 | SELECT DISTINCT 86 | {{ automate_dv.prefix([src_pk], 'a') }}, 87 | a.{{ hashdiff_col_name_alias }}, 88 | a.{{ satellite_name_col_name_alias }}, 89 | {%- if automate_dv.is_something(src_extra_columns) -%} 90 | {{ automate_dv.prefix([src_extra_columns], 'a') }}, 91 | {%- endif %} 92 | a.{{ src_ldts }}, 93 | a.{{ src_source }} 94 | FROM {{ ns.last_cte }} AS a 95 | {%- if automate_dv.is_any_incremental() %} 96 | LEFT JOIN {{ this }} AS d 97 | ON ( 98 | a.{{ hashdiff_col_name_alias }} = d.{{ hashdiff_col_name_alias }} 99 | AND a.{{ src_ldts }} = d.{{ src_ldts }} 100 | AND a.{{ satellite_name_col_name_alias }} = d.{{ satellite_name_col_name_alias }} 101 | ) 102 | WHERE d.{{ hashdiff_col_name_alias }} IS NULL 103 | AND d.{{ src_ldts }} IS NULL 104 | AND d.{{ satellite_name_col_name_alias }} IS NULL 105 | {%- endif %} 106 | ) 107 | 108 | SELECT * FROM records_to_insert 109 | 110 | {%- endmacro -%} 111 | -------------------------------------------------------------------------------- /macros/materialisations/period_mat_helpers/replace_placeholder_with_period_filter.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro replace_placeholder_with_period_filter(core_sql, timestamp_field, start_timestamp, stop_timestamp, offset, period) -%} 7 | 8 | {% set macro = adapter.dispatch('replace_placeholder_with_period_filter', 9 | 'automate_dv')(core_sql=core_sql, 10 | timestamp_field=timestamp_field, 11 | start_timestamp=start_timestamp, 12 | stop_timestamp=stop_timestamp, 13 | offset=offset, 14 | period=period) %} 15 | {% do return(macro) %} 16 | {%- endmacro %} 17 | 18 | 19 | {% macro default__replace_placeholder_with_period_filter(core_sql, timestamp_field, start_timestamp, stop_timestamp, offset, period) %} 20 | 21 | {%- set period_filter -%} 22 | ( 23 | TO_TIMESTAMP({{ timestamp_field }}) >= DATE_TRUNC('{{ period }}', TO_TIMESTAMP('{{ start_timestamp }}') + INTERVAL '{{ offset }} {{ period }}') 24 | AND TO_TIMESTAMP({{ timestamp_field }}) < DATE_TRUNC('{{ period }}', TO_TIMESTAMP('{{ start_timestamp }}') + INTERVAL '{{ offset }} {{ period }}' + INTERVAL '1 {{ period }}')) 25 | AND (TO_TIMESTAMP({{ timestamp_field }}) >= TO_TIMESTAMP('{{ start_timestamp }}') 26 | ) 27 | {%- endset -%} 28 | {%- set filtered_sql = core_sql | replace("__PERIOD_FILTER__", period_filter) -%} 29 | 30 | {% do return(filtered_sql) %} 31 | {% endmacro %} 32 | 33 | 34 | {% macro bigquery__replace_placeholder_with_period_filter(core_sql, timestamp_field, start_timestamp, stop_timestamp, offset, period) %} 35 | {%- if period is in ['day', 'week', 'month', 'quarter', 'year'] -%} 36 | {%- set timestamp_field_type = 'DATE' -%} 37 | {%- elif period is in ['millisecond', 'microsecond', 'second', 'minute', 'hour'] -%} 38 | {%- set timestamp_field_type = 'TIMESTAMP' -%} 39 | {%- else -%} 40 | {%- set timestamp_field_type = 'DATE' -%} 41 | {%- endif -%} 42 | 43 | {%- set period_filter -%} 44 | ({{ timestamp_field_type }}({{ timestamp_field }}) >= DATE_TRUNC({{ timestamp_field_type }}_ADD( {{ timestamp_field_type }}('{{ start_timestamp }}'), INTERVAL {{ offset }} {{ period }}), {{ period }} ) AND 45 | {{ timestamp_field_type }}({{ timestamp_field }}) < DATE_TRUNC({{ timestamp_field_type }}_ADD(TIMESTAMP_ADD( {{ timestamp_field_type }}('{{ start_timestamp }}'), INTERVAL {{ offset }} {{ period }}), INTERVAL 1 {{ period }}), {{ period }} ) 46 | AND TIMESTAMP({{ timestamp_field }}) >= TIMESTAMP('{{ start_timestamp }}')) 47 | {%- endset -%} 48 | 49 | {%- set filtered_sql = core_sql | replace("__PERIOD_FILTER__", period_filter) -%} 50 | 51 | {% do return(filtered_sql) %} 52 | {% endmacro %} 53 | 54 | 55 | {% macro sqlserver__replace_placeholder_with_period_filter(core_sql, timestamp_field, start_timestamp, stop_timestamp, offset, period) %} 56 | {%- if period is in ['microsecond', 'millisecond', 'second'] -%} 57 | {{ automate_dv.sqlserver_datepart_too_small_error(period=period) }} 58 | {%- endif -%} 59 | 60 | {# MSSQL cannot CAST datetime2 strings with more than 7 decimal places #} 61 | {% set start_timestamp_mssql = start_timestamp[0:27] %} 62 | 63 | {%- set period_filter -%} 64 | ( 65 | CAST({{ timestamp_field }} AS DATETIME2) >= DATEADD({{ period }}, DATEDIFF({{ period }}, 0, DATEADD({{ period }}, {{ offset }}, CAST('{{ start_timestamp_mssql }}' AS DATETIME2))), 0) 66 | AND CAST({{ timestamp_field }} AS DATETIME2) < DATEADD({{ period }}, 1, DATEADD({{ period }}, {{ offset }}, CAST('{{ start_timestamp_mssql }}' AS DATETIME2))) 67 | AND (CAST({{ timestamp_field }} AS DATETIME2) >= CAST('{{ start_timestamp_mssql }}' AS DATETIME2)) 68 | ) 69 | {%- endset -%} 70 | 71 | {%- set filtered_sql = core_sql | replace("__PERIOD_FILTER__", period_filter) -%} 72 | 73 | {% do return(filtered_sql) %} 74 | {% endmacro %} 75 | 76 | 77 | {% macro postgres__replace_placeholder_with_period_filter(core_sql, timestamp_field, start_timestamp, stop_timestamp, offset, period) %} 78 | 79 | {%- set period_filter -%} 80 | {{ timestamp_field }}::TIMESTAMP >= DATE_TRUNC('{{ period }}', TIMESTAMP '{{ start_timestamp }}' + INTERVAL '{{ offset }} {{ period }}') 81 | AND {{ timestamp_field }}::TIMESTAMP < DATE_TRUNC('{{ period }}', TIMESTAMP '{{ start_timestamp }}' + INTERVAL '{{ offset }} {{ period }}' + INTERVAL '1 {{ period }}') 82 | AND {{ timestamp_field }}::TIMESTAMP >= TIMESTAMP '{{ start_timestamp }}' 83 | {%- endset -%} 84 | {%- set filtered_sql = core_sql | replace("__PERIOD_FILTER__", period_filter) -%} 85 | 86 | {% do return(filtered_sql) %} 87 | {% endmacro %} 88 | -------------------------------------------------------------------------------- /macros/staging/derive_columns.sql: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Business Thinking Ltd. 2019-2025 3 | * This software includes code developed by the AutomateDV (f.k.a dbtvault) Team at Business Thinking Ltd. Trading as Datavault 4 | */ 5 | 6 | {%- macro derive_columns(source_relation=none, columns=none) -%} 7 | 8 | {{- adapter.dispatch('derive_columns', 'automate_dv')(source_relation=source_relation, columns=columns) -}} 9 | 10 | {%- endmacro %} 11 | 12 | {%- macro default__derive_columns(source_relation=none, columns=none) -%} 13 | 14 | {%- set exclude_columns = [] -%} 15 | {%- set include_columns = [] -%} 16 | {%- set src_columns = [] -%} 17 | {%- set der_columns = [] -%} 18 | 19 | {%- set source_cols = automate_dv.source_columns(source_relation=source_relation) -%} 20 | 21 | {%- if columns is mapping and columns is not none -%} 22 | 23 | {#- Add aliases of derived columns to excludes and full SQL to includes -#} 24 | {%- for derived_column in columns -%} 25 | {%- set column_config = columns[derived_column] -%} 26 | 27 | {%- if automate_dv.is_list(column_config) -%} 28 | {%- set column_list = [] -%} 29 | 30 | {%- for concat_component in column_config -%} 31 | {%- set column_str = automate_dv.as_constant(concat_component) -%} 32 | {%- do column_list.append(column_str) -%} 33 | {%- endfor -%} 34 | 35 | {%- set concat = automate_dv.concat_ws(column_list, "||") -%} 36 | {%- set concat_string = concat ~ " AS " ~ derived_column -%} 37 | 38 | {%- do der_columns.append(concat_string) -%} 39 | {%- else -%} 40 | {%- if column_config is mapping and column_config -%} 41 | {%- set column_escape = column_config['escape'] -%} 42 | 43 | {%- if automate_dv.is_list(column_config['source_column']) -%} 44 | {%- set column_list = [] -%} 45 | 46 | {%- for concat_component in column_config['source_column'] -%} 47 | {%- set column_str = automate_dv.as_constant(concat_component) -%} 48 | {%- if column_escape is true %} 49 | {%- set column_str = automate_dv.escape_column_names(column_str) -%} 50 | {% endif %} 51 | {%- do column_list.append(column_str) -%} 52 | {%- endfor -%} 53 | 54 | {%- set concat = automate_dv.concat_ws(column_list, "||") -%} 55 | {%- set concat_string = concat ~ " AS " ~ derived_column -%} 56 | 57 | {%- do der_columns.append(concat_string) -%} 58 | {%- else -%} 59 | {%- set column_str = automate_dv.as_constant(column_config['source_column']) -%} 60 | {%- if column_escape is true -%} 61 | {%- do der_columns.append(automate_dv.escape_column_names(column_str) ~ " AS " ~ derived_column) -%} 62 | {%- else -%} 63 | {%- do der_columns.append(column_str ~ " AS " ~ derived_column) -%} 64 | {%- endif -%} 65 | {%- endif -%} 66 | {%- else -%} 67 | {%- set column_str = automate_dv.as_constant(column_config) -%} 68 | {%- do der_columns.append(column_str ~ " AS " ~ derived_column) -%} 69 | {%- endif -%} 70 | {%- endif -%} 71 | 72 | {%- do exclude_columns.append(derived_column) -%} 73 | 74 | {%- endfor -%} 75 | 76 | {#- Add all columns from source_model relation -#} 77 | {%- if source_relation is defined and source_relation is not none -%} 78 | 79 | {%- for col in source_cols -%} 80 | {%- if col | lower not in exclude_columns | map('lower') | list -%} 81 | {%- do src_columns.append(col) -%} 82 | {%- endif -%} 83 | {%- endfor -%} 84 | 85 | {%- endif -%} 86 | 87 | {#- Makes sure the columns are appended in a logical order. Source columns then derived columns -#} 88 | {%- set include_columns = src_columns + der_columns -%} 89 | {%- set columns_to_escape = automate_dv.process_columns_to_escape(columns) | list -%} 90 | 91 | {#- Print out all columns in includes -#} 92 | {%- for col in include_columns -%} 93 | {%- if col | lower in columns_to_escape | map('lower') | list -%} 94 | {{- automate_dv.escape_column_name(col) -}}{{ ",\n" if not loop.last }} 95 | 96 | {%- else -%} 97 | {{- col -}}{{ ",\n" if not loop.last }} 98 | {%- endif -%} 99 | {%- endfor -%} 100 | 101 | {%- else -%} 102 | 103 | {%- if execute -%} 104 | 105 | {{ exceptions.raise_compiler_error("Invalid column configuration: 106 | expected format, either: {'source_relation': Relation, 'columns': {column_name: column_value}} 107 | or: {'source_relation': Relation, 'columns': {column_name: {'source_column': column_value, 'escape': true / false}}} 108 | got: {'source_relation': " ~ source_relation ~ ", 'columns': " ~ columns ~ "}") }} 109 | {%- endif %} 110 | 111 | {%- endif %} 112 | 113 | {%- endmacro -%} 114 | --------------------------------------------------------------------------------