├── .gitignore ├── packages.yml ├── dashboards ├── dashboard1.png ├── dashboard2.png ├── dashboard3.png ├── dbt_dataquality.pbix ├── dbt_dataquality_dashboard_preview.gif └── dbt_dataquality-high_level_architecture.png ├── macros ├── load_log_manifest.sql ├── create_resources.sql ├── table │ ├── drop_src_table.sql │ ├── create_src_table.sql │ └── load_src_table.sql ├── stage │ ├── clean_internal_stage.sql │ ├── load_internal_stage.sql │ └── create_internal_stage.sql ├── _get_config.sql ├── schema │ └── create_schema.sql ├── load_log_sources.sql └── load_log_tests.sql ├── dbt_project.yml ├── models ├── tests │ ├── tests_details.sql │ ├── tests_coverage.sql │ ├── tests_overview.sql │ ├── raw_tests_manifest.sql │ └── raw_tests.sql ├── sources.yml └── sources │ ├── sources_overview.sql │ ├── sources_details.sql │ ├── raw_source_freshness_manifest.sql │ └── raw_source_freshness.sql ├── README.md └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /packages/ 3 | /logs/ -------------------------------------------------------------------------------- /packages.yml: -------------------------------------------------------------------------------- 1 | packages: 2 | - package: dbt-labs/dbt_utils 3 | version: 0.8.2 -------------------------------------------------------------------------------- /dashboards/dashboard1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Divergent-Insights/dbt-dataquality/HEAD/dashboards/dashboard1.png -------------------------------------------------------------------------------- /dashboards/dashboard2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Divergent-Insights/dbt-dataquality/HEAD/dashboards/dashboard2.png -------------------------------------------------------------------------------- /dashboards/dashboard3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Divergent-Insights/dbt-dataquality/HEAD/dashboards/dashboard3.png -------------------------------------------------------------------------------- /dashboards/dbt_dataquality.pbix: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Divergent-Insights/dbt-dataquality/HEAD/dashboards/dbt_dataquality.pbix -------------------------------------------------------------------------------- /dashboards/dbt_dataquality_dashboard_preview.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Divergent-Insights/dbt-dataquality/HEAD/dashboards/dbt_dataquality_dashboard_preview.gif -------------------------------------------------------------------------------- /dashboards/dbt_dataquality-high_level_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Divergent-Insights/dbt-dataquality/HEAD/dashboards/dbt_dataquality-high_level_architecture.png -------------------------------------------------------------------------------- /macros/load_log_manifest.sql: -------------------------------------------------------------------------------- 1 | {% macro load_log_manifest() %} 2 | 3 | {% set config = _get_config() %} 4 | {% set log_file = config["dbt_target_path"] ~ '/manifest.json' %} 5 | 6 | {{ load_internal_stage(file=log_file) }} 7 | 8 | {% endmacro %} 9 | -------------------------------------------------------------------------------- /macros/create_resources.sql: -------------------------------------------------------------------------------- 1 | {% macro create_resources(dry_run=False, internal_stage=true, create_default_schema=true) %} 2 | 3 | {% if create_default_schema %} 4 | {{ create_schema(dry_run) }} 5 | {% endif %} 6 | 7 | {% if internal_stage %} 8 | {{ create_internal_stage(dry_run) }} 9 | {% endif %} 10 | 11 | {{ create_src_table(dry_run) }} 12 | 13 | {% endmacro %} 14 | -------------------------------------------------------------------------------- /dbt_project.yml: -------------------------------------------------------------------------------- 1 | name: 'dbt_dataquality' 2 | version: '0.0.1' 3 | 4 | config-version: 2 5 | 6 | require-dbt-version: ">=1.0.0" 7 | 8 | model-paths: ["models"] 9 | macro-paths: ["macros"] 10 | test-paths: ["tests"] 11 | log-path: "logs" 12 | packages-install-path: "packages" 13 | target-path: "target" 14 | clean-targets: ["target", "packages"] 15 | 16 | models: 17 | dbt_dataquality: 18 | sources: 19 | +tags: sources 20 | tests: 21 | +tags: tests 22 | -------------------------------------------------------------------------------- /macros/table/drop_src_table.sql: -------------------------------------------------------------------------------- 1 | {% macro drop_src_table() %} 2 | 3 | {% do log("drop_src_table started", info=True) %} 4 | {% set config = _get_config() %} 5 | 6 | {% set sql %} 7 | drop table if exists {{ config["database"] }}.{{ config["schema"] }}.{{ config["table"] }} 8 | {% endset %} 9 | {% do run_query(sql) %} 10 | {% do log(sql, info=True) %} 11 | 12 | {% do log("drop_src_table completed", info=True) %} 13 | 14 | {% endmacro %} 15 | -------------------------------------------------------------------------------- /macros/stage/clean_internal_stage.sql: -------------------------------------------------------------------------------- 1 | {% macro clean_internal_stage() %} 2 | 3 | {% do log("clean_internal_stage started", info=True) %} 4 | {% set config = _get_config() %} 5 | 6 | {% set sql %} 7 | remove @{{ config["database"] }}.{{ config["schema"] }}.{{ config["stage"] }} pattern='.*.*'; 8 | {% endset %} 9 | {% do run_query(sql) %} 10 | {% do log(sql, info=True) %} 11 | 12 | {% do log("clean_internal_stage completed", info=True) %} 13 | 14 | {% endmacro %} 15 | -------------------------------------------------------------------------------- /macros/_get_config.sql: -------------------------------------------------------------------------------- 1 | {% macro _get_config() %} 2 | 3 | {{ 4 | return( 5 | { 6 | "database" : var('dbt_dataquality_database', target.database), 7 | "schema" : var('dbt_dataquality_schema', target.schema), 8 | "table" : var('dbt_dataquality_table', 'stg_dbt_dataquality'), 9 | "stage" : var('dbt_dataquality_stage', 'dbt_dataquality'), 10 | "dbt_target_path" : (var('dbt_dataquality_target_path', 'target')).rstrip("/") 11 | } 12 | ) 13 | }} 14 | 15 | {% endmacro %} 16 | -------------------------------------------------------------------------------- /macros/schema/create_schema.sql: -------------------------------------------------------------------------------- 1 | {% macro create_schema(dry_run=false) %} 2 | 3 | {% do log("create_schema started", info=True) %} 4 | 5 | {% set config = _get_config() %} 6 | 7 | {% set sql %} 8 | create schema if not exists {{ config["database"] }}.{{ config["schema"] }}; 9 | {% endset %} 10 | 11 | {% if not dry_run %} 12 | {% do run_query(sql) %} 13 | {% endif %} 14 | 15 | {% do log(sql, info=True) %} 16 | 17 | {% do log("create_schema completed", info=True) %} 18 | 19 | {% endmacro %} 20 | -------------------------------------------------------------------------------- /macros/stage/load_internal_stage.sql: -------------------------------------------------------------------------------- 1 | {% macro load_internal_stage(file) %} 2 | 3 | {% do log("load_internal_stage started", info=True) %} 4 | {% set config = _get_config() %} 5 | 6 | -- Populating internal stage 7 | {% set sql %} 8 | put file://{{ file }} @{{ config["database"] }}.{{ config["schema"] }}.{{ config["stage"] }} auto_compress=true; 9 | {% endset %} 10 | {% do run_query(sql) %} 11 | {% do log(sql, info=True) %} 12 | 13 | {% do log("load_internal_stage completed", info=True) %} 14 | 15 | {% endmacro %} 16 | -------------------------------------------------------------------------------- /macros/load_log_sources.sql: -------------------------------------------------------------------------------- 1 | {% macro load_log_sources(load_from_internal_stage=true, clean_stage=true) %} 2 | 3 | -- Removing all files from the internal stage 4 | {% if clean_stage %} 5 | {{ clean_internal_stage() }} 6 | {% endif %} 7 | 8 | {% set config = _get_config() %} 9 | {% set log_file = config["dbt_target_path"] ~ '/sources.json' %} 10 | 11 | {% if load_from_internal_stage %} 12 | {{ load_log_manifest() }} 13 | {{ load_internal_stage(file=log_file) }} 14 | {% endif %} 15 | 16 | {{ load_src_table() }} 17 | 18 | {% endmacro %} 19 | -------------------------------------------------------------------------------- /macros/load_log_tests.sql: -------------------------------------------------------------------------------- 1 | {% macro load_log_tests(load_from_internal_stage=true, clean_stage=true) %} 2 | 3 | -- Removing all files from the internal stage 4 | {% if clean_stage %} 5 | {{ clean_internal_stage() }} 6 | {% endif %} 7 | 8 | {% set config = _get_config() %} 9 | {% set log_file = config["dbt_target_path"] ~ '/run_results.json' %} 10 | 11 | {% if load_from_internal_stage %} 12 | {{ load_log_manifest() }} 13 | {{ load_internal_stage(file=log_file) }} 14 | {% endif %} 15 | 16 | {{ load_src_table() }} 17 | 18 | {% endmacro %} 19 | -------------------------------------------------------------------------------- /macros/stage/create_internal_stage.sql: -------------------------------------------------------------------------------- 1 | {% macro create_internal_stage(dry_run=false) %} 2 | 3 | {% do log("create_internal_stage started", info=True) %} 4 | {% set config = _get_config() %} 5 | 6 | {% set sql %} 7 | create stage if not exists {{ config["database"] }}.{{ config["schema"] }}.{{ config["stage"] }} 8 | file_format = ( type = json ); 9 | {% endset %} 10 | 11 | {% if not dry_run %} 12 | {% do run_query(sql) %} 13 | {% endif %} 14 | 15 | {% do log(sql, info=True) %} 16 | 17 | {% do log("create_internal_stage completed", info=True) %} 18 | 19 | {% endmacro %} 20 | -------------------------------------------------------------------------------- /models/tests/tests_details.sql: -------------------------------------------------------------------------------- 1 | with latest_records as 2 | ( 3 | select 4 | payload_id 5 | ,payload_timestamp_utc 6 | ,unique_id 7 | ,iff(status='success', 'pass', status) status 8 | ,case 9 | when (status = 'error') then 100 10 | when (status = 'fail') then 50 11 | when (status = 'pass' or status = 'success') then 0 12 | else -1 13 | end as status_code 14 | from {{ ref('raw_tests') }} 15 | where payload_timestamp_utc = (select max(payload_timestamp_utc)from {{ ref('raw_tests') }}) 16 | ) 17 | select 18 | tm.payload_id 19 | ,tm.payload_timestamp_utc 20 | ,tm.name test_name 21 | ,tm.tags quality_tag 22 | ,tm.database 23 | ,split_part(tm.file_key_name, '.', -1) table_name 24 | ,tm.column_name 25 | ,lr.status 26 | ,lr.status_code 27 | from latest_records lr 28 | left join {{ ref('raw_tests_manifest') }} tm 29 | on lr.unique_id = tm.unique_id 30 | -------------------------------------------------------------------------------- /macros/table/create_src_table.sql: -------------------------------------------------------------------------------- 1 | {% macro create_src_table(replace=false, dry_run=false) %} 2 | 3 | {% do log("create_src_table started", info=True) %} 4 | {% set config = _get_config() %} 5 | 6 | {% set sql %} 7 | {% if replace == false %} 8 | create table if not exists {{ config["database"] }}.{{ config["schema"] }}.{{ config["table"] }} 9 | {% else %} 10 | create or replace table {{ config["database"] }}.{{ config["schema"] }}.{{ config["table"] }} 11 | {% endif %} 12 | ( 13 | upload_timestamp_utc timestamp_tz, 14 | filename string, 15 | payload variant, 16 | payload_timestamp_utc timestamp_tz, 17 | payload_id string 18 | ); 19 | {% endset %} 20 | 21 | {% if not dry_run %} 22 | {% do run_query(sql) %} 23 | {% endif %} 24 | 25 | {% do log(sql, info=True) %} 26 | 27 | {% do log("create_src_table completed", info=True) %} 28 | 29 | {% endmacro %} 30 | -------------------------------------------------------------------------------- /macros/table/load_src_table.sql: -------------------------------------------------------------------------------- 1 | {% macro load_src_table() %} 2 | 3 | {% do log("load_src_table started", info=True) %} 4 | {% set config = _get_config() %} 5 | 6 | {% set sql %} 7 | begin; 8 | copy into {{ config["database"] }}.{{ config["schema"] }}.{{ config["table"] }} 9 | from 10 | ( 11 | select 12 | sysdate()::timestamp_tz as upload_timestamp_utc, 13 | metadata$filename as filename, 14 | $1 as payload, 15 | $1:metadata:generated_at::timestamp_tz as payload_timestamp_utc, 16 | $1:metadata:invocation_id::string as payload_id 17 | from @{{ config["database"] }}.{{ config["schema"] }}.{{ config["stage"] }} 18 | ) 19 | file_format=(type='json') 20 | on_error='skip_file'; 21 | commit; 22 | {% endset %} 23 | {% do run_query(sql) %} 24 | {% do log(sql, info=True) %} 25 | 26 | {% do log("load_src_table completed", info=True) %} 27 | 28 | {% endmacro %} 29 | -------------------------------------------------------------------------------- /models/sources.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sources: 4 | - name: dbt_dataquality 5 | database: "{{ var('dbt_dataquality_database', target.database) }}" 6 | schema: "{{ var('dbt_dataquality_schema', target.schema) }}" 7 | loader: dbt macro - dbt_dataquality.load_internal_stage 8 | description: sources.json and run_results.json raw data 9 | 10 | tables: 11 | - name: stg_dbt_dataquality 12 | identifier: "{{ var('dbt_dataquality_table', 'stg_dbt_dataquality') }}" 13 | description: Holds the raw payloads from sources.json and run_results.json 14 | columns: 15 | - name: upload_timestamp_utc 16 | description: The UTC time when the dbt log was uploaded (sources.json or run_results.json) 17 | - name: filename 18 | description: The file name of the dbt log uploaded (sources.json or run_results.json) 19 | - name: payload 20 | description: The actual json payload of the dbt log uploaded (sources.json or run_results.json) 21 | - name: payload_timestamp_utc 22 | description: Same as the element "generated_at" as found in the dbt logs (sources.json or run_results.json) 23 | - name: payload_id 24 | description: Same as the element "invocation_id" as found in the dbt logs (sources.json or run_results.json) 25 | -------------------------------------------------------------------------------- /models/tests/tests_coverage.sql: -------------------------------------------------------------------------------- 1 | with error as 2 | ( 3 | select quality_tag, count(*) error 4 | from {{ ref('tests_details') }} 5 | where status = 'error' 6 | group by quality_tag 7 | ) 8 | ,fail as 9 | ( 10 | select quality_tag, count(*) fail 11 | from {{ ref('tests_details') }} 12 | where status = 'fail' 13 | group by quality_tag 14 | ) 15 | ,pass as 16 | ( 17 | select quality_tag, count(*) pass 18 | from {{ ref('tests_details') }} 19 | where status = 'pass' 20 | group by quality_tag 21 | ) 22 | ,all_status as 23 | ( 24 | select 25 | distinct 26 | td.payload_id 27 | ,td.payload_timestamp_utc 28 | ,td.quality_tag 29 | ,coalesce(error.error,0) error 30 | ,coalesce(fail.fail,0) fail 31 | ,coalesce(pass.pass,0) pass 32 | ,(coalesce(error.error,0) + coalesce(fail.fail,0) + coalesce(pass.pass,0)) total 33 | from {{ ref('tests_details') }} td 34 | left join error on td.quality_tag = error.quality_tag 35 | left join fail on td.quality_tag = fail.quality_tag 36 | left join pass on td.quality_tag = pass.quality_tag 37 | ) 38 | select 39 | * 40 | ,sum(total) over (partition by payload_id) as tests_count 41 | ,sum(pass) over (partition by payload_id) as tests_passed 42 | ,((sum(pass) over (partition by payload_id))*100)/(sum(total) over (partition by payload_id)) as overall_tests_success 43 | ,(pass * 100 / total) as quality_coverage 44 | from all_status -------------------------------------------------------------------------------- /models/tests/tests_overview.sql: -------------------------------------------------------------------------------- 1 | with latest_records as 2 | ( 3 | select payload_id, iff(status='success', 'pass', status) status, payload_timestamp_utc 4 | from {{ ref('raw_tests') }} 5 | where payload_timestamp_utc = (select max(payload_timestamp_utc) from {{ ref('raw_tests') }}) 6 | ), 7 | grouped_results as 8 | ( 9 | select payload_id, status, count(status) status_count 10 | from latest_records 11 | group by payload_id, status 12 | ), 13 | pivot_results as 14 | ( 15 | select 16 | payload_id, ifnull("'error'",0) as error, ifnull("'fail'",0) as fail, ifnull("'pass'",0) as pass 17 | from grouped_results 18 | pivot(sum(status_count) for status in ('pass', 'fail', 'error')) 19 | ), 20 | clean_pivot_results as 21 | ( 22 | select 23 | payload_id 24 | ,case 25 | when (error > 0 or fail > 0) then 'Warning: some data quality issues were detected' 26 | else 'It seems that everything is okay' 27 | end as status 28 | ,case 29 | when (error > 0 or fail > 0) then 1 30 | else 0 31 | end as status_code 32 | ,error 33 | ,fail 34 | ,pass 35 | from pivot_results 36 | ) 37 | select 38 | distinct 39 | lr.payload_id 40 | ,lr.payload_timestamp_utc 41 | ,cpv.status 42 | ,cpv.status_code 43 | ,cpv.error 44 | ,cpv.fail 45 | ,cpv.pass 46 | from latest_records lr 47 | left join clean_pivot_results cpv on cpv.payload_id = lr.payload_id 48 | -------------------------------------------------------------------------------- /models/sources/sources_overview.sql: -------------------------------------------------------------------------------- 1 | with latest_records as 2 | ( 3 | select payload_id, status, payload_timestamp_utc 4 | from {{ ref('raw_source_freshness') }} 5 | where payload_timestamp_utc = (select max(payload_timestamp_utc) from {{ ref('raw_source_freshness') }}) 6 | ), 7 | grouped_results as 8 | ( 9 | select payload_id, status, count(status) status_count 10 | from latest_records 11 | group by payload_id, status 12 | ), 13 | pivot_results as 14 | ( 15 | select 16 | payload_id, ifnull("'error'",0) as stale, ifnull("'warn'",0) as warning, ifnull("'pass'",0) as pass 17 | from grouped_results 18 | pivot(sum(status_count) for status in ('error', 'warn', 'pass')) 19 | ), 20 | clean_pivot_results as 21 | ( 22 | select 23 | payload_id 24 | ,case 25 | when (stale > 0 or warning > 0) then 'Warning: some data sources require attention' 26 | else 'It seems that everything is okay' 27 | end as status 28 | ,case 29 | when (stale > 0 or warning > 0) then 1 30 | else 0 31 | end as status_code 32 | ,stale 33 | ,warning 34 | ,pass 35 | from pivot_results pr 36 | ) 37 | select 38 | distinct 39 | lr.payload_id 40 | ,lr.payload_timestamp_utc 41 | ,cpv.status 42 | ,cpv.status_code 43 | ,cpv.stale 44 | ,cpv.warning 45 | ,cpv.pass 46 | from latest_records lr 47 | left join clean_pivot_results cpv on cpv.payload_id = lr.payload_id 48 | -------------------------------------------------------------------------------- /models/sources/sources_details.sql: -------------------------------------------------------------------------------- 1 | with latest_records as 2 | ( 3 | select 4 | payload_id 5 | ,payload_timestamp_utc 6 | ,unique_id 7 | ,status 8 | ,case 9 | when (status = 'error') then 100 10 | when (status = 'warn') then 50 11 | when (status = 'pass') then 0 12 | else -1 13 | end as status_code 14 | ,freshness_warn_count 15 | ,freshness_warn_period 16 | ,freshness_error_count 17 | ,freshness_error_period 18 | ,snapshotted_at 19 | ,freshness_filter 20 | from {{ ref('raw_source_freshness') }} 21 | where payload_timestamp_utc = (select max(payload_timestamp_utc) from {{ ref('raw_source_freshness') }}) 22 | ) 23 | select 24 | lr.payload_id 25 | ,lr.payload_timestamp_utc 26 | ,lr.snapshotted_at 27 | ,lr.freshness_filter 28 | ,sfm.source_name 29 | ,sfm.source_description 30 | ,sfm.loader 31 | ,sfm.database as source_database 32 | ,sfm.schema as source_schema 33 | ,sfm.name as table_name 34 | ,sfm.description table_description 35 | ,( 36 | 'warn: ' || ifnull('after' || lr.freshness_warn_count::string || ' ' || lr.freshness_warn_period , 'undefined') || ', ' || 37 | 'error: ' || ifnull('after' || lr.freshness_error_count::string || ' ' || lr.freshness_error_period , 'undefined') 38 | ) as freshness_check 39 | from latest_records lr 40 | left join {{ ref('raw_source_freshness_manifest') }} sfm 41 | on lr.unique_id = sfm.unique_id 42 | -------------------------------------------------------------------------------- /models/sources/raw_source_freshness_manifest.sql: -------------------------------------------------------------------------------- 1 | {{ 2 | config(materialized='table') 3 | }} 4 | 5 | with dedup_logs as 6 | ( 7 | select s.* 8 | from {{ source('dbt_dataquality', 'stg_dbt_dataquality') }} s 9 | where s.upload_timestamp_utc = ( 10 | select max(upload_timestamp_utc) 11 | from {{ source('dbt_dataquality', 'stg_dbt_dataquality') }} 12 | where filename = 'manifest.json.gz' 13 | ) 14 | ), 15 | flatten_records as 16 | ( 17 | select 18 | payload_id, 19 | payload_timestamp_utc, 20 | sources.key::string unique_id, 21 | sources_content.key, 22 | sources_content.value 23 | from dedup_logs 24 | ,lateral flatten(input => payload:sources) as sources 25 | ,lateral flatten(input => sources.value ) as sources_content 26 | where sources_content.key in 27 | ('loaded_at_field', 'database', 'description', 'loader', 'source_name', 'source_description', 'package_name', 'schema', 'freshness', 'name') 28 | ), 29 | cleaning_records as 30 | ( 31 | select 32 | payload_id, 33 | payload_timestamp_utc, 34 | unique_id, 35 | "'name'"::string name, 36 | "'database'"::string database, 37 | "'description'"::string description, 38 | "'loader'"::string loader, 39 | "'source_name'"::string source_name, 40 | "'source_description'"::string source_description, 41 | "'package_name'"::string package_name, 42 | "'schema'"::string schema, 43 | "'loaded_at_field'"::string loaded_at_field 44 | from flatten_records 45 | pivot(max(value) for key in 46 | ('loaded_at_field', 'database', 'description', 'loader', 'source_name', 'source_description', 'package_name', 'schema', 'freshness', 'name')) 47 | ) 48 | select * from cleaning_records 49 | -------------------------------------------------------------------------------- /models/tests/raw_tests_manifest.sql: -------------------------------------------------------------------------------- 1 | {{ 2 | config(materialized='table') 3 | }} 4 | 5 | with dedup_logs as 6 | ( 7 | select s.* 8 | from {{ source('dbt_dataquality', 'stg_dbt_dataquality') }} s 9 | where s.upload_timestamp_utc = ( 10 | select max(upload_timestamp_utc) 11 | from {{ source('dbt_dataquality', 'stg_dbt_dataquality') }} 12 | where filename = 'manifest.json.gz' 13 | ) 14 | ), 15 | flatten_records as 16 | ( 17 | select 18 | payload_id 19 | ,payload_timestamp_utc 20 | ,tests.key::string unique_id 21 | ,tests_content.key 22 | ,case 23 | when tests_content.key = 'tags' 24 | then coalesce(replace(replace(replace(regexp_substr(replace(replace(tests_content.value,' '),'\'','"'), '(dq:)[^,]+(",)|(dq:)[^,]+("])'),'dq:'),'",'),'"]'),'unkonwn') 25 | when tests_content.key = 'column_name' 26 | then coalesce(tests_content.value, 'n/a') 27 | else tests_content.value 28 | end as value 29 | from dedup_logs 30 | ,lateral flatten(input => payload:nodes) as tests 31 | ,lateral flatten(input => tests.value ) as tests_content 32 | where 33 | tests_content.key in ('name', 'column_name', 'database', 'description', 'file_key_name', 'package_name', 'tags') 34 | and unique_id like 'test%' 35 | ) 36 | , 37 | cleaning_records as 38 | ( 39 | select 40 | payload_id 41 | ,payload_timestamp_utc 42 | ,unique_id 43 | ,"'name'"::string name 44 | ,"'package_name'"::string package_name 45 | ,"'database'"::string database 46 | ,"'description'"::string description 47 | ,"'column_name'"::string column_name 48 | ,"'file_key_name'"::string file_key_name 49 | ,"'tags'"::string tags 50 | from flatten_records 51 | pivot(max(value) for key in 52 | ('name', 'column_name', 'database', 'description', 'file_key_name', 'package_name', 'tags')) 53 | ) 54 | select * from cleaning_records 55 | -------------------------------------------------------------------------------- /models/tests/raw_tests.sql: -------------------------------------------------------------------------------- 1 | {{ 2 | config( 3 | materialized='incremental', 4 | unique_key='id' 5 | ) 6 | }} 7 | 8 | with dedup_logs as 9 | ( 10 | select s.* 11 | from {{ source('dbt_dataquality', 'stg_dbt_dataquality') }} s 12 | where s.upload_timestamp_utc = ( 13 | select max(upload_timestamp_utc) 14 | from {{ source('dbt_dataquality', 'stg_dbt_dataquality') }} 15 | where filename = 'run_results.json.gz' 16 | ) 17 | ), 18 | flatten_records as 19 | ( 20 | select 21 | {{ dbt_utils.surrogate_key(['payload_id', 'payload_timestamp_utc', 'results.value:unique_id']) }} as id, 22 | payload_id, 23 | payload_timestamp_utc, 24 | results.value:unique_id::string as unique_id, 25 | 26 | payload:metadata:dbt_schema_version::string as dbt_schema_version, 27 | payload:metadata:dbt_version::string as dbt_version, 28 | payload:metadata:generated_at::timestamp_tz as generated_at, 29 | payload:metadata:invocation_id::string as invocation_id, 30 | 31 | results.value:status::string as status, 32 | results.value:message::string as message, 33 | results.value:failures::string as failures, 34 | results.value:thread_id::string as thread_id, 35 | results.value:execution_time::float as execution_time, 36 | 37 | results.value:adapter_response:_message::string as adapter_response_message, 38 | results.value:adapter_response:code::string as adapter_response_code, 39 | results.value:adapter_response:rows_affected::string as adapter_response_rows_affected, 40 | 41 | results.value:timing[0]:started_at::timestamp_tz timing_compile_started_at, 42 | results.value:timing[0]:completed_at::timestamp_tz timing_compile_completed_at, 43 | results.value:timing[0]:started_at::timestamp_tz timing_execute_started_at, 44 | results.value:timing[0]:completed_at::timestamp_tz timing_execute_completed_at, 45 | 46 | payload:elapsed_time::float as elapsed_time 47 | from dedup_logs 48 | ,lateral flatten(input => payload:results) as results 49 | 50 | {% if is_incremental() %} 51 | where payload_timestamp_utc > (select max(payload_timestamp_utc) from {{ this }}) 52 | {% endif %} 53 | ) 54 | select * from flatten_records 55 | -------------------------------------------------------------------------------- /models/sources/raw_source_freshness.sql: -------------------------------------------------------------------------------- 1 | {{ 2 | config( 3 | materialized='incremental', 4 | unique_key='id' 5 | ) 6 | }} 7 | 8 | with dedup_logs as 9 | ( 10 | select s.* 11 | from {{ source('dbt_dataquality', 'stg_dbt_dataquality') }} s 12 | where s.upload_timestamp_utc = ( 13 | select max(upload_timestamp_utc) 14 | from {{ source('dbt_dataquality', 'stg_dbt_dataquality') }} 15 | where filename = 'sources.json.gz' 16 | ) 17 | ), 18 | flatten_records as 19 | ( 20 | select 21 | {{ dbt_utils.surrogate_key(['payload_id', 'payload_timestamp_utc', 'results.value:unique_id']) }} as id, 22 | payload_id, 23 | payload_timestamp_utc, 24 | results.value:unique_id::string as unique_id, 25 | payload:metadata:dbt_schema_version::string as dbt_schema_version, 26 | payload:metadata:dbt_version::string as dbt_version, 27 | payload:metadata:generated_at::timestamp_tz as generated_at, 28 | payload:metadata:invocation_id::string as invocation_id, 29 | results.value:max_loaded_at::timestamp_tz as max_loaded_at, 30 | results.value:snapshotted_at::timestamp_tz as snapshotted_at, 31 | results.value:max_loaded_at_time_ago_in_s::float as max_loaded_at_time_ago_in_s, 32 | results.value:status::string as status, 33 | results.value:criteria:warn_after:count::int as freshness_warn_count, 34 | results.value:criteria:warn_after:period::string as freshness_warn_period, 35 | results.value:criteria:error_after:count::int as freshness_error_count, 36 | results.value:criteria:error_after:period::string as freshness_error_period, 37 | results.value:criteria:filter::string as freshness_filter, 38 | results.value:thread_id::string as thread_id, 39 | results.value:execution_time::float as execution_time, 40 | results.value:timing[0]:started_at::timestamp_tz as compile_started_at, 41 | results.value:timing[0]:completed_at::timestamp_tz as compile_completed_at, 42 | results.value:timing[1]:started_at::timestamp_tz as execute_started_at, 43 | results.value:timing[1]:completed_at::timestamp_tz as execute_completed_at 44 | from dedup_logs 45 | ,lateral flatten(input => payload:results) as results 46 | 47 | {% if is_incremental() %} 48 | where payload_timestamp_utc > (select max(payload_timestamp_utc) from {{ this }}) 49 | {% endif %} 50 | ) 51 | select * from flatten_records 52 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dbt Data Quality 2 | 3 | This [dbt](https://github.com/dbt-labs/dbt-core) package helps you to 4 | 5 | - Access and report on the outputs from `dbt source freshness` ([sources.json](https://docs.getdbt.com/reference/artifacts/sources-json) and [manifest.json](https://docs.getdbt.com/reference/artifacts/manifest-json)) 6 | - Access and report on the outputs from `dbt test` ([run_results.json](https://docs.getdbt.com/reference/artifacts/run-results-json) and [manifest.json](https://docs.getdbt.com/reference/artifacts/manifest-json)) 7 | 8 | 9 | 10 | ## Prerequisites 11 | 12 | - This package is compatible with dbt 1.0.0 and later 13 | - This packages uses Snowflake as the backend for reporting (contributions to support other backend engines are welcomed) 14 | - Snowflake credentials with the right level of access to create/destroy and configure the following objects: 15 | - Database (optional) 16 | - Schema (optional) 17 | - Internal stage (recommended but optional). Alternatively, you can use an external stage 18 | - Table 19 | 20 | ## Contributions 21 | We love contributions! Currently, we don't have a roadmap for this package so feel free to help where you can 22 | 23 | Here's some ideas where we would love your contribution: 24 | 25 | - Adding support for other databases such as Microsoft SQL Server and PostgreSQL 26 | - Extending the downstream data models and incorporate more comprehensive data quality testing coverage and advanced metrics 27 | - Adding new models to capture logging data historically (ideally a customisable rolling window) 28 | - Contributing new dashboards from different tools such as Tableau 29 | 30 | If you have any questions, you can contact us at info@divergentinsights.com.au 31 | 32 | ## High Level Architecture 33 | 34 | ![High-Level Architecture](https://raw.githubusercontent.com/Divergent-Insights/dbt-dataquality/main/dashboards/dbt_dataquality-high_level_architecture.png) 35 | 36 | ## Architecture Overview 37 | 38 | As per the high-level architecture diagram, these are the different functionalities that this package provides: 39 | 40 | - (Optional) Creation of Snowflake resources to store and make available dbt logging information 41 | - Create a database (optional) - this is only provided for convenience and very unlikely to be required by you 42 | - Creates a schema (optional) 43 | - Creates an internal stage (optional) 44 | - Creates a variant-column staging table 45 | 46 | - Loads dbt logging information on an internal stage 47 | - This is achieved via a set of dbt macros together leveraging Snowflake PUT command 48 | 49 | - Copies dbt logging information into a Snowflake table 50 | - This is achieved via a set of dbt macros together leveraging Sowflake COPY command 51 | 52 | - Creating and populating simple dbt models to report on `dbt source freshness` and `dbt tests` 53 | - Raw logging data is modelled downstream and contextualised for reporting purposes 54 | 55 | - Bonus - it provides a ready-to-go Power BI dashboard built on top the dbt models created by the package to showcase all features 56 | 57 | --- 58 | 59 | ## Usage 60 | 61 | ### Package Configuration 62 | 63 | Optionally, set any relevant variables in your dbt_project.yml 64 | 65 | ``` 66 | vars: 67 | dbt_dataquality: 68 | dbt_dataquality_database: my_database # optional, default is target.database 69 | dbt_dataquality_schema: my_schema # optional, default is target.schema 70 | dbt_dataquality_table: my_table # optional, default is 'stg_dbt_dataquality' 71 | dbt_dataquality_stage: my_internal_stage | my_external_stage, default is 'dbt_dataquality'), 72 | dbt_dataquality_target_path: my_dbt_target_directory # optional, default is 'target' 73 | ``` 74 | **Important**: when using an external stage you need to set the parameter `load_from_internal_stage` to `False` on the load_log_* macros. See below for more details 75 | 76 | ### Resources Creation 77 | 78 | Use the macro `create_resources` to create the backend resources required by the package 79 | - If you have the right permissions, you should be able to run this macro to create all resources required by the dbt_dataquality package 80 | - For example, a successful run of `dbt run-operation create_resources` will give you the schema, table and staging tables required by the package 81 | 82 | If you are in a complex environment with stringent permissions, you can run the macro in "dry mode" which will give you the SQL required by the macro. Once you have the SQL you can copy and paste and run manually the parts of the query that make sense 83 | - For example, `dbt run-operation create_resources --args '{dry_run:True}'` 84 | 85 | Also, keep in mind that the "create_resources" macro creates an internal stage by default. If you are wanting to load log files via an external stage then you can disable the creation of the internal stage 86 | - For example, `dbt run-operation create_resources --args '{internal_stage:False}'` 87 | 88 | ### Generating some log files 89 | 90 | Optionally, do a regular run of dbt source freshness or dbt test on your local project to generate some logging files 91 | - For example ```dbt run``` or ```dbt test``` 92 | 93 | ### Loading log files - Internal Stage 94 | 95 | Use the load macros provided by the dbt_quality package to load the dbt logging information that's required 96 | - Use the macro `load_log_sources` to load sources.json and manifest.json files 97 | - Use the macro `load_log_tests` to load run_results.json and manifest.json files 98 | 99 | Note that the `load_log_sources` and `load_log_tests` macros automatically upload the relevant log and manifest files 100 | For example, the macro `load_log_sources` loads sources.json and manifest.json and the macro `load_log_tests` loads the files run_results.json and manifest.json 101 | 102 | ### Loading log files - External Stage 103 | To load data from an external stage, you must: 104 | - Workout on your own how to create, configure and load the data to the external stage 105 | - In this case, when running the `create_resources` macro set the parameter `internal_stage` to `False` 106 | - For example: `dbt run-operation create_resources --args '{internal_stage: False}'` 107 | - Set the package variable `dbt_dataquality_stage: my_external_stage` (as described at the beginning of the Usage section) 108 | - When running the `load_log_sources` and `load_log_tests` macros set the parameter `load_from_internal_stage` to `False` 109 | - For example: `dbt run-operation load_log_sources --args '{load_from_internal_stage: False}'` 110 | 111 | ### Create and populate downstream models 112 | 113 | - Use `dbt run --select dbt_quality.sources` to load source freshness logs 114 | - Use `dbt run --select dbt_quality.tests` to load tests logs 115 | 116 | ## Data Quality Attributes 117 | This package supports capturing and reporting on Data Quality Attributes. This is a very popular feature! 118 | 119 | To use this functionality just follow these simple steps: 120 | 121 | ### Add tests to your models 122 | Just add tests to your models following [the standard dbt testing process](https://docs.getdbt.com/docs/building-a-dbt-project/tests) 123 | Tip: you may want to use some tests from the awesome dbt package [dbt-expectations](https://github.com/calogica/dbt-expectations#expect_row_values_to_have_recent_data) 124 | 125 | ### Tag your tests 126 | Tag any tests that you want to report on with **your preferred data quality attributes** 127 | 128 | To keep things simple at Divergent Insights we use [the ISO/IEC 25012:2008 standard](https://www.iso.org/standard/35736.html) to report on data quality (refer to the image below) 129 | ![Data Product Quality](https://iso25000.com/images/figures/ISO_25012_en.png) 130 | 131 | You can read more about ISO 25012 [here](https://iso25000.com/index.php/en/iso-25000-standards/iso-25012); however, here's a summary of the key Data Quality Attributes defined by the standard: 132 | - **Accuracy**: the degree to which data has attributes that correctly represent the true value of the intended attribute of a concept or event in a specific context of use. 133 | - **Completeness**: the degree to which subject data associated with an entity has values for all expected attributes and related entity instances in a specific context of use. 134 | - **Consistency**: the degree to which data has attributes that are free from contradiction and are coherent with other data in a specific context of use. It can be either or both among data regarding one entity and across similar data for comparable entities. 135 | - **Credibility**: the degree to which data has attributes that are regarded as true and believable by users in a specific context of use. Credibility includes the concept of authenticity (the truthfulness of origins, attributions, commitments). 136 | - **Currentness / Timeliness**: the degree to which data has attributes that are of the right age in a specific context of use. 137 | 138 | Please note that 139 | - Tags **MUST** be prefixed with "dq:", for example `dq:accuracy` or `dq:timeliness` 140 | - Any tag prefixed with "dq:" will be automatically detected and reported on by the package 141 | - In our case, we use four tags aligned to ISO 25012: `dq:accuracy`, `dq:completeness`, `dq:consistency` and `dq:timeliness` (we don't use credibility due to obvious reasons) 142 | - If you add two or more "dq:" tags, only the first tag sorted alphabetically is processed 143 | 144 | 145 | ### Usage Summary 146 | Here's all the steps put together: 147 | ``` 148 | dbt run-operation create_resources 149 | 150 | dbt source freshness 151 | dbt run-operation load_log_sources 152 | dbt run --select dbt_dataquality.sources 153 | 154 | dbt test 155 | dbt run-operation load_log_tests 156 | dbt run --select dbt_dataquality.tests 157 | 158 | # Optionally, the dbt_dataquality package uses incremental models so don't forget to use the option `--full-refresh` to rebuild them 159 | # For example 160 | dbt run --full-refresh --select dbt_dataquality.sources 161 | dbt run --full-refresh --select dbt_dataquality.tests 162 | ``` 163 | 164 | ## Dashboarding Data Quality Information 165 | - The models created will allow you to dome some simple but powerful reporting on your data quality (see images below) 166 | - This package includes a nice and simple Power BI sample dashboard to get you going! 167 | 168 | ### Sources Overview Dashboard 169 | ![Sample Dashboard](https://raw.githubusercontent.com/Divergent-Insights/dbt-dataquality/main/dashboards/dashboard1.png) 170 | 171 | ### Tests Overview Dashboard 172 | ![Sample Dashboard](https://raw.githubusercontent.com/Divergent-Insights/dbt-dataquality/main/dashboards/dashboard2.png) 173 | 174 | ### Data Quality Attributes 175 | ![Sample Dashboard](https://raw.githubusercontent.com/Divergent-Insights/dbt-dataquality/main/dashboards/dashboard3.png) 176 | 177 | --- 178 | 179 | ## TODO 180 | - Adding testing suite 181 | - Adding more complex downstream metrics on Data Quality Coverage 182 | - When the time is right, adding support for old and new [dbt artifacts schema versions](https://docs.getdbt.com/reference/artifacts/dbt-artifacts), currently only v3 is supported 183 | 184 | ## License 185 | All the content of this repository is licensed under the [**Apache License 2.0**](https://github.com/Divergent-Insights/dbt-dataquality/blob/main/LICENSE) 186 | 187 | This is a permissive license whose main conditions require preservation of copyright and license notices. Contributors provide an express grant of patent rights. Licensed works, modifications, and larger works may be distributed under different terms and without source code. 188 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2022 Divergent-Insights 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------