├── dbsqlclone ├── utils │ ├── __init__.py │ ├── client.py │ ├── dump_dashboard.py │ ├── clone_dashboard.py │ └── load_dashboard.py ├── __init__.py └── clone_resources.py ├── .DS_Store ├── .gitignore ├── config_export.json ├── config_import.json ├── setup.py ├── conf_example.json ├── export_dashboard_to_file.py ├── import_dashboard_from_file.py ├── test ├── test.py ├── 19394330-2274-4b4b-90ce-d415a7ff2130.json └── 6f73dd1b-17b1-49d0-9a11-b3772a2c3357.json ├── LICENCE └── README.md /dbsqlclone/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dbsqlclone/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/QuentinAmbard/databricks-sql-clone/HEAD/.DS_Store -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dashboards 2 | dbsqlclone/__pycache__/ 3 | dbsqlclone/utils/__pycache__/ 4 | dbsqlclone.egg-info/ 5 | build/ 6 | dist/ 7 | state.json 8 | .idea 9 | utils/__pycache__ 10 | secret.json 11 | config.json 12 | *.iml 13 | -------------------------------------------------------------------------------- /config_export.json: -------------------------------------------------------------------------------- 1 | { 2 | "source": { 3 | "url": "https://yourdbe.cloud.databricks.com", 4 | "token": "dapixxxxxxxxxxxxxxxxxx", 5 | "dashboard_tags": ["xxxxx"] 6 | }, 7 | "dashboard_id":"xxxx-xxxxx-xxxxx", 8 | "dashboard_folder":"./dashboards/" 9 | } 10 | -------------------------------------------------------------------------------- /config_import.json: -------------------------------------------------------------------------------- 1 | { 2 | "targets": [ 3 | { 4 | "url": "https://yourtargetdb.cloud.databricks.com", 5 | "endpoint_id":"xxxxxxxxx", 6 | "permissions":[ 7 | { 8 | "group_name": "users", 9 | "permission_level": "CAN_RUN" 10 | }, 11 | { 12 | "group_name": "admins", 13 | "permission_level": "CAN_MANAGE" 14 | } 15 | ] 16 | } 17 | ], 18 | "dashboard_id":"317f4809-8d9d-4956-a79a-6eee51412217", 19 | "dashboard_folder":"./dashboards/" 20 | 21 | } 22 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | #python setup.py clean --all bdist_wheel 4 | setup( 5 | #this will be the package name you will see, e.g. the output of 'conda list' in anaconda prompt 6 | name = 'dbsqlclone', 7 | #some version number you may wish to add - increment this after every update 8 | version='0.1.19', 9 | packages=find_packages(exclude=["tests", "tests.*"]), 10 | setup_requires=["wheel"], 11 | include_package_data=True, 12 | install_requires=["requests"], 13 | license_files = ('LICENSE',) 14 | ) -------------------------------------------------------------------------------- /dbsqlclone/utils/client.py: -------------------------------------------------------------------------------- 1 | class Client(): 2 | def __init__(self, url, token, permissions = [], endpoint_id = None, dashboard_tags = None): 3 | self.url = url 4 | self.headers = {"Authorization": "Bearer " + token, 'Content-type': 'application/json'} 5 | self.permissions = None 6 | if permissions is not None: 7 | self.permissions = {"access_control_list": permissions} 8 | self.data_source_id = None 9 | self.endpoint_id = endpoint_id 10 | self.dashboard_tags = dashboard_tags 11 | 12 | def permisions_defined(self): 13 | return self.permissions is not None and len(self.permissions["access_control_list"]) > 0 -------------------------------------------------------------------------------- /conf_example.json: -------------------------------------------------------------------------------- 1 | { 2 | "source": { 3 | "url": "https://xxxx.databricks.com", 4 | "token": "xxxxx", 5 | "dashboard_tags": ["qoe"] 6 | }, 7 | "delete_target_dashboards": true, 8 | "targets": [ 9 | { 10 | "url": "https://xxxx.cloud.databricks.com", 11 | "token": "xxxxxx", 12 | "endpoint_id": "xxxxxxxxxx4da979", 13 | "permissions":[ 14 | { 15 | "group_name": "users", 16 | "permission_level": "CAN_RUN" 17 | }, 18 | { 19 | "user_name": "xxx@xxx.com", 20 | "permission_level": "CAN_MANAGE" 21 | } 22 | ] 23 | }, 24 | { 25 | "url": "https://xxxxxxxxxxxx.xx.azuredatabricks.net", 26 | "token": "xxxxxxxx", 27 | "endpoint_id": "xxxxxxxxxx4da979", 28 | "permissions": null 29 | } 30 | ] 31 | } -------------------------------------------------------------------------------- /export_dashboard_to_file.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from utils import clone_dashboard 3 | from utils.client import Client 4 | from utils.dump_dashboard import dump_dashboard 5 | import json 6 | 7 | 8 | def get_client(config_file): 9 | with open(config_file, "r") as r: 10 | config = json.loads(r.read()) 11 | source = Client(config["source"]["url"], config["source"]["token"], 12 | dashboard_tags=config["source"]["dashboard_tags"]) 13 | return source, config["dashboard_id"], config["dashboard_folder"] 14 | 15 | 16 | parser = argparse.ArgumentParser() 17 | parser.add_argument("--config_file", default="config_export.json", required=False, 18 | help="configuration file containing credential and dashboard to clone") 19 | args = parser.parse_args() 20 | source_client,dashboard_id_to_save,dashboard_folder_to_save = get_client(args.config_file) 21 | dump_dashboard(source_client,dashboard_id_to_save,dashboard_folder_to_save) 22 | -------------------------------------------------------------------------------- /import_dashboard_from_file.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from utils import clone_dashboard 3 | from utils.load_dashboard import load_dashboard 4 | from utils.client import Client 5 | import json 6 | 7 | 8 | def get_client(config_file,pat_token): 9 | with open(config_file, "r") as r: 10 | config = json.loads(r.read()) 11 | targets = [] 12 | for target in config["targets"]: 13 | print("Token ->("+pat_token+")") 14 | client = Client(target["url"], pat_token, permissions=target["permissions"]) 15 | if "endpoint_id" in target: 16 | client.endpoint_id = target["endpoint_id"] 17 | if "sql_database_name" in target: 18 | client.sql_database_name = target["sql_database_name"] 19 | targets.append(client) 20 | return targets,config["dashboard_id"], config["dashboard_folder"] 21 | 22 | 23 | parser = argparse.ArgumentParser() 24 | parser.add_argument("--config_file", default="config_import.json", required=False, 25 | help="Configuration file containing credential and dashboard to clone") 26 | parser.add_argument("--pat_token", required=True, 27 | help="Personal Access Token to for your workspace") 28 | args = parser.parse_args() 29 | 30 | target_clients, dashboard_id_to_load,dashboard_folder = get_client(args.config_file, args.pat_token) 31 | workspace_state = {} 32 | load_dashboard(target_clients[0], dashboard_id_to_load, workspace_state, dashboard_folder) 33 | 34 | -------------------------------------------------------------------------------- /dbsqlclone/clone_resources.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from .utils import clone_dashboard 3 | from .utils.client import Client 4 | import json 5 | 6 | 7 | def get_client(config_file): 8 | with open(config_file, "r") as r: 9 | config = json.loads(r.read()) 10 | source = Client(config["source"]["url"], config["source"]["token"], 11 | dashboard_tags=config["source"]["dashboard_tags"]) 12 | targets = [] 13 | for target in config["targets"]: 14 | client = Client(target["url"], target["token"], permissions=target["permissions"]) 15 | if "endpoint_id" in target: 16 | client.endpoint_id = target["endpoint_id"] 17 | targets.append(client) 18 | return source, targets, config["delete_target_dashboards"] 19 | 20 | 21 | parser = argparse.ArgumentParser() 22 | parser.add_argument("--config_file", default="config.json", required=False, 23 | help="configuration file containing credential and dashboard to clone") 24 | parser.add_argument("--state_file", default="state.json", required=False, 25 | help="state containing the links between the already cloned dashboard. Used to update resources") 26 | args = parser.parse_args() 27 | 28 | source_client, target_clients, delete_target_dashboards = get_client(args.config_file) 29 | 30 | try: 31 | with open("state.json", "r") as r: 32 | state = json.loads(r.read()) 33 | except: 34 | print("state isn't available, create an empty one") 35 | state = {} 36 | 37 | clone_dashboard.delete_queries(target_clients[0], "") 38 | 39 | for target_client in target_clients: 40 | clone_dashboard.set_data_source_id_from_endpoint_id(target_client) 41 | clone_dashboard.delete_and_clone_dashboards_with_tags(source_client, target_client, source_client.dashboard_tags, 42 | delete_target_dashboards, state) 43 | -------------------------------------------------------------------------------- /test/test.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from dbsqlclone.utils import load_dashboard 3 | from dbsqlclone.utils import dump_dashboard 4 | from dbsqlclone.utils import clone_dashboard 5 | from dbsqlclone.utils.client import Client 6 | import json 7 | 8 | 9 | def get_client(config_file): 10 | with open(config_file, "r") as r: 11 | config = json.loads(r.read()) 12 | source = Client(config["source"]["url"], config["source"]["token"], 13 | dashboard_tags=config["source"]["dashboard_tags"]) 14 | targets = [] 15 | for target in config["targets"]: 16 | client = Client(target["url"], target["token"], permissions=target["permissions"]) 17 | if "endpoint_id" in target: 18 | client.endpoint_id = target["endpoint_id"] 19 | targets.append(client) 20 | return source, targets, config["delete_target_dashboards"] 21 | 22 | 23 | parser = argparse.ArgumentParser() 24 | parser.add_argument("--config_file", default="config.json", required=False, 25 | help="configuration file containing credential and dashboard to clone") 26 | parser.add_argument("--state_file", default="state.json", required=False, 27 | help="state containing the links between the already cloned dashboard. Used to update resources") 28 | args = parser.parse_args() 29 | 30 | source_client, target_clients, delete_target_dashboards = get_client(args.config_file) 31 | 32 | #dashboard_to_clone = "98c5d5df-1f6c-4c63-84d4-760283c15846" 33 | 34 | target_client = target_clients[0] 35 | 36 | import logging 37 | logging.basicConfig() 38 | load_dashboard.logger.setLevel(logging.DEBUG) 39 | 40 | #dashboard_def = dump_dashboard.get_dashboard_definition_by_id(source_client, dashboard_to_clone) 41 | #print(dashboard_def) 42 | #To recreate a new dashboard 43 | dashboard_to_clone = "19394330-2274-4b4b-90ce-d415a7ff2130" 44 | with open(f"test/{dashboard_to_clone}.json", "r") as r: 45 | dashboard_def = json.loads(r.read()) 46 | 47 | #target_client.data_source_id = "aa143a10-aad0-41a3-a7bd-9158962b4d2c" 48 | from dbsqlclone.utils import load_dashboard 49 | load_dashboard.max_workers = 1 50 | target_client.endpoint_id = "2076de1d9dc195fd" 51 | clone_dashboard.set_data_source_id_from_endpoint_id(target_client) 52 | print(target_client.data_source_id) 53 | existing_id = "9fc6a3bb-ff36-4e06-b5f9-912d7e77dc05" 54 | #state = load_dashboard.clone_dashboard_without_saved_state(dashboard_def, target_client, existing_id) 55 | state = load_dashboard.clone_dashboard(dashboard_def, target_client) 56 | print(state) 57 | assert state['new_id'] == existing_id 58 | #load_dashboard.clone_dashboard(dashboard_def, target_client, {}, None) 59 | 60 | 61 | -------------------------------------------------------------------------------- /dbsqlclone/utils/dump_dashboard.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from .client import Client 3 | import json 4 | from concurrent.futures import ThreadPoolExecutor 5 | import collections 6 | import os 7 | import logging 8 | 9 | logger = logging.getLogger('dbsqlclone.dump') 10 | 11 | 12 | def dump_dashboards(source_client: Client, dashboard_ids): 13 | params = [(source_client, id) for id in dashboard_ids] 14 | with ThreadPoolExecutor(max_workers=10) as executor: 15 | collections.deque(executor.map(lambda args, f=dump_dashboard: f(*args), params)) 16 | 17 | def dump_dashboard(source_client: Client, dashboard_id, folder_prefix="./dashboards/"): 18 | dashboard = get_dashboard_definition_by_id(source_client, dashboard_id) 19 | if not folder_prefix.endswith("/"): 20 | folder_prefix += "/" 21 | if not os.path.exists(folder_prefix): 22 | os.makedirs(folder_prefix) 23 | with open(f'{folder_prefix}dashboard-{dashboard_id}.json', 'w') as file: 24 | file.write(json.dumps(dashboard, indent=4, sort_keys=True)) 25 | 26 | def get_dashboard_definition_by_id(source_client: Client, dashboard_id): 27 | logger.debug(f"getting dashboard definition from {dashboard_id}...") 28 | result = {"queries": [], "id": dashboard_id} 29 | dashboard = requests.get(source_client.url+"/api/2.0/preview/sql/dashboards/"+dashboard_id, headers = source_client.headers).json() 30 | result["dashboard"] = dashboard 31 | query_ids = list() 32 | param_query_ids = set() 33 | 34 | def recursively_append_param_queries(q): 35 | for p in q["options"]["parameters"]: 36 | if "queryId" in p: 37 | query_ids.insert(0, p["queryId"]) 38 | param_query_ids.add(p["queryId"]) 39 | #get the details of the underlying query to recursively append children queries from parameters if any 40 | child_q = requests.get(source_client.url + "/api/2.0/preview/sql/queries/" + p["queryId"], headers=source_client.headers).json() 41 | recursively_append_param_queries(child_q) 42 | #fetch all the queries required for the widgets, recursively 43 | for widget in dashboard["widgets"]: 44 | if "visualization" in widget: 45 | #First we need to add the queries from the parameters to make sure we clone them too 46 | if "options" in widget["visualization"]["query"] and \ 47 | "parameters" in widget["visualization"]["query"]["options"]: 48 | recursively_append_param_queries(widget["visualization"]["query"]) 49 | query_ids.append(widget["visualization"]["query"]["id"]) 50 | 51 | #removes duplicated but keep order (we need to start with the param queries first) 52 | query_ids = list(dict.fromkeys(query_ids)) 53 | for query_id in query_ids: 54 | q = requests.get(source_client.url + "/api/2.0/preview/sql/queries/" + query_id, headers=source_client.headers).json() 55 | q["is_parameter_query"] = query_id in param_query_ids 56 | result["queries"].append(q) 57 | return result -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | Databricks SQL Clone 2 | 3 | Copyright (2019) Databricks, Inc. 4 | 5 | This library (the "Software") may not be used except in connection with the Licensee's use of the Databricks Platform Services pursuant 6 | to an Agreement (defined below) between Licensee (defined below) and Databricks, Inc. ("Databricks"). The Object Code version of the 7 | Software shall be deemed part of the Downloadable Services under the Agreement, or if the Agreement does not define Downloadable Services, 8 | Subscription Services, or if neither are defined then the term in such Agreement that refers to the applicable Databricks Platform 9 | Services (as defined below) shall be substituted herein for “Downloadable Services.” Licensee's use of the Software must comply at 10 | all times with any restrictions applicable to the Downlodable Services and Subscription Services, generally, and must be used in 11 | accordance with any applicable documentation. For the avoidance of doubt, the Software constitutes Databricks Confidential Information 12 | under the Agreement. 13 | 14 | Additionally, and notwithstanding anything in the Agreement to the contrary: 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 16 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 17 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 18 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | * you may view, make limited copies of, and may compile the Source Code version of the Software into an Object Code version of the 20 | Software. For the avoidance of doubt, you may not make derivative works of Software (or make any any changes to the Source Code 21 | version of the unless you have agreed to separate terms with Databricks permitting such modifications (e.g., a contribution license 22 | agreement)). 23 | 24 | If you have not agreed to an Agreement or otherwise do not agree to these terms, you may not use the Software or view, copy or compile 25 | the Source Code of the Software. 26 | 27 | This license terminates automatically upon the termination of the Agreement or Licensee's breach of these terms. Additionally, 28 | Databricks may terminate this license at any time on notice. Upon termination, you must permanently delete the Software and all 29 | copies thereof (including the Source Code). 30 | 31 | Agreement: the agreement between Databricks and Licensee governing the use of the Databricks Platform Services, which shall be, with 32 | respect to Databricks, the Databricks Terms of Service located at www.databricks.com/termsofservice, and with respect to Databricks 33 | Community Edition, the Community Edition Terms of Service located at www.databricks.com/ce-termsofuse, in each case unless Licensee 34 | has entered into a separate written agreement with Databricks governing the use of the applicable Databricks Platform Services. 35 | 36 | Databricks Platform Services: the Databricks services or the Databricks Community Edition services, according to where the Software is used. 37 | 38 | Licensee: the user of the Software, or, if the Software is being used on behalf of a company, the company. 39 | 40 | Object Code: is version of the Software produced when an interpreter or a compiler translates the Source Code into recognizable and 41 | executable machine code. 42 | 43 | Source Code: the human readable portion of the Software. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Databricks dashboard clone 2 | Unofficial project to allow Databricks SQL dashboard copy from one workspace to another. 3 | 4 | ## Resource clone 5 | 6 | ### Install: 7 | 8 | ``` 9 | pip install dbsqlclone 10 | ``` 11 | 12 | ### Setup for built-in clone: 13 | Create a file named `config.json` and put your credentials. You need to define the source (where the resources will be copied from) and a list of targets (where the resources will be cloned). 14 | 15 | ```json 16 | { 17 | "source": { 18 | "url": "https://xxxxx.cloud.databricks.com", 19 | "token": "xxxxxxx", /* your PAT token*/ 20 | "dashboard_tags": ["field_demos"] /* Dashboards having any of these tags matching will be cloned from the SOURCE */ 21 | }, 22 | "delete_target_dashboards": true, /* Erase the dashboards and queries in the targets having the same tags in TARGETS. If false, won't do anything (might endup with duplicates). */ 23 | "targets": [ 24 | { 25 | "url": "https:/xxxxxxx.cloud.databricks.com", 26 | "token": "xxxxxxx", 27 | "endpoint_id": "xxxxxxxxxx4da979", /* Optional, will use the first endpoint available if not set. At least 1 endpoint must exist in the workspace.*/ 28 | "permissions":[ /* Optional, force the permissions to this set of values. In this example we add a CAN_RUN for All Users.*/ 29 | { 30 | "user_name": "xxx@xxx.com", 31 | "permission_level": "CAN_MANAGE" 32 | }, 33 | { 34 | "group_name": "users", 35 | "permission_level": "CAN_RUN" 36 | } 37 | ] 38 | }, 39 | { 40 | "url": "https://xxxxxxx.azuredatabricks.net", 41 | "token": "xxxxxxx" 42 | } 43 | ] 44 | } 45 | ``` 46 | 47 | `endpoint_id` the ID of the endpoint we'll attach to the queries. 48 | 49 | To find your `endpoint_id` on each target workspace, click in one of your endpoint. 50 | The endpoint ID is in the URL: `https://xxxx.azuredatabricks.net/sql/endpoints/?o=xxxx` 51 | 52 | ### Run: 53 | Run the `clone_resources.py` script to clone all the resources 54 | 55 | ## Dashboard update 56 | If a state file (`json.state`) exists and the dashboards+queries have already be cloned, the clone operation will try to update the existing dashboards and queries. 57 | 58 | Visualizations and widgets are systematically destroyed and re-created to simplify synchronization. 59 | 60 | If your state is out of sync, delete the entry matching your target to re-delete all content in the target and re-clone from scratch. 61 | 62 | You can delete the state of a single workspace by searching the entry in the json state information. 63 | ### State file structure 64 | ``` 65 | { 66 | "SOURCE_URL-TARGET_URL": { 67 | "SOURCE_DASHBOARD_ID": { 68 | "queries": { 69 | "SOURCE_QUERY_ID": { 70 | "new_id": "TARGET_QUERY_ID", 71 | "visualizations": { 72 | "SOURCE_VISUALIZATION_ID": "TARGET_VISUALIZATION_ID",... 73 | } 74 | },... 75 | }, 76 | "new_id": "TARGET_DASHBOARD_ID" 77 | } 78 | } 79 | } 80 | ``` 81 | 82 | ## Custom usage / Working with git 83 | 84 | ### Direct api usage 85 | 86 | Get dashboard definition as json: 87 | ``` 88 | from dbsqlclone.utils.client import Client 89 | from dbsqlclone.utils import dump_dashboard 90 | 91 | source_client = Client("", "") 92 | dashboard_id_to_save = "xxx-xxx-xxx-xxx" 93 | dashboard_def = dump_dashboard.get_dashboard_definition_by_id(source_client, dashboard_id_to_save) 94 | print(json.dumps(dashboard_def)) 95 | ``` 96 | 97 | Create the dashboard from the definition. This will just create a new one. 98 | ``` 99 | target_client.endpoint_id = "the endpoint=warehouse ID" 100 | #We need to find the datasource from endpoint id 101 | clone_dashboard.set_data_source_id_from_endpoint_id(target_client) 102 | from dbsqlclone.utils import load_dashboard 103 | load_dashboard.clone_dashboard(dashboard_def, target_client, state={}, path=None) 104 | ``` 105 | 106 | Override an existing dashboard. This will try to update the dashboard queries when the query name match, and delete all queries not matching. 107 | ``` 108 | target_client.data_source_id = "the datasource or warehouse ID to use" 109 | dashboard_to_override = "xxx-xxx-xxx-xxx" 110 | load_dashboard.clone_dashboard_without_saved_state(dashboard_def, target_client, dashboard_to_override) 111 | ``` 112 | 113 | 114 | ### Saving dashboards as json with state file created in current folder: 115 | ``` 116 | source_client = Client("", "") 117 | dashboard_id_to_save = "xxx-xxx-xxx-xxx" 118 | dump_dashboard.dump_dashboard(source_client, dashboard_id_to_save, "./dashboards/") 119 | ``` 120 | 121 | 122 | Dashboard jsons definition will then be saved under the specified folder `./dashboards/` and can be saved in git as required. 123 | 124 | 125 | ### Loading dashboards from json with state file created in current folder: 126 | Once the json is saved, you can load it to build or update the dashboard in any workspace. 127 | 128 | `dashboard_state` contains the link between the source and the cloned dashboard. 129 | It is used to update the dashboard clone and avoid having to delete/re-recreate the cloned dashboard everytime. 130 | 131 | Loading the dashboard will update the sate definition. If you don't care about it you can pass an empty dict `{}` 132 | 133 | ``` 134 | target_client = Client("", "") 135 | workspace_state = {} 136 | dashboard_id_to_load = "xxx-xxx-xxx-xxx" 137 | load_dashboard.load_dashboard(target_client, dashboard_id_to_load, workspace_state, "./dashboards/") 138 | ``` -------------------------------------------------------------------------------- /dbsqlclone/utils/clone_dashboard.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | import requests 4 | import json 5 | 6 | from dbsqlclone.utils import load_dashboard 7 | from dbsqlclone.utils.client import Client 8 | from concurrent.futures import ThreadPoolExecutor 9 | import collections 10 | from dbsqlclone.utils.dump_dashboard import dump_dashboards 11 | import logging 12 | 13 | logger = logging.getLogger('dbsqlclone.clone') 14 | 15 | def get_all_dashboards(client: Client, tags = []): 16 | return get_all_item(client, "dashboards", tags) 17 | 18 | def get_all_queries(client: Client, tags = []): 19 | return get_all_item(client, "queries", tags) 20 | 21 | def delete_dashboard(client: Client, tags=[], ids_to_skip={}): 22 | logger.debug(f"cleaning up dashboards with tags in {tags}...") 23 | for d in get_all_dashboards(client, tags): 24 | if d['id'] not in ids_to_skip: 25 | logger.debug(f"deleting dashboard {d['id']} - {d['name']}") 26 | with requests.delete(client.url+"/api/2.0/preview/sql/dashboards/"+d["id"], headers = client.headers, timeout=120) as r: 27 | r.json() 28 | 29 | def delete_queries(client: Client, tags=[], ids_to_skip={}): 30 | logger.debug(f"cleaning up queries with tags in {tags}...") 31 | queries_to_delete = get_all_queries(client, tags) 32 | params = [(client, q) for q in queries_to_delete if q['id'] not in ids_to_skip] 33 | with ThreadPoolExecutor(max_workers=load_dashboard.max_workers) as executor: 34 | collections.deque(executor.map(lambda args, f=delete_query: f(*args), params)) 35 | 36 | def delete_query(client: Client, q): 37 | logger.debug(f"deleting query {q['id']} - {q['name']}") 38 | with requests.delete(client.url+"/api/2.0/preview/sql/queries/"+q["id"], headers = client.headers, timeout=120) as r: 39 | return r.json() 40 | 41 | def get_all_item(client: Client, item, tags = []): 42 | assert item == "queries" or item == "dashboards" 43 | page_size = 250 44 | def get_all_dashboards(dashboards, page): 45 | with requests.get(client.url+"/api/2.0/preview/sql/"+item, headers = client.headers, params={"page_size": page_size, "page": page}, timeout=120) as r: 46 | r = r.json() 47 | #Filter to keep only dashboard with the proper tags 48 | dashboards_tags = [d for d in r["results"] if len(set(d["tags"]) & set(tags)) > 0] 49 | dashboards.extend(dashboards_tags) 50 | if len(r["results"]) >= page_size: 51 | dashboards = get_all_dashboards(dashboards, page+1) 52 | return dashboards 53 | return get_all_dashboards([], 1) 54 | 55 | def delete_and_clone_dashboards_with_tags(source_client: Client, target_client: Client, tags: List, 56 | delete_target_dashboards: bool, state): 57 | assert len(tags) > 0 58 | logger.debug(f"fetching existing dashboard with tags in {tags}...") 59 | workspace_state_id = source_client.url+"-"+target_client.url 60 | 61 | dashboards_to_clone = get_all_dashboards(source_client, tags) 62 | if workspace_state_id not in state: 63 | state[workspace_state_id] = {} 64 | workspace_state = state[workspace_state_id] 65 | 66 | logger.debug(f"start cloning {len(dashboards_to_clone)} dashboards...") 67 | dashboard_to_clone_ids = [d["id"] for d in dashboards_to_clone] 68 | 69 | dump_dashboards(source_client, dashboard_to_clone_ids) 70 | state[workspace_state_id] = load_dashboards(target_client, dashboard_to_clone_ids, workspace_state) 71 | 72 | # Cleanup all existing resources, but skip the queries used in the new dashboard (to support update) 73 | if delete_target_dashboards: 74 | new_queries = set() 75 | new_dashboards = set() 76 | for origin_dashboard_id in state[workspace_state_id]: 77 | new_dashboards.add(state[workspace_state_id][origin_dashboard_id]["new_id"]) 78 | for origin_query_id in state[workspace_state_id][origin_dashboard_id]["queries"]: 79 | new_queries.add(state[workspace_state_id][origin_dashboard_id]["queries"][origin_query_id]["new_id"]) 80 | delete_queries(target_client, tags, new_queries) 81 | delete_dashboard(target_client, tags, new_dashboards) 82 | 83 | logger.debug("-----------------------") 84 | logger.debug("import complete. Saving state for further update/analysis.") 85 | logger.debug(state) 86 | with open('state.json', 'w') as file: 87 | file.write(json.dumps(state, indent=4, sort_keys=True)) 88 | 89 | 90 | def set_data_source_id_from_endpoint_id(client): 91 | logger.debug("Fetching endpoints to extract data_source id...") 92 | with requests.get(client.url+"/api/2.0/preview/sql/data_sources", headers=client.headers, timeout=120) as r: 93 | data_sources = r.json() 94 | assert len(data_sources) > 0, "No endpoints available. Please create at least 1 endpoint before cloning the dashboards." 95 | if client.endpoint_id is None: 96 | logger.debug(f"No endpoint id found. Using the first endpoint available: {data_sources[0]}") 97 | client.data_source_id = data_sources[0]['id'] 98 | for data_source in data_sources: 99 | if "endpoint_id" in data_source and data_source['endpoint_id'] == client.endpoint_id: 100 | logger.debug(f"found datasource {data_source['id']} for endpoint {data_source['endpoint_id']}") 101 | client.data_source_id = data_source['id'] 102 | break 103 | assert client.data_source_id is not None, f"Couldn't find an endpoint with ID {client.endpoint_id} in workspace {client.url}. Please use the endpoint ID from the URL." 104 | 105 | 106 | 107 | 108 | -------------------------------------------------------------------------------- /dbsqlclone/utils/load_dashboard.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import requests 4 | from dbsqlclone.utils.client import Client 5 | from concurrent.futures import ThreadPoolExecutor 6 | import json 7 | import collections 8 | import logging 9 | 10 | from .dump_dashboard import get_dashboard_definition_by_id 11 | from .clone_dashboard import delete_query 12 | 13 | logger = logging.getLogger('dbsqlclone.load') 14 | 15 | max_workers = 3 16 | 17 | def load_dashboards(target_client: Client, dashboard_ids, workspace_state): 18 | if workspace_state is None: 19 | workspace_state = {} 20 | params = [(target_client, dashboard_id, workspace_state[dashboard_id] if dashboard_id in workspace_state else {}) for dashboard_id in dashboard_ids] 21 | 22 | with ThreadPoolExecutor(max_workers=max_workers) as executor: 23 | for (dashboard_id, dashboard_state) in executor.map(lambda args, f=load_dashboard: f(*args), params): 24 | workspace_state[dashboard_id] = dashboard_state 25 | return workspace_state 26 | 27 | def load_dashboard(target_client: Client, dashboard_id, dashboard_state, folder_prefix="./dashboards/"): 28 | if not folder_prefix.endswith("/"): 29 | folder_prefix += "/" 30 | with open(f'{folder_prefix}dashboard-{dashboard_id}.json', 'r') as r: 31 | dashboard = json.loads(r.read()) 32 | dashboard_state = clone_dashboard(dashboard, target_client, dashboard_state) 33 | return dashboard_id, dashboard_state 34 | 35 | #Try to match the existing query based on the name. This is to avoid having to delete/recreate the queries everytime 36 | def recreate_dashboard_state(target_client, dashboard, dashboard_id): 37 | #Get the definition of the existing dashboard 38 | existing_dashboard = get_dashboard_definition_by_id(target_client, dashboard_id) 39 | state = {"queries": {}, "visualizations": {}, "new_id": existing_dashboard["dashboard"]["id"]} 40 | queries_not_matching = [] 41 | for q in existing_dashboard["queries"]: 42 | matching_query = next((existing_q for existing_q in dashboard["queries"] if q['name'] == existing_q['name']), None) 43 | if matching_query is None: 44 | queries_not_matching.append(q) 45 | else: 46 | state["queries"][matching_query['id']] = {"new_id": q['id']} 47 | return state, queries_not_matching 48 | 49 | 50 | #Override the existing_dashboard_id queries by trying to match them by name. If the name change, will create a new query and delete the existing one. 51 | def clone_dashboard_without_saved_state(dashboard, target_client: Client, existing_dashboard_id, parent: str = None): 52 | dashboard_state, queries_not_matching = recreate_dashboard_state(target_client, dashboard, existing_dashboard_id) 53 | logger.debug(dashboard_state) 54 | state = clone_dashboard(dashboard, target_client, dashboard_state, parent) 55 | for q in queries_not_matching: 56 | logger.debug(f"deleting query {q}") 57 | delete_query(target_client, q) 58 | return state 59 | 60 | def clone_dashboard(dashboard, target_client: Client, dashboard_state: dict = None, parent: str = None): 61 | if dashboard_state is None: 62 | dashboard_state = {} 63 | if "queries" not in dashboard_state: 64 | dashboard_state["queries"] = {} 65 | 66 | def load_query(q): 67 | #We need to replace the param queries with the newly created one 68 | if "parameters" in q["options"]: 69 | for p in q["options"]["parameters"]: 70 | if "queryId" in p: 71 | p["queryId"] = dashboard_state["queries"][p["queryId"]]["new_id"] 72 | if "parentQueryId" in p: 73 | del p["parentQueryId"] 74 | #if "value" in p: 75 | # del p["value"] 76 | #if "$$value" in p: 77 | # del p["$$value"] 78 | new_query = clone_or_update_query(dashboard_state, q, target_client, parent) 79 | if "id" not in new_query: 80 | print(f"Warning - query wasn't properly created, import might fail: {new_query}") 81 | else: 82 | if target_client.permisions_defined(): 83 | with requests.post(target_client.url+"/api/2.0/preview/sql/permissions/queries/"+new_query["id"], headers = target_client.headers, json=target_client.permissions) as r: 84 | permissions = r.json() 85 | logger.debug(f" Permissions set to {permissions}") 86 | visualizations = clone_query_visualization(target_client, q, new_query) 87 | dashboard_state["queries"][q["id"]] = {"new_id": new_query["id"], "visualizations": visualizations} 88 | 89 | #First loads the queries used as parameters. They need to be loaded first as the other will depend on these 90 | tasks = [] 91 | for q in dashboard["queries"] : 92 | if "is_parameter_query" not in q or q["is_parameter_query"]: 93 | load_query(q) 94 | task = { 95 | "task_key": "run_param_query_"+dashboard_state["queries"][q["id"]]["new_id"], 96 | "sql_task": { 97 | "query": { 98 | "query_id": dashboard_state["queries"][q["id"]]["new_id"] 99 | }, 100 | "warehouse_id": target_client.endpoint_id 101 | } 102 | } 103 | #Make sure they are sequential as we can have table creation concurrent exception otherwise. 104 | if len(tasks) > 0: 105 | task["depends_on"] = {"task_key": tasks[len(tasks)-1]["task_key"]} 106 | tasks.append(task) 107 | #Params requests now need to be run first. Starts a job to run them all and wait for the job... 108 | if len(tasks) > 0: 109 | print("Your dashboard has widgets. Widget queries need to be run first to be able to load other queries.") 110 | print("DBSQL-CLONE will now submit a job run to start the param queries. This will take a few minutes without serverless (few sec with serverless)...") 111 | print(f"You can check the progress in {target_client.url}#job/runs") 112 | from datetime import datetime 113 | now = datetime.now() 114 | dt_string = now.strftime("%d-%m-%Y-%H-%M-%S") 115 | settings = { 116 | "run_name": "dbdemos_init_param_queries_"+dt_string, 117 | "tasks": tasks 118 | } 119 | with requests.post(target_client.url+"/api/2.1/jobs/runs/submit", headers = target_client.headers, json=settings) as r: 120 | run = r.json() 121 | if 'error_code' in run: 122 | print(f"ERROR initializing the param queries job: {run} - params= {settings}. Downstream import will likely fail.") 123 | for i in range(100): 124 | i =+ 1 125 | with requests.get(target_client.url+"/api/2.1/jobs/runs/get", headers = target_client.headers, params=run) as r: 126 | if i >= 99 or "result_state" in r.json()["state"]: 127 | if i >= 99: 128 | print(f"ERROR initializing param queries. it looks like your init job is still running: {run} .") 129 | elif r.json()["state"]["result_state"] != "SUCCESS": 130 | print("ERROR initializing param queries. This will likely make next import to run as param query results are needed before using them in a new query.") 131 | print(f"To fix this issue, try to manually run the queries as defined in the job run {run}") 132 | else: 133 | print("Param queries initialization successful. Resume dashboard import...") 134 | break 135 | else: 136 | print(f"Waiting for parameter queries to run as they're needed to import the next queries ({run})...") 137 | time.sleep(10) 138 | 139 | 140 | 141 | #Then loads everything else, no matter the order 142 | with ThreadPoolExecutor(max_workers=max_workers) as executor: 143 | collections.deque(executor.map(load_query, [q for q in dashboard["queries"] if "is_parameter_query" in q and not q["is_parameter_query"]])) 144 | 145 | duplicate_dashboard(target_client, dashboard["dashboard"], dashboard_state, parent) 146 | return dashboard_state 147 | 148 | def clone_or_update_query(dashboard_state, q, target_client, parent): 149 | q_creation = { 150 | "data_source_id": target_client.data_source_id, 151 | "query": q["query"], 152 | "name": q["name"], 153 | "description": q["description"], 154 | "schedule": q.get("schedule", None), 155 | "tags": q.get("tags", None), 156 | "options": q["options"] 157 | } 158 | #Folder where the query will be installed 159 | if parent is not None: 160 | q_creation['parent'] = parent 161 | new_query = None 162 | if q['id'] in dashboard_state["queries"]: 163 | existing_query_id = dashboard_state["queries"][q['id']]["new_id"] 164 | # check if the query still exists (it might have been manually deleted by mistake) 165 | with requests.get(target_client.url + "/api/2.0/preview/sql/queries/" + existing_query_id, headers=target_client.headers) as r: 166 | existing_query = r.json() 167 | if 'id' in existing_query and 'moved_to_trash_at' not in existing_query: 168 | logger.debug(f" updating the existing query {existing_query_id}") 169 | with requests.post(target_client.url + "/api/2.0/preview/sql/queries/" + existing_query_id, headers=target_client.headers, json=q_creation) as r: 170 | new_query = r.json() 171 | if "visualizations" not in new_query: 172 | raise Exception(f"can't update query or query without vis. Shouldn't happen: {new_query} - {q_creation} - {existing_query_id}") 173 | # Delete all query visualization to reset its settings 174 | for v in new_query["visualizations"]: 175 | logger.debug(f" deleting query visualization {v['id']}") 176 | with requests.delete(target_client.url + "/api/2.0/preview/sql/visualizations/" + v["id"], headers=target_client.headers) as r: 177 | r.json() 178 | if not new_query: 179 | logger.debug(f" cloning query {q_creation}...") 180 | with requests.post(target_client.url + "/api/2.0/preview/sql/queries", headers=target_client.headers,json=q_creation) as r: 181 | new_query = r.json() 182 | logger.debug(f" new query created: {new_query} {r.text} {r.status_code}...") 183 | return new_query 184 | 185 | 186 | def clone_query_visualization(client: Client, query, target_query): 187 | # Sort both lists to retain visualization order on the query screen 188 | def get_first_vis(q): 189 | orig_table_visualizations = sorted( 190 | [i for i in q["visualizations"] if i["type"] == "TABLE"], 191 | key=lambda x: x["id"], 192 | ) 193 | if len(orig_table_visualizations) > 0: 194 | return orig_table_visualizations[0] 195 | return None 196 | #Update the default(first) visualization to match the existing one: 197 | # Sort this table like orig_table_visualizations. 198 | # The first elements in these lists should mirror one another. 199 | orig_default_table = get_first_vis(query) 200 | mapping = {} 201 | if orig_default_table: 202 | target_default_table = get_first_vis(target_query) 203 | default_table_viz_data = { 204 | "name": orig_default_table["name"], 205 | "description": orig_default_table["description"], 206 | "options": orig_default_table["options"] 207 | } 208 | if target_default_table is not None: 209 | mapping[orig_default_table["id"]] = target_default_table["id"] 210 | logger.debug(f" updating default Viz {target_default_table['id']}...") 211 | with requests.post(client.url+"/api/2.0/preview/sql/visualizations/"+target_default_table["id"], headers = client.headers, json=default_table_viz_data) as r: 212 | r.json() 213 | #Then create the other visualizations 214 | for v in sorted(query["visualizations"], key=lambda x: x["id"]): 215 | logger.debug(f" cloning Viz {v['id']}...") 216 | data = { 217 | "name": v["name"], 218 | "description": v["description"], 219 | "options": v["options"], 220 | "type": v["type"], 221 | "query_plan": v["query_plan"], 222 | "query_id": target_query["id"], 223 | } 224 | with requests.post(client.url+"/api/2.0/preview/sql/visualizations", headers = client.headers, json=data) as r: 225 | new_v = r.json() 226 | if "id" not in new_v: 227 | raise Exception(f"couldn't create visualization - shouldn't happen {new_v} - {data}") 228 | mapping[v["id"]] = new_v["id"] 229 | return mapping 230 | 231 | 232 | def duplicate_dashboard(client: Client, dashboard, dashboard_state, parent): 233 | data = {"name": dashboard["name"], 234 | "tags": dashboard["tags"], 235 | "data_source_id": client.data_source_id} 236 | #Folder where the dashboard will be installed 237 | if parent is not None: 238 | data['parent'] = parent 239 | 240 | new_dashboard = None 241 | if "new_id" in dashboard_state: 242 | with requests.get(client.url+"/api/2.0/preview/sql/dashboards/"+dashboard_state["new_id"], headers = client.headers) as r: 243 | existing_dashboard = r.json() 244 | if "options" in existing_dashboard and "moved_to_trash_at" not in existing_dashboard["options"]: 245 | logger.debug(" dashboard exists, updating it") 246 | with requests.post(client.url+"/api/2.0/preview/sql/dashboards/"+dashboard_state["new_id"], headers = client.headers, json=data) as r: 247 | new_dashboard = r.json() 248 | if "widgets" not in new_dashboard: 249 | logger.debug(f"ERROR: dashboard doesn't have widget, shouldn't happen - {new_dashboard}") 250 | else: 251 | #Drop all the widgets and re-create them 252 | for widget in new_dashboard["widgets"]: 253 | logger.debug(f" deleting widget {widget['id']} from existing dashboard {new_dashboard['id']}") 254 | with requests.delete(client.url+"/api/2.0/preview/sql/widgets/"+widget['id'], headers = client.headers) as r: 255 | r.json() 256 | else: 257 | logger.debug(" couldn't find the dashboard defined in the state, it probably has been deleted.") 258 | if new_dashboard is None: 259 | logger.debug(f" creating new dashboard...") 260 | with requests.post(client.url+"/api/2.0/preview/sql/dashboards", headers = client.headers, json=data) as r: 261 | new_dashboard = r.json() 262 | dashboard_state["new_id"] = new_dashboard["id"] 263 | if client.permisions_defined(): 264 | with requests.post(client.url+"/api/2.0/preview/sql/permissions/dashboards/"+new_dashboard["id"], headers = client.headers, json=client.permissions) as r: 265 | permissions = r.json() 266 | logger.debug(f" Dashboard permissions set to {permissions}") 267 | 268 | def load_widget(widget): 269 | logger.debug(f" cloning widget {widget}...") 270 | visualization_id_clone = None 271 | if "visualization" in widget: 272 | query_id = widget["visualization"]["query"]["id"] 273 | visualization_id = widget["visualization"]["id"] 274 | visualization_id_clone = dashboard_state["queries"][query_id]["visualizations"][visualization_id] 275 | data = { 276 | "dashboard_id": new_dashboard["id"], 277 | "visualization_id": visualization_id_clone, 278 | "text": widget["text"], 279 | "options": widget["options"], 280 | "width": widget["width"] 281 | } 282 | with requests.post(client.url+"/api/2.0/preview/sql/widgets", headers = client.headers, json=data) as r: 283 | r.json() 284 | 285 | with ThreadPoolExecutor(max_workers=max_workers) as executor: 286 | collections.deque(executor.map(load_widget, dashboard["widgets"])) 287 | 288 | 289 | return new_dashboard -------------------------------------------------------------------------------- /test/19394330-2274-4b4b-90ce-d415a7ff2130.json: -------------------------------------------------------------------------------- 1 | { 2 | "queries": [ 3 | { 4 | "id": "2c039fe6-7197-4844-a430-e4de3f974c92", 5 | "name": "Churn - Customers churned - Universal", 6 | "description": null, 7 | "query": "SELECT count(*) as past_churn FROM dbdemos_c360.churn_users WHERE churn=1", 8 | "is_draft": false, 9 | "updated_at": "2023-01-25T13:12:41Z", 10 | "created_at": "2023-01-25T13:12:36Z", 11 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 12 | "options": { 13 | "parent": "folders/4156927442168299", 14 | "apply_auto_limit": false, 15 | "folder_node_status": "ACTIVE", 16 | "folder_node_internal_name": "tree/791251337284969", 17 | "parameters": [] 18 | }, 19 | "version": 1, 20 | "tags": [], 21 | "is_safe": true, 22 | "user_id": 7644138420879474, 23 | "run_as_role": null, 24 | "run_as_service_principal_id": null, 25 | "schedule": null, 26 | "latest_query_data_id": "a4799b13-ded3-4576-a087-73ecf9b3802f", 27 | "visualizations": [ 28 | { 29 | "id": "00f0ef63-15ec-4285-b62b-51d5e291642a", 30 | "type": "TABLE", 31 | "name": "Table", 32 | "description": "", 33 | "options": { 34 | "version": 2 35 | }, 36 | "updated_at": "2023-01-25T13:12:40Z", 37 | "created_at": "2023-01-25T13:12:40Z", 38 | "query_plan": null 39 | }, 40 | { 41 | "id": "515b67d1-8560-4bf8-9b4a-82dac577abc2", 42 | "type": "COUNTER", 43 | "name": "Customer churned", 44 | "description": "", 45 | "options": { 46 | "counterLabel": "", 47 | "counterColName": "past_churn", 48 | "rowNumber": 1, 49 | "targetRowNumber": 1, 50 | "stringDecimal": 0, 51 | "stringDecChar": ".", 52 | "stringThouSep": ",", 53 | "tooltipFormat": "0,0.000", 54 | "condensed": true, 55 | "withRowNumber": true, 56 | "stringSuffix": " customers" 57 | }, 58 | "updated_at": "2023-01-25T13:13:26Z", 59 | "created_at": "2023-01-25T13:12:40Z", 60 | "query_plan": null 61 | }, 62 | { 63 | "id": "b183ef59-baf0-4066-9b33-a011481374e2", 64 | "type": "TABLE", 65 | "name": "Table", 66 | "description": "", 67 | "options": { 68 | "version": 2 69 | }, 70 | "updated_at": "2023-01-25T13:12:41Z", 71 | "created_at": "2023-01-25T13:12:41Z", 72 | "query_plan": null 73 | }, 74 | { 75 | "id": "bbdf748e-d632-493e-8c85-e2f9d49c85fd", 76 | "type": "TABLE", 77 | "name": "Table", 78 | "description": "", 79 | "options": { 80 | "version": 2 81 | }, 82 | "updated_at": "2023-01-25T13:12:39Z", 83 | "created_at": "2023-01-25T13:12:36Z", 84 | "query_plan": null 85 | } 86 | ], 87 | "is_favorite": false, 88 | "last_modified_by": { 89 | "id": 7644138420879474, 90 | "name": "Quentin Ambard", 91 | "email": "quentin.ambard@databricks.com", 92 | "profile_image_url": "https://www.gravatar.com/avatar/62d8f76888369585890af5b5dce9a395?s=40&d=identicon", 93 | "is_db_admin": false 94 | }, 95 | "query_draft": null, 96 | "job_id": null, 97 | "user": { 98 | "id": 7644138420879474, 99 | "name": "Quentin Ambard", 100 | "email": "quentin.ambard@databricks.com", 101 | "profile_image_url": "https://www.gravatar.com/avatar/62d8f76888369585890af5b5dce9a395?s=40&d=identicon", 102 | "is_db_admin": false 103 | }, 104 | "parent": "folders/4156927442168299", 105 | "is_archived": false, 106 | "can_edit": true, 107 | "permission_tier": "CAN_MANAGE", 108 | "is_parameter_query": false 109 | }, 110 | { 111 | "id": "fca73ed5-70ea-47bb-a085-fe71e3a9d8f1", 112 | "name": "Churn - Amount sales per month - Universal", 113 | "description": null, 114 | "query": "SELECT sum(amount), date_format(to_timestamp(u.creation_date, \"MM-dd-yyyy H:mm:ss\"), \"yyyy-MM\") m FROM dbdemos_c360.churn_orders o \n\t\tINNER JOIN dbdemos_c360.churn_users u using (user_id)\n\t\t\tgroup by m", 115 | "is_draft": false, 116 | "updated_at": "2023-01-25T13:12:41Z", 117 | "created_at": "2023-01-25T13:12:36Z", 118 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 119 | "options": { 120 | "parent": "folders/4156927442168299", 121 | "apply_auto_limit": false, 122 | "folder_node_status": "ACTIVE", 123 | "folder_node_internal_name": "tree/791251337284970", 124 | "visualization_control_order": [], 125 | "parameters": [] 126 | }, 127 | "version": 1, 128 | "tags": [], 129 | "is_safe": true, 130 | "user_id": 7644138420879474, 131 | "run_as_role": null, 132 | "run_as_service_principal_id": null, 133 | "schedule": null, 134 | "latest_query_data_id": "aa7860de-a703-4be2-b0d5-bc305d21e645", 135 | "visualizations": [ 136 | { 137 | "id": "104db01f-168d-4fa4-a81c-89af73b848ad", 138 | "type": "CHART", 139 | "name": "Amount sales per month", 140 | "description": "", 141 | "options": { 142 | "version": 2, 143 | "globalSeriesType": "area", 144 | "sortX": true, 145 | "sortY": true, 146 | "legend": { 147 | "traceorder": "normal" 148 | }, 149 | "xAxis": { 150 | "type": "-", 151 | "labels": { 152 | "enabled": true 153 | }, 154 | "title": { 155 | "text": "Date" 156 | } 157 | }, 158 | "yAxis": [ 159 | { 160 | "type": "-", 161 | "title": { 162 | "text": "Monthly Recurrent Revenue" 163 | } 164 | }, 165 | { 166 | "type": "-", 167 | "opposite": true 168 | } 169 | ], 170 | "alignYAxesAtZero": false, 171 | "error_y": { 172 | "type": "data", 173 | "visible": true 174 | }, 175 | "series": { 176 | "stacking": null, 177 | "error_y": { 178 | "type": "data", 179 | "visible": true 180 | } 181 | }, 182 | "seriesOptions": { 183 | "column_c0c32ed033612": { 184 | "yAxis": 0, 185 | "type": "area", 186 | "color": "#87BFE0" 187 | } 188 | }, 189 | "valuesOptions": {}, 190 | "direction": { 191 | "type": "counterclockwise" 192 | }, 193 | "sizemode": "diameter", 194 | "coefficient": 1, 195 | "numberFormat": "0,0[.]00000", 196 | "percentFormat": "0[.]00%", 197 | "textFormat": "", 198 | "missingValuesAsZero": true, 199 | "useAggregationsUi": true, 200 | "swappedAxes": false, 201 | "dateTimeFormat": "YYYY-MM-DD HH:mm", 202 | "showDataLabels": false, 203 | "columnConfigurationMap": { 204 | "x": { 205 | "column": "m", 206 | "id": "column_c0c32ed034641" 207 | }, 208 | "y": [ 209 | { 210 | "id": "column_c0c32ed033612", 211 | "column": "sum(amount)", 212 | "transform": "SUM" 213 | } 214 | ] 215 | }, 216 | "isAggregationOn": true, 217 | "condensed": true, 218 | "withRowNumber": true 219 | }, 220 | "updated_at": "2023-01-25T13:13:26Z", 221 | "created_at": "2023-01-25T13:12:40Z", 222 | "query_plan": null 223 | }, 224 | { 225 | "id": "b951f526-49e7-4302-a73e-1100dd0f4926", 226 | "type": "TABLE", 227 | "name": "Table", 228 | "description": "", 229 | "options": {}, 230 | "updated_at": "2023-01-25T13:12:41Z", 231 | "created_at": "2023-01-25T13:12:41Z", 232 | "query_plan": null 233 | }, 234 | { 235 | "id": "bec558e2-acc3-4854-8084-f0d1b22b88c5", 236 | "type": "TABLE", 237 | "name": "Table", 238 | "description": "", 239 | "options": {}, 240 | "updated_at": "2023-01-25T13:12:41Z", 241 | "created_at": "2023-01-25T13:12:41Z", 242 | "query_plan": null 243 | }, 244 | { 245 | "id": "e8753217-6f65-4f68-bc30-7cdad10b0373", 246 | "type": "TABLE", 247 | "name": "Table", 248 | "description": "", 249 | "options": {}, 250 | "updated_at": "2023-01-25T13:12:39Z", 251 | "created_at": "2023-01-25T13:12:36Z", 252 | "query_plan": null 253 | } 254 | ], 255 | "is_favorite": false, 256 | "last_modified_by": { 257 | "id": 7644138420879474, 258 | "name": "Quentin Ambard", 259 | "email": "quentin.ambard@databricks.com", 260 | "profile_image_url": "https://www.gravatar.com/avatar/62d8f76888369585890af5b5dce9a395?s=40&d=identicon", 261 | "is_db_admin": false 262 | }, 263 | "query_draft": null, 264 | "job_id": null, 265 | "user": { 266 | "id": 7644138420879474, 267 | "name": "Quentin Ambard", 268 | "email": "quentin.ambard@databricks.com", 269 | "profile_image_url": "https://www.gravatar.com/avatar/62d8f76888369585890af5b5dce9a395?s=40&d=identicon", 270 | "is_db_admin": false 271 | }, 272 | "parent": "folders/4156927442168299", 273 | "is_archived": false, 274 | "can_edit": true, 275 | "permission_tier": "CAN_MANAGE", 276 | "is_parameter_query": false 277 | }, 278 | { 279 | "id": "52c433dd-1d96-4b8c-a7f7-a43f9931dfc9", 280 | "name": "Churn - Avg Monthly Charges by Payment Method - Universal", 281 | "description": null, 282 | "query": "SELECT canal, sum(amount)/100 as MRR FROM dbdemos_c360.churn_orders o \n\t\tINNER JOIN dbdemos_c360.churn_users using (user_id)\n\t\t\t\tWHERE canal is not null and month(to_timestamp(transaction_date, 'MM-dd-yyyy HH:mm:ss')) = \n\t\t (select max(month(to_timestamp(transaction_date, 'MM-dd-yyyy HH:mm:ss'))) from dbdemos_c360.churn_orders) \n group by canal;", 283 | "is_draft": false, 284 | "updated_at": "2023-01-25T13:12:41Z", 285 | "created_at": "2023-01-25T13:12:36Z", 286 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 287 | "options": { 288 | "parent": "folders/4156927442168299", 289 | "apply_auto_limit": false, 290 | "folder_node_status": "ACTIVE", 291 | "folder_node_internal_name": "tree/791251337284971", 292 | "parameters": [] 293 | }, 294 | "version": 1, 295 | "tags": [], 296 | "is_safe": true, 297 | "user_id": 7644138420879474, 298 | "run_as_role": null, 299 | "run_as_service_principal_id": null, 300 | "schedule": null, 301 | "latest_query_data_id": "bdcbe71d-9a39-4207-a4b0-36e4c605de26", 302 | "visualizations": [ 303 | { 304 | "id": "05109623-d900-466b-b7fc-fdc939906fa8", 305 | "type": "TABLE", 306 | "name": "Table", 307 | "description": "", 308 | "options": { 309 | "version": 2 310 | }, 311 | "updated_at": "2023-01-25T13:12:40Z", 312 | "created_at": "2023-01-25T13:12:40Z", 313 | "query_plan": null 314 | }, 315 | { 316 | "id": "7422fa69-d472-4f99-9656-a5149410c7af", 317 | "type": "TABLE", 318 | "name": "Table", 319 | "description": "", 320 | "options": { 321 | "version": 2 322 | }, 323 | "updated_at": "2023-01-25T13:12:41Z", 324 | "created_at": "2023-01-25T13:12:41Z", 325 | "query_plan": null 326 | }, 327 | { 328 | "id": "9daab35f-cf12-44ae-8644-857a4d08e28c", 329 | "type": "TABLE", 330 | "name": "Table", 331 | "description": "", 332 | "options": { 333 | "version": 2 334 | }, 335 | "updated_at": "2023-01-25T13:12:39Z", 336 | "created_at": "2023-01-25T13:12:36Z", 337 | "query_plan": null 338 | }, 339 | { 340 | "id": "a9366c7d-1cc5-4a74-ad19-cd9b57c9cf36", 341 | "type": "CHART", 342 | "name": "Chart", 343 | "description": "", 344 | "options": { 345 | "version": 2, 346 | "globalSeriesType": "bubble", 347 | "sortX": true, 348 | "sortY": true, 349 | "legend": { 350 | "traceorder": "normal", 351 | "enabled": true, 352 | "placement": "auto" 353 | }, 354 | "xAxis": { 355 | "type": "-", 356 | "labels": { 357 | "enabled": true 358 | }, 359 | "title": { 360 | "text": "PaymentMethod" 361 | } 362 | }, 363 | "yAxis": [ 364 | { 365 | "type": "linear", 366 | "title": { 367 | "text": "AvgMonthlyCharges" 368 | } 369 | }, 370 | { 371 | "type": "linear", 372 | "opposite": true, 373 | "title": { 374 | "text": null 375 | } 376 | } 377 | ], 378 | "alignYAxesAtZero": false, 379 | "error_y": { 380 | "type": "data", 381 | "visible": true 382 | }, 383 | "series": { 384 | "stacking": null, 385 | "error_y": { 386 | "type": "data", 387 | "visible": true 388 | } 389 | }, 390 | "seriesOptions": { 391 | "MRR": { 392 | "yAxis": 0 393 | } 394 | }, 395 | "valuesOptions": {}, 396 | "direction": { 397 | "type": "counterclockwise" 398 | }, 399 | "sizemode": "diameter", 400 | "coefficient": 0.003, 401 | "numberFormat": "0,0[.]00000", 402 | "percentFormat": "0[.]00%", 403 | "textFormat": "", 404 | "missingValuesAsZero": true, 405 | "useAggregationsUi": false, 406 | "swappedAxes": false, 407 | "dateTimeFormat": "YYYY-MM-DD HH:mm", 408 | "showDataLabels": false, 409 | "columnConfigurationMap": { 410 | "x": { 411 | "column": "canal" 412 | }, 413 | "y": [ 414 | { 415 | "column": "MRR" 416 | } 417 | ], 418 | "size": { 419 | "column": "MRR" 420 | }, 421 | "series": { 422 | "column": null 423 | } 424 | }, 425 | "condensed": true, 426 | "withRowNumber": true 427 | }, 428 | "updated_at": "2023-01-25T13:13:26Z", 429 | "created_at": "2023-01-25T13:12:41Z", 430 | "query_plan": null 431 | } 432 | ], 433 | "is_favorite": false, 434 | "last_modified_by": { 435 | "id": 7644138420879474, 436 | "name": "Quentin Ambard", 437 | "email": "quentin.ambard@databricks.com", 438 | "profile_image_url": "https://www.gravatar.com/avatar/62d8f76888369585890af5b5dce9a395?s=40&d=identicon", 439 | "is_db_admin": false 440 | }, 441 | "query_draft": null, 442 | "job_id": null, 443 | "user": { 444 | "id": 7644138420879474, 445 | "name": "Quentin Ambard", 446 | "email": "quentin.ambard@databricks.com", 447 | "profile_image_url": "https://www.gravatar.com/avatar/62d8f76888369585890af5b5dce9a395?s=40&d=identicon", 448 | "is_db_admin": false 449 | }, 450 | "parent": "folders/4156927442168299", 451 | "is_archived": false, 452 | "can_edit": true, 453 | "permission_tier": "CAN_MANAGE", 454 | "is_parameter_query": false 455 | }, 456 | { 457 | "id": "5243b9e9-bd6d-474f-853a-8b25f4ee67d6", 458 | "name": "Churn - Churn by Gender - Universal", 459 | "description": null, 460 | "query": "\nSELECT gender, count(gender) as total_churn FROM dbdemos_c360.churn_features where churn = 1 GROUP BY gender\n\n\n", 461 | "is_draft": false, 462 | "updated_at": "2023-01-25T13:12:41Z", 463 | "created_at": "2023-01-25T13:12:36Z", 464 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 465 | "options": { 466 | "parent": "folders/4156927442168299", 467 | "apply_auto_limit": false, 468 | "folder_node_status": "ACTIVE", 469 | "folder_node_internal_name": "tree/791251337284972", 470 | "parameters": [] 471 | }, 472 | "version": 1, 473 | "tags": [], 474 | "is_safe": true, 475 | "user_id": 7644138420879474, 476 | "run_as_role": null, 477 | "run_as_service_principal_id": null, 478 | "schedule": null, 479 | "latest_query_data_id": "b14eab28-3183-4290-9287-bde04634f567", 480 | "visualizations": [ 481 | { 482 | "id": "33aef9a7-d4ab-41d1-a7d2-6a246658a2b9", 483 | "type": "TABLE", 484 | "name": "Table", 485 | "description": "", 486 | "options": { 487 | "version": 2 488 | }, 489 | "updated_at": "2023-01-25T13:12:40Z", 490 | "created_at": "2023-01-25T13:12:40Z", 491 | "query_plan": null 492 | }, 493 | { 494 | "id": "622a4763-b8a1-485a-994e-9d593242f020", 495 | "type": "TABLE", 496 | "name": "Table", 497 | "description": "", 498 | "options": { 499 | "version": 2 500 | }, 501 | "updated_at": "2023-01-25T13:12:39Z", 502 | "created_at": "2023-01-25T13:12:36Z", 503 | "query_plan": null 504 | }, 505 | { 506 | "id": "82bd3bd9-7c58-4410-94af-587870ff2743", 507 | "type": "TABLE", 508 | "name": "Table", 509 | "description": "", 510 | "options": { 511 | "version": 2 512 | }, 513 | "updated_at": "2023-01-25T13:12:41Z", 514 | "created_at": "2023-01-25T13:12:41Z", 515 | "query_plan": null 516 | }, 517 | { 518 | "id": "de8a29c2-fd17-4bfc-969e-361b9c9bf1e6", 519 | "type": "CHART", 520 | "name": "Chart", 521 | "description": "", 522 | "options": { 523 | "version": 2, 524 | "globalSeriesType": "pie", 525 | "sortX": true, 526 | "sortY": true, 527 | "legend": { 528 | "traceorder": "normal", 529 | "enabled": true, 530 | "placement": "auto" 531 | }, 532 | "xAxis": { 533 | "type": "-", 534 | "labels": { 535 | "enabled": true 536 | }, 537 | "title": { 538 | "text": "gender_Male" 539 | } 540 | }, 541 | "yAxis": [ 542 | { 543 | "type": "-", 544 | "title": { 545 | "text": "total_churn" 546 | } 547 | }, 548 | { 549 | "type": "-", 550 | "opposite": true, 551 | "title": { 552 | "text": null 553 | } 554 | } 555 | ], 556 | "alignYAxesAtZero": false, 557 | "error_y": { 558 | "type": "data", 559 | "visible": true 560 | }, 561 | "series": { 562 | "stacking": null, 563 | "error_y": { 564 | "type": "data", 565 | "visible": true 566 | } 567 | }, 568 | "seriesOptions": { 569 | "total_churn": { 570 | "yAxis": 0 571 | } 572 | }, 573 | "valuesOptions": { 574 | "0": { 575 | "color": "#F6C17F" 576 | }, 577 | "1": { 578 | "color": "#AB506F" 579 | } 580 | }, 581 | "direction": { 582 | "type": "counterclockwise" 583 | }, 584 | "sizemode": "diameter", 585 | "coefficient": 1, 586 | "numberFormat": "0,0[.]00000", 587 | "percentFormat": "0[.]00%", 588 | "textFormat": "", 589 | "missingValuesAsZero": true, 590 | "useAggregationsUi": false, 591 | "swappedAxes": false, 592 | "dateTimeFormat": "YYYY-MM-DD HH:mm", 593 | "showDataLabels": true, 594 | "columnConfigurationMap": { 595 | "x": { 596 | "column": "gender" 597 | }, 598 | "y": [ 599 | { 600 | "column": "total_churn" 601 | } 602 | ], 603 | "series": { 604 | "column": null 605 | } 606 | }, 607 | "showPlotlyControls": true, 608 | "condensed": true, 609 | "withRowNumber": true, 610 | "isAggregationOn": false 611 | }, 612 | "updated_at": "2023-01-25T13:13:26Z", 613 | "created_at": "2023-01-25T13:12:41Z", 614 | "query_plan": null 615 | } 616 | ], 617 | "is_favorite": false, 618 | "last_modified_by": { 619 | "id": 7644138420879474, 620 | "name": "Quentin Ambard", 621 | "email": "quentin.ambard@databricks.com", 622 | "profile_image_url": "https://www.gravatar.com/avatar/62d8f76888369585890af5b5dce9a395?s=40&d=identicon", 623 | "is_db_admin": false 624 | }, 625 | "query_draft": null, 626 | "job_id": null, 627 | "user": { 628 | "id": 7644138420879474, 629 | "name": "Quentin Ambard", 630 | "email": "quentin.ambard@databricks.com", 631 | "profile_image_url": "https://www.gravatar.com/avatar/62d8f76888369585890af5b5dce9a395?s=40&d=identicon", 632 | "is_db_admin": false 633 | }, 634 | "parent": "folders/4156927442168299", 635 | "is_archived": false, 636 | "can_edit": true, 637 | "permission_tier": "CAN_MANAGE", 638 | "is_parameter_query": false 639 | } 640 | ], 641 | "id": "19394330-2274-4b4b-90ce-d415a7ff2130", 642 | "dashboard": { 643 | "id": "19394330-2274-4b4b-90ce-d415a7ff2130", 644 | "slug": "customer-churn---universal", 645 | "name": "Customer Churn - Universal", 646 | "user_id": 7644138420879474, 647 | "dashboard_filters_enabled": false, 648 | "widgets": [ 649 | { 650 | "id": "28c4220e-a116-4131-887b-77242dc09478", 651 | "width": 1, 652 | "options": { 653 | "isHidden": false, 654 | "position": { 655 | "autoHeight": false, 656 | "sizeX": 6, 657 | "sizeY": 1, 658 | "minSizeX": 1, 659 | "maxSizeX": 6, 660 | "minSizeY": 1, 661 | "maxSizeY": 1000, 662 | "col": 0, 663 | "row": 15 664 | }, 665 | "parameterMappings": {} 666 | }, 667 | "dashboard_id": "19394330-2274-4b4b-90ce-d415a7ff2130", 668 | "text": "Field Demo dashboard. Please do not edit\n![tracking_img](https://www.google-analytics.com/collect?v=1>m=GTM-NKQ8TT7&tid=UA-163989034-1&cid=555&aip=1&t=event&ec=field_demos&ea=display&dp=%2F42_field_demos%2Ffeatures%2Fmlops%2Fchurn&dt=DASHBOARD_CHURN)", 669 | "updated_at": "2023-01-25T13:12:51Z", 670 | "created_at": "2023-01-25T13:12:51Z" 671 | }, 672 | { 673 | "id": "3bc6c07f-421c-4585-8c34-b15c51890472", 674 | "width": 1, 675 | "options": { 676 | "parameterMappings": {}, 677 | "title": "Past Churn", 678 | "description": "", 679 | "isHidden": false, 680 | "position": { 681 | "autoHeight": false, 682 | "sizeX": 1, 683 | "sizeY": 4, 684 | "minSizeX": 1, 685 | "maxSizeX": 6, 686 | "minSizeY": 1, 687 | "maxSizeY": 1000, 688 | "col": 0, 689 | "row": 4 690 | }, 691 | "overrideColors": false 692 | }, 693 | "dashboard_id": "19394330-2274-4b4b-90ce-d415a7ff2130", 694 | "text": "", 695 | "updated_at": "2023-01-25T13:12:51Z", 696 | "created_at": "2023-01-25T13:12:51Z", 697 | "visualization": { 698 | "id": "515b67d1-8560-4bf8-9b4a-82dac577abc2", 699 | "type": "COUNTER", 700 | "name": "Customer churned", 701 | "description": "", 702 | "options": { 703 | "counterLabel": "", 704 | "counterColName": "past_churn", 705 | "rowNumber": 1, 706 | "targetRowNumber": 1, 707 | "stringDecimal": 0, 708 | "stringDecChar": ".", 709 | "stringThouSep": ",", 710 | "tooltipFormat": "0,0.000", 711 | "condensed": true, 712 | "withRowNumber": true, 713 | "stringSuffix": " customers" 714 | }, 715 | "updated_at": "2023-01-25T13:13:26Z", 716 | "created_at": "2023-01-25T13:12:40Z", 717 | "query_plan": null, 718 | "query": { 719 | "id": "2c039fe6-7197-4844-a430-e4de3f974c92", 720 | "name": "Churn - Customers churned - Universal", 721 | "description": null, 722 | "query": "SELECT count(*) as past_churn FROM dbdemos_c360.churn_users WHERE churn=1", 723 | "is_draft": false, 724 | "updated_at": "2023-01-25T13:12:41Z", 725 | "created_at": "2023-01-25T13:12:36Z", 726 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 727 | "options": { 728 | "parent": "folders/4156927442168299", 729 | "apply_auto_limit": false, 730 | "folder_node_status": "ACTIVE", 731 | "folder_node_internal_name": "tree/791251337284969", 732 | "parameters": [] 733 | }, 734 | "version": 1, 735 | "tags": [], 736 | "is_safe": true, 737 | "user_id": 7644138420879474, 738 | "run_as_role": null, 739 | "run_as_service_principal_id": null, 740 | "schedule": null, 741 | "can_edit": true 742 | } 743 | } 744 | }, 745 | { 746 | "id": "c120b414-1fb0-46fd-bc5b-04b4045c3c62", 747 | "width": 1, 748 | "options": { 749 | "parameterMappings": {}, 750 | "title": "MRR over time", 751 | "description": "", 752 | "isHidden": false, 753 | "position": { 754 | "autoHeight": false, 755 | "sizeX": 3, 756 | "sizeY": 7, 757 | "minSizeX": 1, 758 | "maxSizeX": 6, 759 | "minSizeY": 5, 760 | "maxSizeY": 1000, 761 | "col": 3, 762 | "row": 19 763 | }, 764 | "overrideColors": false 765 | }, 766 | "dashboard_id": "19394330-2274-4b4b-90ce-d415a7ff2130", 767 | "text": "", 768 | "updated_at": "2023-01-25T13:12:51Z", 769 | "created_at": "2023-01-25T13:12:51Z", 770 | "visualization": { 771 | "id": "104db01f-168d-4fa4-a81c-89af73b848ad", 772 | "type": "CHART", 773 | "name": "Amount sales per month", 774 | "description": "", 775 | "options": { 776 | "version": 2, 777 | "globalSeriesType": "area", 778 | "sortX": true, 779 | "sortY": true, 780 | "legend": { 781 | "traceorder": "normal" 782 | }, 783 | "xAxis": { 784 | "type": "-", 785 | "labels": { 786 | "enabled": true 787 | }, 788 | "title": { 789 | "text": "Date" 790 | } 791 | }, 792 | "yAxis": [ 793 | { 794 | "type": "-", 795 | "title": { 796 | "text": "Monthly Recurrent Revenue" 797 | } 798 | }, 799 | { 800 | "type": "-", 801 | "opposite": true 802 | } 803 | ], 804 | "alignYAxesAtZero": false, 805 | "error_y": { 806 | "type": "data", 807 | "visible": true 808 | }, 809 | "series": { 810 | "stacking": null, 811 | "error_y": { 812 | "type": "data", 813 | "visible": true 814 | } 815 | }, 816 | "seriesOptions": { 817 | "column_c0c32ed033612": { 818 | "yAxis": 0, 819 | "type": "area", 820 | "color": "#87BFE0" 821 | } 822 | }, 823 | "valuesOptions": {}, 824 | "direction": { 825 | "type": "counterclockwise" 826 | }, 827 | "sizemode": "diameter", 828 | "coefficient": 1, 829 | "numberFormat": "0,0[.]00000", 830 | "percentFormat": "0[.]00%", 831 | "textFormat": "", 832 | "missingValuesAsZero": true, 833 | "useAggregationsUi": true, 834 | "swappedAxes": false, 835 | "dateTimeFormat": "YYYY-MM-DD HH:mm", 836 | "showDataLabels": false, 837 | "columnConfigurationMap": { 838 | "x": { 839 | "column": "m", 840 | "id": "column_c0c32ed034641" 841 | }, 842 | "y": [ 843 | { 844 | "id": "column_c0c32ed033612", 845 | "column": "sum(amount)", 846 | "transform": "SUM" 847 | } 848 | ] 849 | }, 850 | "isAggregationOn": true, 851 | "condensed": true, 852 | "withRowNumber": true 853 | }, 854 | "updated_at": "2023-01-25T13:13:26Z", 855 | "created_at": "2023-01-25T13:12:40Z", 856 | "query_plan": null, 857 | "query": { 858 | "id": "fca73ed5-70ea-47bb-a085-fe71e3a9d8f1", 859 | "name": "Churn - Amount sales per month - Universal", 860 | "description": null, 861 | "query": "SELECT sum(amount), date_format(to_timestamp(u.creation_date, \"MM-dd-yyyy H:mm:ss\"), \"yyyy-MM\") m FROM dbdemos_c360.churn_orders o \n\t\tINNER JOIN dbdemos_c360.churn_users u using (user_id)\n\t\t\tgroup by m", 862 | "is_draft": false, 863 | "updated_at": "2023-01-25T13:12:41Z", 864 | "created_at": "2023-01-25T13:12:36Z", 865 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 866 | "options": { 867 | "parent": "folders/4156927442168299", 868 | "apply_auto_limit": false, 869 | "folder_node_status": "ACTIVE", 870 | "folder_node_internal_name": "tree/791251337284970", 871 | "visualization_control_order": [], 872 | "parameters": [] 873 | }, 874 | "version": 1, 875 | "tags": [], 876 | "is_safe": true, 877 | "user_id": 7644138420879474, 878 | "run_as_role": null, 879 | "run_as_service_principal_id": null, 880 | "schedule": null, 881 | "can_edit": true 882 | } 883 | } 884 | }, 885 | { 886 | "id": "6d205725-756b-4b73-8926-e7d044254a40", 887 | "width": 1, 888 | "options": { 889 | "parameterMappings": {}, 890 | "title": "Avg Monthly Charges by Payment Method", 891 | "description": "", 892 | "isHidden": false, 893 | "position": { 894 | "autoHeight": false, 895 | "sizeX": 3, 896 | "sizeY": 7, 897 | "minSizeX": 1, 898 | "maxSizeX": 6, 899 | "minSizeY": 5, 900 | "maxSizeY": 1000, 901 | "col": 0, 902 | "row": 8 903 | } 904 | }, 905 | "dashboard_id": "19394330-2274-4b4b-90ce-d415a7ff2130", 906 | "text": "", 907 | "updated_at": "2023-01-25T13:12:52Z", 908 | "created_at": "2023-01-25T13:12:52Z", 909 | "visualization": { 910 | "id": "a9366c7d-1cc5-4a74-ad19-cd9b57c9cf36", 911 | "type": "CHART", 912 | "name": "Chart", 913 | "description": "", 914 | "options": { 915 | "version": 2, 916 | "globalSeriesType": "bubble", 917 | "sortX": true, 918 | "sortY": true, 919 | "legend": { 920 | "traceorder": "normal", 921 | "enabled": true, 922 | "placement": "auto" 923 | }, 924 | "xAxis": { 925 | "type": "-", 926 | "labels": { 927 | "enabled": true 928 | }, 929 | "title": { 930 | "text": "PaymentMethod" 931 | } 932 | }, 933 | "yAxis": [ 934 | { 935 | "type": "linear", 936 | "title": { 937 | "text": "AvgMonthlyCharges" 938 | } 939 | }, 940 | { 941 | "type": "linear", 942 | "opposite": true, 943 | "title": { 944 | "text": null 945 | } 946 | } 947 | ], 948 | "alignYAxesAtZero": false, 949 | "error_y": { 950 | "type": "data", 951 | "visible": true 952 | }, 953 | "series": { 954 | "stacking": null, 955 | "error_y": { 956 | "type": "data", 957 | "visible": true 958 | } 959 | }, 960 | "seriesOptions": { 961 | "MRR": { 962 | "yAxis": 0 963 | } 964 | }, 965 | "valuesOptions": {}, 966 | "direction": { 967 | "type": "counterclockwise" 968 | }, 969 | "sizemode": "diameter", 970 | "coefficient": 0.003, 971 | "numberFormat": "0,0[.]00000", 972 | "percentFormat": "0[.]00%", 973 | "textFormat": "", 974 | "missingValuesAsZero": true, 975 | "useAggregationsUi": false, 976 | "swappedAxes": false, 977 | "dateTimeFormat": "YYYY-MM-DD HH:mm", 978 | "showDataLabels": false, 979 | "columnConfigurationMap": { 980 | "x": { 981 | "column": "canal" 982 | }, 983 | "y": [ 984 | { 985 | "column": "MRR" 986 | } 987 | ], 988 | "size": { 989 | "column": "MRR" 990 | }, 991 | "series": { 992 | "column": null 993 | } 994 | }, 995 | "condensed": true, 996 | "withRowNumber": true 997 | }, 998 | "updated_at": "2023-01-25T13:13:26Z", 999 | "created_at": "2023-01-25T13:12:41Z", 1000 | "query_plan": null, 1001 | "query": { 1002 | "id": "52c433dd-1d96-4b8c-a7f7-a43f9931dfc9", 1003 | "name": "Churn - Avg Monthly Charges by Payment Method - Universal", 1004 | "description": null, 1005 | "query": "SELECT canal, sum(amount)/100 as MRR FROM dbdemos_c360.churn_orders o \n\t\tINNER JOIN dbdemos_c360.churn_users using (user_id)\n\t\t\t\tWHERE canal is not null and month(to_timestamp(transaction_date, 'MM-dd-yyyy HH:mm:ss')) = \n\t\t (select max(month(to_timestamp(transaction_date, 'MM-dd-yyyy HH:mm:ss'))) from dbdemos_c360.churn_orders) \n group by canal;", 1006 | "is_draft": false, 1007 | "updated_at": "2023-01-25T13:12:41Z", 1008 | "created_at": "2023-01-25T13:12:36Z", 1009 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 1010 | "options": { 1011 | "parent": "folders/4156927442168299", 1012 | "apply_auto_limit": false, 1013 | "folder_node_status": "ACTIVE", 1014 | "folder_node_internal_name": "tree/791251337284971", 1015 | "parameters": [] 1016 | }, 1017 | "version": 1, 1018 | "tags": [], 1019 | "is_safe": true, 1020 | "user_id": 7644138420879474, 1021 | "run_as_role": null, 1022 | "run_as_service_principal_id": null, 1023 | "schedule": null, 1024 | "can_edit": true 1025 | } 1026 | } 1027 | }, 1028 | { 1029 | "id": "81285102-44e4-41aa-8d53-4da9539e1f93", 1030 | "width": 1, 1031 | "options": { 1032 | "isHidden": false, 1033 | "position": { 1034 | "autoHeight": false, 1035 | "sizeX": 2, 1036 | "sizeY": 8, 1037 | "minSizeX": 1, 1038 | "maxSizeX": 6, 1039 | "minSizeY": 5, 1040 | "maxSizeY": 1000, 1041 | "col": 4, 1042 | "row": 0 1043 | }, 1044 | "parameterMappings": {} 1045 | }, 1046 | "dashboard_id": "19394330-2274-4b4b-90ce-d415a7ff2130", 1047 | "text": "", 1048 | "updated_at": "2023-01-25T13:12:52Z", 1049 | "created_at": "2023-01-25T13:12:52Z", 1050 | "visualization": { 1051 | "id": "de8a29c2-fd17-4bfc-969e-361b9c9bf1e6", 1052 | "type": "CHART", 1053 | "name": "Chart", 1054 | "description": "", 1055 | "options": { 1056 | "version": 2, 1057 | "globalSeriesType": "pie", 1058 | "sortX": true, 1059 | "sortY": true, 1060 | "legend": { 1061 | "traceorder": "normal", 1062 | "enabled": true, 1063 | "placement": "auto" 1064 | }, 1065 | "xAxis": { 1066 | "type": "-", 1067 | "labels": { 1068 | "enabled": true 1069 | }, 1070 | "title": { 1071 | "text": "gender_Male" 1072 | } 1073 | }, 1074 | "yAxis": [ 1075 | { 1076 | "type": "-", 1077 | "title": { 1078 | "text": "total_churn" 1079 | } 1080 | }, 1081 | { 1082 | "type": "-", 1083 | "opposite": true, 1084 | "title": { 1085 | "text": null 1086 | } 1087 | } 1088 | ], 1089 | "alignYAxesAtZero": false, 1090 | "error_y": { 1091 | "type": "data", 1092 | "visible": true 1093 | }, 1094 | "series": { 1095 | "stacking": null, 1096 | "error_y": { 1097 | "type": "data", 1098 | "visible": true 1099 | } 1100 | }, 1101 | "seriesOptions": { 1102 | "total_churn": { 1103 | "yAxis": 0 1104 | } 1105 | }, 1106 | "valuesOptions": { 1107 | "0": { 1108 | "color": "#F6C17F" 1109 | }, 1110 | "1": { 1111 | "color": "#AB506F" 1112 | } 1113 | }, 1114 | "direction": { 1115 | "type": "counterclockwise" 1116 | }, 1117 | "sizemode": "diameter", 1118 | "coefficient": 1, 1119 | "numberFormat": "0,0[.]00000", 1120 | "percentFormat": "0[.]00%", 1121 | "textFormat": "", 1122 | "missingValuesAsZero": true, 1123 | "useAggregationsUi": false, 1124 | "swappedAxes": false, 1125 | "dateTimeFormat": "YYYY-MM-DD HH:mm", 1126 | "showDataLabels": true, 1127 | "columnConfigurationMap": { 1128 | "x": { 1129 | "column": "gender" 1130 | }, 1131 | "y": [ 1132 | { 1133 | "column": "total_churn" 1134 | } 1135 | ], 1136 | "series": { 1137 | "column": null 1138 | } 1139 | }, 1140 | "showPlotlyControls": true, 1141 | "condensed": true, 1142 | "withRowNumber": true, 1143 | "isAggregationOn": false 1144 | }, 1145 | "updated_at": "2023-01-25T13:13:26Z", 1146 | "created_at": "2023-01-25T13:12:41Z", 1147 | "query_plan": null, 1148 | "query": { 1149 | "id": "5243b9e9-bd6d-474f-853a-8b25f4ee67d6", 1150 | "name": "Churn - Churn by Gender - Universal", 1151 | "description": null, 1152 | "query": "\nSELECT gender, count(gender) as total_churn FROM dbdemos_c360.churn_features where churn = 1 GROUP BY gender\n\n\n", 1153 | "is_draft": false, 1154 | "updated_at": "2023-01-25T13:12:41Z", 1155 | "created_at": "2023-01-25T13:12:36Z", 1156 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 1157 | "options": { 1158 | "parent": "folders/4156927442168299", 1159 | "apply_auto_limit": false, 1160 | "folder_node_status": "ACTIVE", 1161 | "folder_node_internal_name": "tree/791251337284972", 1162 | "parameters": [] 1163 | }, 1164 | "version": 1, 1165 | "tags": [], 1166 | "is_safe": true, 1167 | "user_id": 7644138420879474, 1168 | "run_as_role": null, 1169 | "run_as_service_principal_id": null, 1170 | "schedule": null, 1171 | "can_edit": true 1172 | } 1173 | } 1174 | } 1175 | ], 1176 | "options": { 1177 | "run_as_role": "viewer", 1178 | "refresh_schedules": [] 1179 | }, 1180 | "is_draft": false, 1181 | "tags": [ 1182 | "mlops_field_demos", 1183 | "field_demos", 1184 | "field_demos_retail" 1185 | ], 1186 | "updated_at": "2023-01-25T13:12:46Z", 1187 | "created_at": "2022-02-07T14:03:21Z", 1188 | "version": 351, 1189 | "color_palette": null, 1190 | "run_as_role": "viewer", 1191 | "run_as_service_principal_id": null, 1192 | "refresh_schedules": [], 1193 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 1194 | "is_favorite": true, 1195 | "user": { 1196 | "id": 7644138420879474, 1197 | "name": "Quentin Ambard", 1198 | "email": "quentin.ambard@databricks.com", 1199 | "profile_image_url": "https://www.gravatar.com/avatar/62d8f76888369585890af5b5dce9a395?s=40&d=identicon", 1200 | "is_db_admin": false 1201 | }, 1202 | "is_archived": false, 1203 | "can_edit": true, 1204 | "permission_tier": "CAN_MANAGE" 1205 | } 1206 | } -------------------------------------------------------------------------------- /test/6f73dd1b-17b1-49d0-9a11-b3772a2c3357.json: -------------------------------------------------------------------------------- 1 | { 2 | "queries": [ 3 | { 4 | "id": "89625892-ee6a-4bb7-abe6-7c40fe9c0f87", 5 | "name": "DLT-retail-data-quality-stats", 6 | "description": null, 7 | "query": "select \n date(timestamp) day, sum(failed_records)/sum(output_records)*100 failure_rate, sum(output_records) output_records from dbdemos_c360.dlt_expectations \ngroup by day order by day", 8 | "is_draft": false, 9 | "updated_at": "2023-01-17T16:11:27Z", 10 | "created_at": "2023-01-17T16:11:21Z", 11 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 12 | "options": { 13 | "parent": "folders/4156927442168299", 14 | "apply_auto_limit": false, 15 | "folder_node_status": "ACTIVE", 16 | "folder_node_internal_name": "tree/4156927442757238", 17 | "parameters": [] 18 | }, 19 | "version": 1, 20 | "tags": [], 21 | "is_safe": true, 22 | "user_id": 7644138420879474, 23 | "run_as_role": null, 24 | "run_as_service_principal_id": null, 25 | "schedule": null, 26 | "latest_query_data_id": "b771ddcf-6a35-4fbc-a889-95c75b4f122c", 27 | "visualizations": [ 28 | { 29 | "id": "0682a37e-d167-4b14-8847-e2d774b6a5eb", 30 | "type": "TABLE", 31 | "name": "Table", 32 | "description": "", 33 | "options": {}, 34 | "updated_at": "2023-01-17T16:11:24Z", 35 | "created_at": "2023-01-17T16:11:24Z", 36 | "query_plan": null 37 | }, 38 | { 39 | "id": "53c6c91c-fbfe-4e5c-96a7-d77997737c75", 40 | "type": "COUNTER", 41 | "name": "Daily row ingested", 42 | "description": "", 43 | "options": { 44 | "counterLabel": "Daily row ingested", 45 | "counterColName": "output_records", 46 | "rowNumber": 1, 47 | "targetRowNumber": 1, 48 | "stringDecimal": 0, 49 | "stringDecChar": ".", 50 | "stringThouSep": ",", 51 | "tooltipFormat": "0,0.000", 52 | "showPlotlyControls": true 53 | }, 54 | "updated_at": "2023-01-17T16:14:19Z", 55 | "created_at": "2023-01-17T16:11:27Z", 56 | "query_plan": null 57 | }, 58 | { 59 | "id": "a50cddc1-34fa-4213-8d74-c6234f1d7f28", 60 | "type": "TABLE", 61 | "name": "Table", 62 | "description": "", 63 | "options": {}, 64 | "updated_at": "2023-01-17T16:11:24Z", 65 | "created_at": "2023-01-17T16:11:21Z", 66 | "query_plan": null 67 | }, 68 | { 69 | "id": "af612aa6-e95b-42f2-818c-bb3329163499", 70 | "type": "COUNTER", 71 | "name": "Invalid input schema", 72 | "description": "", 73 | "options": { 74 | "counterLabel": "Invalid input schema", 75 | "counterColName": "failure_rate", 76 | "rowNumber": 1, 77 | "targetRowNumber": 1, 78 | "stringDecimal": 2, 79 | "stringDecChar": ".", 80 | "stringThouSep": ",", 81 | "tooltipFormat": "0,0.000", 82 | "useAggregationsUi": false, 83 | "showPlotlyControls": true, 84 | "stringPrefix": "", 85 | "stringSuffix": "%" 86 | }, 87 | "updated_at": "2023-01-17T16:14:19Z", 88 | "created_at": "2023-01-17T16:11:26Z", 89 | "query_plan": null 90 | }, 91 | { 92 | "id": "fbb4a557-2450-40d9-b155-1bb9a2cb0bec", 93 | "type": "COUNTER", 94 | "name": "Quality failure rate", 95 | "description": "", 96 | "options": { 97 | "counterLabel": "Quality Failure rate", 98 | "counterColName": "failure_rate", 99 | "rowNumber": 11, 100 | "targetRowNumber": 1, 101 | "stringDecimal": 2, 102 | "stringDecChar": ".", 103 | "stringThouSep": ",", 104 | "tooltipFormat": "0,0.000", 105 | "showPlotlyControls": true, 106 | "stringPrefix": "", 107 | "stringSuffix": "%", 108 | "formatTargetValue": true, 109 | "targetColName": "" 110 | }, 111 | "updated_at": "2023-01-17T16:11:49Z", 112 | "created_at": "2023-01-17T16:11:25Z", 113 | "query_plan": null 114 | } 115 | ], 116 | "is_favorite": false, 117 | "last_modified_by": { 118 | "id": 7644138420879474, 119 | "name": "Quentin Ambard", 120 | "email": "quentin.ambard@databricks.com", 121 | "profile_image_url": "https://www.gravatar.com/avatar/62d8f76888369585890af5b5dce9a395?s=40&d=identicon", 122 | "is_db_admin": false 123 | }, 124 | "query_draft": null, 125 | "job_id": null, 126 | "user": { 127 | "id": 7644138420879474, 128 | "name": "Quentin Ambard", 129 | "email": "quentin.ambard@databricks.com", 130 | "profile_image_url": "https://www.gravatar.com/avatar/62d8f76888369585890af5b5dce9a395?s=40&d=identicon", 131 | "is_db_admin": false 132 | }, 133 | "parent": "folders/4156927442168299", 134 | "is_archived": false, 135 | "can_edit": true, 136 | "permission_tier": "CAN_MANAGE" 137 | }, 138 | { 139 | "id": "a71a57a5-a118-4bda-a651-a2600c25f771", 140 | "name": "DLT-retail-data-quality", 141 | "description": null, 142 | "query": "select sum(passed_records) as passed_records,\n sum(failed_records) as failed_records,\n sum(dropped_records) as dropped_records,\n sum(output_records) as output_records,\n sum(failed_records)/sum(output_records)*100 as failure_rate,\n name,\n dataset,\n date(timestamp) as date from dbdemos_c360.dlt_expectations values group by date, dataset, name;", 143 | "is_draft": false, 144 | "updated_at": "2023-01-17T16:11:32Z", 145 | "created_at": "2023-01-17T16:11:27Z", 146 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 147 | "options": { 148 | "parent": "folders/4156927442168299", 149 | "apply_auto_limit": false, 150 | "folder_node_status": "ACTIVE", 151 | "folder_node_internal_name": "tree/4156927442757239", 152 | "parameters": [] 153 | }, 154 | "version": 1, 155 | "tags": [], 156 | "is_safe": true, 157 | "user_id": 7644138420879474, 158 | "run_as_role": null, 159 | "run_as_service_principal_id": null, 160 | "schedule": null, 161 | "latest_query_data_id": "3ade8027-5a84-4eb3-8658-fa387157bf12", 162 | "visualizations": [ 163 | { 164 | "id": "1f5a8079-0f5d-4e58-8ad1-a817008c4f93", 165 | "type": "CHART", 166 | "name": "Daily failure per dataset", 167 | "description": "", 168 | "options": { 169 | "version": 2, 170 | "globalSeriesType": "column", 171 | "sortX": true, 172 | "legend": { 173 | "enabled": true, 174 | "placement": "auto", 175 | "traceorder": "normal" 176 | }, 177 | "xAxis": { 178 | "type": "-", 179 | "labels": { 180 | "enabled": true 181 | }, 182 | "title": { 183 | "text": "date" 184 | } 185 | }, 186 | "yAxis": [ 187 | { 188 | "type": "-", 189 | "title": { 190 | "text": "failed_records" 191 | } 192 | }, 193 | { 194 | "type": "-", 195 | "opposite": true, 196 | "title": { 197 | "text": null 198 | } 199 | } 200 | ], 201 | "alignYAxesAtZero": false, 202 | "error_y": { 203 | "type": "data", 204 | "visible": true 205 | }, 206 | "series": { 207 | "stacking": "stack", 208 | "error_y": { 209 | "type": "data", 210 | "visible": true 211 | } 212 | }, 213 | "seriesOptions": { 214 | "failed_records": { 215 | "yAxis": 0 216 | }, 217 | "user_gold_dlt": { 218 | "color": "#F58742" 219 | }, 220 | "spend_silver_dlt": { 221 | "color": "#F5C61B" 222 | }, 223 | "user_silver_dlt": { 224 | "color": "#A58AFF" 225 | }, 226 | "users_bronze_dlt": { 227 | "color": "#C63FA9", 228 | "type": "column" 229 | }, 230 | "valid_score": { 231 | "color": "#C44427" 232 | }, 233 | "valid_id": { 234 | "color": "#D67C1C" 235 | }, 236 | "valid_age": { 237 | "color": "#D6C31C" 238 | }, 239 | "correct_schema": { 240 | "color": "#A58AFF" 241 | }, 242 | "valid_income": { 243 | "color": "#C63FA9" 244 | } 245 | }, 246 | "valuesOptions": {}, 247 | "direction": { 248 | "type": "counterclockwise" 249 | }, 250 | "sizemode": "diameter", 251 | "coefficient": 1, 252 | "numberFormat": "0,0[.]00000", 253 | "percentFormat": "0[.]00%", 254 | "textFormat": "", 255 | "missingValuesAsZero": true, 256 | "useAggregationsUi": false, 257 | "swappedAxes": false, 258 | "showDataLabels": false, 259 | "dateTimeFormat": "YYYY-MM-DD HH:mm", 260 | "columnConfigurationMap": { 261 | "x": { 262 | "column": "date" 263 | }, 264 | "y": [ 265 | { 266 | "column": "failed_records" 267 | } 268 | ], 269 | "series": { 270 | "column": "name" 271 | } 272 | }, 273 | "showPlotlyControls": true 274 | }, 275 | "updated_at": "2023-01-17T16:14:19Z", 276 | "created_at": "2023-01-17T16:11:32Z", 277 | "query_plan": null 278 | }, 279 | { 280 | "id": "2b36c2b7-3a73-4759-b5fc-143ae496fc71", 281 | "type": "TABLE", 282 | "name": "Table", 283 | "description": "", 284 | "options": {}, 285 | "updated_at": "2023-01-17T16:11:31Z", 286 | "created_at": "2023-01-17T16:11:31Z", 287 | "query_plan": null 288 | }, 289 | { 290 | "id": "9ab27851-42c2-4eda-b3c5-fc4d9e7e2cf3", 291 | "type": "TABLE", 292 | "name": "Table", 293 | "description": "", 294 | "options": {}, 295 | "updated_at": "2023-01-17T16:11:30Z", 296 | "created_at": "2023-01-17T16:11:27Z", 297 | "query_plan": null 298 | }, 299 | { 300 | "id": "fc3d291c-9dea-45db-a2bb-a0746d6e5fb9", 301 | "type": "CHART", 302 | "name": "Customer pipeline data quality", 303 | "description": "", 304 | "options": { 305 | "version": 2, 306 | "globalSeriesType": "line", 307 | "sortX": true, 308 | "legend": { 309 | "enabled": true, 310 | "placement": "auto", 311 | "traceorder": "normal" 312 | }, 313 | "xAxis": { 314 | "type": "-", 315 | "labels": { 316 | "enabled": true 317 | }, 318 | "title": { 319 | "text": "date" 320 | } 321 | }, 322 | "yAxis": [ 323 | { 324 | "type": "-", 325 | "title": { 326 | "text": "dropped_records" 327 | } 328 | }, 329 | { 330 | "type": "-", 331 | "opposite": true, 332 | "title": { 333 | "text": "passed_records" 334 | } 335 | } 336 | ], 337 | "alignYAxesAtZero": false, 338 | "error_y": { 339 | "type": "data", 340 | "visible": true 341 | }, 342 | "series": { 343 | "stacking": null, 344 | "error_y": { 345 | "type": "data", 346 | "visible": true 347 | } 348 | }, 349 | "seriesOptions": { 350 | "dropped_records": { 351 | "yAxis": 0, 352 | "type": "area" 353 | }, 354 | "passed_records": { 355 | "yAxis": 1, 356 | "type": "area" 357 | } 358 | }, 359 | "valuesOptions": {}, 360 | "direction": { 361 | "type": "counterclockwise" 362 | }, 363 | "sizemode": "diameter", 364 | "coefficient": 1, 365 | "numberFormat": "0,0[.]00000", 366 | "percentFormat": "0[.]00%", 367 | "textFormat": "", 368 | "missingValuesAsZero": true, 369 | "useAggregationsUi": false, 370 | "showDataLabels": false, 371 | "dateTimeFormat": "YYYY-MM-DD HH:mm", 372 | "columnConfigurationMap": { 373 | "x": { 374 | "column": "date" 375 | }, 376 | "y": [ 377 | { 378 | "column": "dropped_records" 379 | }, 380 | { 381 | "column": "passed_records" 382 | } 383 | ] 384 | }, 385 | "showPlotlyControls": true, 386 | "swappedAxes": false 387 | }, 388 | "updated_at": "2023-01-17T16:14:19Z", 389 | "created_at": "2023-01-17T16:11:31Z", 390 | "query_plan": null 391 | } 392 | ], 393 | "is_favorite": false, 394 | "last_modified_by": { 395 | "id": 7644138420879474, 396 | "name": "Quentin Ambard", 397 | "email": "quentin.ambard@databricks.com", 398 | "profile_image_url": "https://www.gravatar.com/avatar/62d8f76888369585890af5b5dce9a395?s=40&d=identicon", 399 | "is_db_admin": false 400 | }, 401 | "query_draft": null, 402 | "job_id": null, 403 | "user": { 404 | "id": 7644138420879474, 405 | "name": "Quentin Ambard", 406 | "email": "quentin.ambard@databricks.com", 407 | "profile_image_url": "https://www.gravatar.com/avatar/62d8f76888369585890af5b5dce9a395?s=40&d=identicon", 408 | "is_db_admin": false 409 | }, 410 | "parent": "folders/4156927442168299", 411 | "is_archived": false, 412 | "can_edit": true, 413 | "permission_tier": "CAN_MANAGE" 414 | }, 415 | { 416 | "id": "d282c253-dc4c-469a-9190-8cc170bd3ebb", 417 | "name": "DLT-retail-data-quality-per-table", 418 | "description": null, 419 | "query": "select 'passed_records' as type,\n sum(passed_records) as value, \n sum(failed_records)/sum(output_records)*100 as failure_rate,\n dataset \n from field_demos_retail.dlt_expectations group by dataset\nunion\nselect 'failed_records' as type,\n sum(failed_records) as value, \n sum(failed_records)/sum(output_records)*100 as failure_rate,\n dataset \n from field_demos_retail.dlt_expectations group by dataset\nunion\nselect 'dropped_records' as type,\n sum(dropped_records) as value, \n sum(failed_records)/sum(output_records)*100 as failure_rate,\n dataset \n from dbdemos_c360.dlt_expectations group by dataset\n", 420 | "is_draft": false, 421 | "updated_at": "2023-01-17T16:11:38Z", 422 | "created_at": "2023-01-17T16:11:33Z", 423 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 424 | "options": { 425 | "parent": "folders/4156927442168299", 426 | "apply_auto_limit": false, 427 | "folder_node_status": "ACTIVE", 428 | "folder_node_internal_name": "tree/4156927442757241", 429 | "parameters": [] 430 | }, 431 | "version": 1, 432 | "tags": [], 433 | "is_safe": true, 434 | "user_id": 7644138420879474, 435 | "run_as_role": null, 436 | "run_as_service_principal_id": null, 437 | "schedule": null, 438 | "latest_query_data_id": "22ede635-47fb-429b-a265-141b034bdd74", 439 | "visualizations": [ 440 | { 441 | "id": "0e83be66-8d97-4ae1-9990-576b3f925c24", 442 | "type": "TABLE", 443 | "name": "Table", 444 | "description": "", 445 | "options": {}, 446 | "updated_at": "2023-01-17T16:11:37Z", 447 | "created_at": "2023-01-17T16:11:37Z", 448 | "query_plan": null 449 | }, 450 | { 451 | "id": "508f2a52-d42a-443d-bf01-3198cfca1d91", 452 | "type": "TABLE", 453 | "name": "Table", 454 | "description": "", 455 | "options": {}, 456 | "updated_at": "2023-01-17T16:11:36Z", 457 | "created_at": "2023-01-17T16:11:33Z", 458 | "query_plan": null 459 | }, 460 | { 461 | "id": "bae5c711-7d5a-4567-9326-ad324ce5de76", 462 | "type": "CHART", 463 | "name": "Status per expectation", 464 | "description": "", 465 | "options": { 466 | "version": 2, 467 | "globalSeriesType": "pie", 468 | "sortX": true, 469 | "legend": { 470 | "enabled": true, 471 | "placement": "auto", 472 | "traceorder": "normal" 473 | }, 474 | "xAxis": { 475 | "type": "-", 476 | "labels": { 477 | "enabled": true 478 | }, 479 | "title": { 480 | "text": "type" 481 | } 482 | }, 483 | "yAxis": [ 484 | { 485 | "type": "-", 486 | "title": { 487 | "text": "value" 488 | } 489 | }, 490 | { 491 | "type": "-", 492 | "opposite": true, 493 | "title": { 494 | "text": null 495 | } 496 | } 497 | ], 498 | "alignYAxesAtZero": false, 499 | "error_y": { 500 | "type": "data", 501 | "visible": true 502 | }, 503 | "series": { 504 | "stacking": null, 505 | "error_y": { 506 | "type": "data", 507 | "visible": true 508 | } 509 | }, 510 | "seriesOptions": { 511 | "value": { 512 | "yAxis": 0 513 | } 514 | }, 515 | "valuesOptions": { 516 | "dropped_records": { 517 | "color": "#E92828" 518 | }, 519 | "passed_records": { 520 | "color": "#799CFF" 521 | }, 522 | "failed_records": { 523 | "color": "#FB8D3D" 524 | } 525 | }, 526 | "direction": { 527 | "type": "counterclockwise" 528 | }, 529 | "sizemode": "diameter", 530 | "coefficient": 1, 531 | "numberFormat": "0,0[.]00000", 532 | "percentFormat": "0[.]00%", 533 | "textFormat": "", 534 | "missingValuesAsZero": true, 535 | "useAggregationsUi": false, 536 | "showDataLabels": true, 537 | "dateTimeFormat": "YYYY-MM-DD HH:mm", 538 | "columnConfigurationMap": { 539 | "x": { 540 | "column": "type" 541 | }, 542 | "y": [ 543 | { 544 | "column": "value" 545 | } 546 | ], 547 | "series": { 548 | "column": "dataset" 549 | } 550 | }, 551 | "showPlotlyControls": true, 552 | "swappedAxes": false 553 | }, 554 | "updated_at": "2023-01-17T16:14:19Z", 555 | "created_at": "2023-01-17T16:11:38Z", 556 | "query_plan": null 557 | } 558 | ], 559 | "is_favorite": false, 560 | "last_modified_by": { 561 | "id": 7644138420879474, 562 | "name": "Quentin Ambard", 563 | "email": "quentin.ambard@databricks.com", 564 | "profile_image_url": "https://www.gravatar.com/avatar/62d8f76888369585890af5b5dce9a395?s=40&d=identicon", 565 | "is_db_admin": false 566 | }, 567 | "query_draft": null, 568 | "job_id": null, 569 | "user": { 570 | "id": 7644138420879474, 571 | "name": "Quentin Ambard", 572 | "email": "quentin.ambard@databricks.com", 573 | "profile_image_url": "https://www.gravatar.com/avatar/62d8f76888369585890af5b5dce9a395?s=40&d=identicon", 574 | "is_db_admin": false 575 | }, 576 | "parent": "folders/4156927442168299", 577 | "is_archived": false, 578 | "can_edit": true, 579 | "permission_tier": "CAN_MANAGE" 580 | } 581 | ], 582 | "id": "6f73dd1b-17b1-49d0-9a11-b3772a2c3357", 583 | "dashboard": { 584 | "id": "6f73dd1b-17b1-49d0-9a11-b3772a2c3357", 585 | "slug": "dlt---retail-data-quality-stats", 586 | "name": "DLT - Retail Data Quality Stats", 587 | "user_id": 7644138420879474, 588 | "dashboard_filters_enabled": false, 589 | "widgets": [ 590 | { 591 | "id": "731ac691-ab61-4d61-92cc-6da9fbe01c36", 592 | "width": 1, 593 | "options": { 594 | "isHidden": false, 595 | "position": { 596 | "autoHeight": false, 597 | "sizeX": 6, 598 | "sizeY": 4, 599 | "minSizeX": 1, 600 | "maxSizeX": 6, 601 | "minSizeY": 1, 602 | "maxSizeY": 1000, 603 | "col": 0, 604 | "row": 0 605 | } 606 | }, 607 | "dashboard_id": "6f73dd1b-17b1-49d0-9a11-b3772a2c3357", 608 | "text": "# Retail ingestion quality tracker\nThis dashboard leverage the Delta Live Table expectation metrics to track our data quality over the ingestion pipeline.\n\nThe pipeline is incrementally consuming new data (each hour or in real time) and the dashboard is updated accordingly. Open notebook `02.1-Delta-Live-Table-Ingestion` to visualize the Delta Live Table pipeline.\n\nData has been prepared using a SQL query over the value from the `/system/events` table. Open `02.3-DLT-expectation-dashboard-data-prep` for more details.", 609 | "updated_at": "2023-01-17T16:11:46Z", 610 | "created_at": "2023-01-17T16:11:46Z" 611 | }, 612 | { 613 | "id": "ccfccdd5-1cd0-4886-97d3-42fc1a4a1f70", 614 | "width": 1, 615 | "options": { 616 | "parameterMappings": {}, 617 | "title": "Ingestion", 618 | "description": "", 619 | "isHidden": false, 620 | "position": { 621 | "autoHeight": false, 622 | "sizeX": 2, 623 | "sizeY": 4, 624 | "minSizeX": 1, 625 | "maxSizeX": 6, 626 | "minSizeY": 1, 627 | "maxSizeY": 1000, 628 | "col": 0, 629 | "row": 4 630 | } 631 | }, 632 | "dashboard_id": "6f73dd1b-17b1-49d0-9a11-b3772a2c3357", 633 | "text": "", 634 | "updated_at": "2023-01-17T16:11:47Z", 635 | "created_at": "2023-01-17T16:11:47Z", 636 | "visualization": { 637 | "id": "53c6c91c-fbfe-4e5c-96a7-d77997737c75", 638 | "type": "COUNTER", 639 | "name": "Daily row ingested", 640 | "description": "", 641 | "options": { 642 | "counterLabel": "Daily row ingested", 643 | "counterColName": "output_records", 644 | "rowNumber": 1, 645 | "targetRowNumber": 1, 646 | "stringDecimal": 0, 647 | "stringDecChar": ".", 648 | "stringThouSep": ",", 649 | "tooltipFormat": "0,0.000", 650 | "showPlotlyControls": true 651 | }, 652 | "updated_at": "2023-01-17T16:14:19Z", 653 | "created_at": "2023-01-17T16:11:27Z", 654 | "query_plan": null, 655 | "query": { 656 | "id": "89625892-ee6a-4bb7-abe6-7c40fe9c0f87", 657 | "name": "DLT-retail-data-quality-stats", 658 | "description": null, 659 | "query": "select \n date(timestamp) day, sum(failed_records)/sum(output_records)*100 failure_rate, sum(output_records) output_records from dbdemos_c360.dlt_expectations \ngroup by day order by day", 660 | "is_draft": false, 661 | "updated_at": "2023-01-17T16:11:27Z", 662 | "created_at": "2023-01-17T16:11:21Z", 663 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 664 | "options": { 665 | "parent": "folders/4156927442168299", 666 | "apply_auto_limit": false, 667 | "folder_node_status": "ACTIVE", 668 | "folder_node_internal_name": "tree/4156927442757238", 669 | "parameters": [] 670 | }, 671 | "version": 1, 672 | "tags": [], 673 | "is_safe": true, 674 | "user_id": 7644138420879474, 675 | "run_as_role": null, 676 | "run_as_service_principal_id": null, 677 | "schedule": null, 678 | "can_edit": true 679 | } 680 | } 681 | }, 682 | { 683 | "id": "cfa813b6-b6e9-4e6f-8244-83744e5fc885", 684 | "width": 1, 685 | "options": { 686 | "parameterMappings": {}, 687 | "title": "Daily ingestion error per expectations\n", 688 | "description": "", 689 | "isHidden": false, 690 | "position": { 691 | "autoHeight": false, 692 | "sizeX": 6, 693 | "sizeY": 7, 694 | "minSizeX": 1, 695 | "maxSizeX": 6, 696 | "minSizeY": 5, 697 | "maxSizeY": 1000, 698 | "col": 0, 699 | "row": 16 700 | } 701 | }, 702 | "dashboard_id": "6f73dd1b-17b1-49d0-9a11-b3772a2c3357", 703 | "text": "", 704 | "updated_at": "2023-01-17T16:11:48Z", 705 | "created_at": "2023-01-17T16:11:48Z", 706 | "visualization": { 707 | "id": "1f5a8079-0f5d-4e58-8ad1-a817008c4f93", 708 | "type": "CHART", 709 | "name": "Daily failure per dataset", 710 | "description": "", 711 | "options": { 712 | "version": 2, 713 | "globalSeriesType": "column", 714 | "sortX": true, 715 | "legend": { 716 | "enabled": true, 717 | "placement": "auto", 718 | "traceorder": "normal" 719 | }, 720 | "xAxis": { 721 | "type": "-", 722 | "labels": { 723 | "enabled": true 724 | }, 725 | "title": { 726 | "text": "date" 727 | } 728 | }, 729 | "yAxis": [ 730 | { 731 | "type": "-", 732 | "title": { 733 | "text": "failed_records" 734 | } 735 | }, 736 | { 737 | "type": "-", 738 | "opposite": true, 739 | "title": { 740 | "text": null 741 | } 742 | } 743 | ], 744 | "alignYAxesAtZero": false, 745 | "error_y": { 746 | "type": "data", 747 | "visible": true 748 | }, 749 | "series": { 750 | "stacking": "stack", 751 | "error_y": { 752 | "type": "data", 753 | "visible": true 754 | } 755 | }, 756 | "seriesOptions": { 757 | "failed_records": { 758 | "yAxis": 0 759 | }, 760 | "user_gold_dlt": { 761 | "color": "#F58742" 762 | }, 763 | "spend_silver_dlt": { 764 | "color": "#F5C61B" 765 | }, 766 | "user_silver_dlt": { 767 | "color": "#A58AFF" 768 | }, 769 | "users_bronze_dlt": { 770 | "color": "#C63FA9", 771 | "type": "column" 772 | }, 773 | "valid_score": { 774 | "color": "#C44427" 775 | }, 776 | "valid_id": { 777 | "color": "#D67C1C" 778 | }, 779 | "valid_age": { 780 | "color": "#D6C31C" 781 | }, 782 | "correct_schema": { 783 | "color": "#A58AFF" 784 | }, 785 | "valid_income": { 786 | "color": "#C63FA9" 787 | } 788 | }, 789 | "valuesOptions": {}, 790 | "direction": { 791 | "type": "counterclockwise" 792 | }, 793 | "sizemode": "diameter", 794 | "coefficient": 1, 795 | "numberFormat": "0,0[.]00000", 796 | "percentFormat": "0[.]00%", 797 | "textFormat": "", 798 | "missingValuesAsZero": true, 799 | "useAggregationsUi": false, 800 | "swappedAxes": false, 801 | "showDataLabels": false, 802 | "dateTimeFormat": "YYYY-MM-DD HH:mm", 803 | "columnConfigurationMap": { 804 | "x": { 805 | "column": "date" 806 | }, 807 | "y": [ 808 | { 809 | "column": "failed_records" 810 | } 811 | ], 812 | "series": { 813 | "column": "name" 814 | } 815 | }, 816 | "showPlotlyControls": true 817 | }, 818 | "updated_at": "2023-01-17T16:14:19Z", 819 | "created_at": "2023-01-17T16:11:32Z", 820 | "query_plan": null, 821 | "query": { 822 | "id": "a71a57a5-a118-4bda-a651-a2600c25f771", 823 | "name": "DLT-retail-data-quality", 824 | "description": null, 825 | "query": "select sum(passed_records) as passed_records,\n sum(failed_records) as failed_records,\n sum(dropped_records) as dropped_records,\n sum(output_records) as output_records,\n sum(failed_records)/sum(output_records)*100 as failure_rate,\n name,\n dataset,\n date(timestamp) as date from dbdemos_c360.dlt_expectations values group by date, dataset, name;", 826 | "is_draft": false, 827 | "updated_at": "2023-01-17T16:11:32Z", 828 | "created_at": "2023-01-17T16:11:27Z", 829 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 830 | "options": { 831 | "parent": "folders/4156927442168299", 832 | "apply_auto_limit": false, 833 | "folder_node_status": "ACTIVE", 834 | "folder_node_internal_name": "tree/4156927442757239", 835 | "parameters": [] 836 | }, 837 | "version": 1, 838 | "tags": [], 839 | "is_safe": true, 840 | "user_id": 7644138420879474, 841 | "run_as_role": null, 842 | "run_as_service_principal_id": null, 843 | "schedule": null, 844 | "can_edit": true 845 | } 846 | } 847 | }, 848 | { 849 | "id": "9aabbf58-0497-4489-9e15-b18b571e77b0", 850 | "width": 1, 851 | "options": { 852 | "parameterMappings": {}, 853 | "title": "Invalid data", 854 | "description": "", 855 | "isHidden": false, 856 | "position": { 857 | "autoHeight": false, 858 | "sizeX": 2, 859 | "sizeY": 4, 860 | "minSizeX": 1, 861 | "maxSizeX": 6, 862 | "minSizeY": 1, 863 | "maxSizeY": 1000, 864 | "col": 2, 865 | "row": 4 866 | } 867 | }, 868 | "dashboard_id": "6f73dd1b-17b1-49d0-9a11-b3772a2c3357", 869 | "text": "", 870 | "updated_at": "2023-01-17T16:11:49Z", 871 | "created_at": "2023-01-17T16:11:49Z", 872 | "visualization": { 873 | "id": "fbb4a557-2450-40d9-b155-1bb9a2cb0bec", 874 | "type": "COUNTER", 875 | "name": "Quality failure rate", 876 | "description": "", 877 | "options": { 878 | "counterLabel": "Quality Failure rate", 879 | "counterColName": "failure_rate", 880 | "rowNumber": 11, 881 | "targetRowNumber": 1, 882 | "stringDecimal": 2, 883 | "stringDecChar": ".", 884 | "stringThouSep": ",", 885 | "tooltipFormat": "0,0.000", 886 | "showPlotlyControls": true, 887 | "stringPrefix": "", 888 | "stringSuffix": "%", 889 | "formatTargetValue": true, 890 | "targetColName": "" 891 | }, 892 | "updated_at": "2023-01-17T16:11:49Z", 893 | "created_at": "2023-01-17T16:11:25Z", 894 | "query_plan": null, 895 | "query": { 896 | "id": "89625892-ee6a-4bb7-abe6-7c40fe9c0f87", 897 | "name": "DLT-retail-data-quality-stats", 898 | "description": null, 899 | "query": "select \n date(timestamp) day, sum(failed_records)/sum(output_records)*100 failure_rate, sum(output_records) output_records from dbdemos_c360.dlt_expectations \ngroup by day order by day", 900 | "is_draft": false, 901 | "updated_at": "2023-01-17T16:11:27Z", 902 | "created_at": "2023-01-17T16:11:21Z", 903 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 904 | "options": { 905 | "parent": "folders/4156927442168299", 906 | "apply_auto_limit": false, 907 | "folder_node_status": "ACTIVE", 908 | "folder_node_internal_name": "tree/4156927442757238", 909 | "parameters": [] 910 | }, 911 | "version": 1, 912 | "tags": [], 913 | "is_safe": true, 914 | "user_id": 7644138420879474, 915 | "run_as_role": null, 916 | "run_as_service_principal_id": null, 917 | "schedule": null, 918 | "can_edit": true 919 | } 920 | } 921 | }, 922 | { 923 | "id": "c164dd9a-4bc9-4ba0-8851-16ad1e090ed3", 924 | "width": 1, 925 | "options": { 926 | "parameterMappings": {}, 927 | "title": "Quality stat per table", 928 | "description": "", 929 | "isHidden": false, 930 | "position": { 931 | "autoHeight": false, 932 | "sizeX": 3, 933 | "sizeY": 8, 934 | "minSizeX": 1, 935 | "maxSizeX": 6, 936 | "minSizeY": 5, 937 | "maxSizeY": 1000, 938 | "col": 3, 939 | "row": 8 940 | } 941 | }, 942 | "dashboard_id": "6f73dd1b-17b1-49d0-9a11-b3772a2c3357", 943 | "text": "", 944 | "updated_at": "2023-01-17T16:11:50Z", 945 | "created_at": "2023-01-17T16:11:50Z", 946 | "visualization": { 947 | "id": "bae5c711-7d5a-4567-9326-ad324ce5de76", 948 | "type": "CHART", 949 | "name": "Status per expectation", 950 | "description": "", 951 | "options": { 952 | "version": 2, 953 | "globalSeriesType": "pie", 954 | "sortX": true, 955 | "legend": { 956 | "enabled": true, 957 | "placement": "auto", 958 | "traceorder": "normal" 959 | }, 960 | "xAxis": { 961 | "type": "-", 962 | "labels": { 963 | "enabled": true 964 | }, 965 | "title": { 966 | "text": "type" 967 | } 968 | }, 969 | "yAxis": [ 970 | { 971 | "type": "-", 972 | "title": { 973 | "text": "value" 974 | } 975 | }, 976 | { 977 | "type": "-", 978 | "opposite": true, 979 | "title": { 980 | "text": null 981 | } 982 | } 983 | ], 984 | "alignYAxesAtZero": false, 985 | "error_y": { 986 | "type": "data", 987 | "visible": true 988 | }, 989 | "series": { 990 | "stacking": null, 991 | "error_y": { 992 | "type": "data", 993 | "visible": true 994 | } 995 | }, 996 | "seriesOptions": { 997 | "value": { 998 | "yAxis": 0 999 | } 1000 | }, 1001 | "valuesOptions": { 1002 | "dropped_records": { 1003 | "color": "#E92828" 1004 | }, 1005 | "passed_records": { 1006 | "color": "#799CFF" 1007 | }, 1008 | "failed_records": { 1009 | "color": "#FB8D3D" 1010 | } 1011 | }, 1012 | "direction": { 1013 | "type": "counterclockwise" 1014 | }, 1015 | "sizemode": "diameter", 1016 | "coefficient": 1, 1017 | "numberFormat": "0,0[.]00000", 1018 | "percentFormat": "0[.]00%", 1019 | "textFormat": "", 1020 | "missingValuesAsZero": true, 1021 | "useAggregationsUi": false, 1022 | "showDataLabels": true, 1023 | "dateTimeFormat": "YYYY-MM-DD HH:mm", 1024 | "columnConfigurationMap": { 1025 | "x": { 1026 | "column": "type" 1027 | }, 1028 | "y": [ 1029 | { 1030 | "column": "value" 1031 | } 1032 | ], 1033 | "series": { 1034 | "column": "dataset" 1035 | } 1036 | }, 1037 | "showPlotlyControls": true, 1038 | "swappedAxes": false 1039 | }, 1040 | "updated_at": "2023-01-17T16:14:19Z", 1041 | "created_at": "2023-01-17T16:11:38Z", 1042 | "query_plan": null, 1043 | "query": { 1044 | "id": "d282c253-dc4c-469a-9190-8cc170bd3ebb", 1045 | "name": "DLT-retail-data-quality-per-table", 1046 | "description": null, 1047 | "query": "select 'passed_records' as type,\n sum(passed_records) as value, \n sum(failed_records)/sum(output_records)*100 as failure_rate,\n dataset \n from field_demos_retail.dlt_expectations group by dataset\nunion\nselect 'failed_records' as type,\n sum(failed_records) as value, \n sum(failed_records)/sum(output_records)*100 as failure_rate,\n dataset \n from field_demos_retail.dlt_expectations group by dataset\nunion\nselect 'dropped_records' as type,\n sum(dropped_records) as value, \n sum(failed_records)/sum(output_records)*100 as failure_rate,\n dataset \n from dbdemos_c360.dlt_expectations group by dataset\n", 1048 | "is_draft": false, 1049 | "updated_at": "2023-01-17T16:11:38Z", 1050 | "created_at": "2023-01-17T16:11:33Z", 1051 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 1052 | "options": { 1053 | "parent": "folders/4156927442168299", 1054 | "apply_auto_limit": false, 1055 | "folder_node_status": "ACTIVE", 1056 | "folder_node_internal_name": "tree/4156927442757241", 1057 | "parameters": [] 1058 | }, 1059 | "version": 1, 1060 | "tags": [], 1061 | "is_safe": true, 1062 | "user_id": 7644138420879474, 1063 | "run_as_role": null, 1064 | "run_as_service_principal_id": null, 1065 | "schedule": null, 1066 | "can_edit": true 1067 | } 1068 | } 1069 | }, 1070 | { 1071 | "id": "58bb65fc-4f60-48a2-b20c-31d5b99e6aa2", 1072 | "width": 1, 1073 | "options": { 1074 | "parameterMappings": {}, 1075 | "title": "Daily ingestion and failure", 1076 | "description": "", 1077 | "isHidden": false, 1078 | "position": { 1079 | "autoHeight": false, 1080 | "sizeX": 3, 1081 | "sizeY": 8, 1082 | "minSizeX": 1, 1083 | "maxSizeX": 6, 1084 | "minSizeY": 5, 1085 | "maxSizeY": 1000, 1086 | "col": 0, 1087 | "row": 8 1088 | } 1089 | }, 1090 | "dashboard_id": "6f73dd1b-17b1-49d0-9a11-b3772a2c3357", 1091 | "text": "", 1092 | "updated_at": "2023-01-17T16:11:51Z", 1093 | "created_at": "2023-01-17T16:11:51Z", 1094 | "visualization": { 1095 | "id": "fc3d291c-9dea-45db-a2bb-a0746d6e5fb9", 1096 | "type": "CHART", 1097 | "name": "Customer pipeline data quality", 1098 | "description": "", 1099 | "options": { 1100 | "version": 2, 1101 | "globalSeriesType": "line", 1102 | "sortX": true, 1103 | "legend": { 1104 | "enabled": true, 1105 | "placement": "auto", 1106 | "traceorder": "normal" 1107 | }, 1108 | "xAxis": { 1109 | "type": "-", 1110 | "labels": { 1111 | "enabled": true 1112 | }, 1113 | "title": { 1114 | "text": "date" 1115 | } 1116 | }, 1117 | "yAxis": [ 1118 | { 1119 | "type": "-", 1120 | "title": { 1121 | "text": "dropped_records" 1122 | } 1123 | }, 1124 | { 1125 | "type": "-", 1126 | "opposite": true, 1127 | "title": { 1128 | "text": "passed_records" 1129 | } 1130 | } 1131 | ], 1132 | "alignYAxesAtZero": false, 1133 | "error_y": { 1134 | "type": "data", 1135 | "visible": true 1136 | }, 1137 | "series": { 1138 | "stacking": null, 1139 | "error_y": { 1140 | "type": "data", 1141 | "visible": true 1142 | } 1143 | }, 1144 | "seriesOptions": { 1145 | "dropped_records": { 1146 | "yAxis": 0, 1147 | "type": "area" 1148 | }, 1149 | "passed_records": { 1150 | "yAxis": 1, 1151 | "type": "area" 1152 | } 1153 | }, 1154 | "valuesOptions": {}, 1155 | "direction": { 1156 | "type": "counterclockwise" 1157 | }, 1158 | "sizemode": "diameter", 1159 | "coefficient": 1, 1160 | "numberFormat": "0,0[.]00000", 1161 | "percentFormat": "0[.]00%", 1162 | "textFormat": "", 1163 | "missingValuesAsZero": true, 1164 | "useAggregationsUi": false, 1165 | "showDataLabels": false, 1166 | "dateTimeFormat": "YYYY-MM-DD HH:mm", 1167 | "columnConfigurationMap": { 1168 | "x": { 1169 | "column": "date" 1170 | }, 1171 | "y": [ 1172 | { 1173 | "column": "dropped_records" 1174 | }, 1175 | { 1176 | "column": "passed_records" 1177 | } 1178 | ] 1179 | }, 1180 | "showPlotlyControls": true, 1181 | "swappedAxes": false 1182 | }, 1183 | "updated_at": "2023-01-17T16:14:19Z", 1184 | "created_at": "2023-01-17T16:11:31Z", 1185 | "query_plan": null, 1186 | "query": { 1187 | "id": "a71a57a5-a118-4bda-a651-a2600c25f771", 1188 | "name": "DLT-retail-data-quality", 1189 | "description": null, 1190 | "query": "select sum(passed_records) as passed_records,\n sum(failed_records) as failed_records,\n sum(dropped_records) as dropped_records,\n sum(output_records) as output_records,\n sum(failed_records)/sum(output_records)*100 as failure_rate,\n name,\n dataset,\n date(timestamp) as date from dbdemos_c360.dlt_expectations values group by date, dataset, name;", 1191 | "is_draft": false, 1192 | "updated_at": "2023-01-17T16:11:32Z", 1193 | "created_at": "2023-01-17T16:11:27Z", 1194 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 1195 | "options": { 1196 | "parent": "folders/4156927442168299", 1197 | "apply_auto_limit": false, 1198 | "folder_node_status": "ACTIVE", 1199 | "folder_node_internal_name": "tree/4156927442757239", 1200 | "parameters": [] 1201 | }, 1202 | "version": 1, 1203 | "tags": [], 1204 | "is_safe": true, 1205 | "user_id": 7644138420879474, 1206 | "run_as_role": null, 1207 | "run_as_service_principal_id": null, 1208 | "schedule": null, 1209 | "can_edit": true 1210 | } 1211 | } 1212 | }, 1213 | { 1214 | "id": "7763f27e-2bb3-437a-a973-45c11691af5a", 1215 | "width": 1, 1216 | "options": { 1217 | "isHidden": false, 1218 | "position": { 1219 | "autoHeight": false, 1220 | "sizeX": 6, 1221 | "sizeY": 1, 1222 | "minSizeX": 1, 1223 | "maxSizeX": 6, 1224 | "minSizeY": 1, 1225 | "maxSizeY": 1000, 1226 | "col": 0, 1227 | "row": 23 1228 | } 1229 | }, 1230 | "dashboard_id": "6f73dd1b-17b1-49d0-9a11-b3772a2c3357", 1231 | "text": "Field Demo dashboard. Please do not edit\n![tracking_img](https://www.google-analytics.com/collect?v=1>m=GTM-NKQ8TT7&tid=UA-163989034-1&cid=555&aip=1&t=event&ec=field_demos&ea=display&dp=%2F42_field_demos%2Fretail%2Fnotebook&dt=DASHBOARD_RETAIL_DLT_QUALITY)", 1232 | "updated_at": "2023-01-17T16:11:51Z", 1233 | "created_at": "2023-01-17T16:11:51Z" 1234 | }, 1235 | { 1236 | "id": "314048cc-d07e-4a67-b091-f949d2281158", 1237 | "width": 1, 1238 | "options": { 1239 | "parameterMappings": {}, 1240 | "title": "Input schema", 1241 | "description": "", 1242 | "isHidden": false, 1243 | "position": { 1244 | "autoHeight": false, 1245 | "sizeX": 2, 1246 | "sizeY": 4, 1247 | "minSizeX": 1, 1248 | "maxSizeX": 6, 1249 | "minSizeY": 1, 1250 | "maxSizeY": 1000, 1251 | "col": 4, 1252 | "row": 4 1253 | } 1254 | }, 1255 | "dashboard_id": "6f73dd1b-17b1-49d0-9a11-b3772a2c3357", 1256 | "text": "", 1257 | "updated_at": "2023-01-17T16:11:52Z", 1258 | "created_at": "2023-01-17T16:11:52Z", 1259 | "visualization": { 1260 | "id": "af612aa6-e95b-42f2-818c-bb3329163499", 1261 | "type": "COUNTER", 1262 | "name": "Invalid input schema", 1263 | "description": "", 1264 | "options": { 1265 | "counterLabel": "Invalid input schema", 1266 | "counterColName": "failure_rate", 1267 | "rowNumber": 1, 1268 | "targetRowNumber": 1, 1269 | "stringDecimal": 2, 1270 | "stringDecChar": ".", 1271 | "stringThouSep": ",", 1272 | "tooltipFormat": "0,0.000", 1273 | "useAggregationsUi": false, 1274 | "showPlotlyControls": true, 1275 | "stringPrefix": "", 1276 | "stringSuffix": "%" 1277 | }, 1278 | "updated_at": "2023-01-17T16:14:19Z", 1279 | "created_at": "2023-01-17T16:11:26Z", 1280 | "query_plan": null, 1281 | "query": { 1282 | "id": "89625892-ee6a-4bb7-abe6-7c40fe9c0f87", 1283 | "name": "DLT-retail-data-quality-stats", 1284 | "description": null, 1285 | "query": "select \n date(timestamp) day, sum(failed_records)/sum(output_records)*100 failure_rate, sum(output_records) output_records from dbdemos_c360.dlt_expectations \ngroup by day order by day", 1286 | "is_draft": false, 1287 | "updated_at": "2023-01-17T16:11:27Z", 1288 | "created_at": "2023-01-17T16:11:21Z", 1289 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 1290 | "options": { 1291 | "parent": "folders/4156927442168299", 1292 | "apply_auto_limit": false, 1293 | "folder_node_status": "ACTIVE", 1294 | "folder_node_internal_name": "tree/4156927442757238", 1295 | "parameters": [] 1296 | }, 1297 | "version": 1, 1298 | "tags": [], 1299 | "is_safe": true, 1300 | "user_id": 7644138420879474, 1301 | "run_as_role": null, 1302 | "run_as_service_principal_id": null, 1303 | "schedule": null, 1304 | "can_edit": true 1305 | } 1306 | } 1307 | } 1308 | ], 1309 | "options": { 1310 | "run_as_role": "viewer", 1311 | "refresh_schedules": [ 1312 | { 1313 | "id": "caeee496-e18c-4cb3-8045-221d3643ba20", 1314 | "cron": "16 15 13 * * ?", 1315 | "active": true, 1316 | "job_id": "e318265968e722b9c63eb4d2c127e054ea117dbd" 1317 | } 1318 | ], 1319 | "schedule_failures": 0 1320 | }, 1321 | "is_draft": false, 1322 | "tags": [ 1323 | "field_demos", 1324 | "field_demos_retail", 1325 | "dlt_data_quality" 1326 | ], 1327 | "updated_at": "2023-01-17T16:11:39Z", 1328 | "created_at": "2021-10-21T08:09:27Z", 1329 | "version": 863, 1330 | "color_palette": null, 1331 | "run_as_role": "viewer", 1332 | "run_as_service_principal_id": null, 1333 | "refresh_schedules": [ 1334 | { 1335 | "id": "caeee496-e18c-4cb3-8045-221d3643ba20", 1336 | "cron": "16 15 13 * * ?", 1337 | "active": true, 1338 | "job_id": "e318265968e722b9c63eb4d2c127e054ea117dbd" 1339 | } 1340 | ], 1341 | "data_source_id": "cef03be2-9a33-425a-bd0d-6ce0127aff32", 1342 | "is_favorite": true, 1343 | "user": { 1344 | "id": 7644138420879474, 1345 | "name": "Quentin Ambard", 1346 | "email": "quentin.ambard@databricks.com", 1347 | "profile_image_url": "https://www.gravatar.com/avatar/62d8f76888369585890af5b5dce9a395?s=40&d=identicon", 1348 | "is_db_admin": false 1349 | }, 1350 | "is_archived": false, 1351 | "can_edit": true, 1352 | "permission_tier": "CAN_MANAGE" 1353 | } 1354 | } --------------------------------------------------------------------------------