├── src ├── __init__.py ├── class_Legendary_model.pkl ├── __pycache__ │ ├── training_script.cpython-39.pyc │ └── gen_data_and_simulate_drift.cpython-39.pyc ├── crontab │ └── crontab ├── model_training_script.py ├── app.py ├── roi.py ├── training_script.py ├── fake_data_script.py ├── compute_metrics_script.py └── gen_data_and_simulate_drift.py ├── docker └── influxdb │ ├── Dockerfile │ └── influxdb.sh ├── images ├── pulsarml-logo.png ├── pulsar-demo-workflow.png └── PulsarDemoFlowDiagram_v1.jpeg ├── class_Legendary_model.pkl ├── models ├── kidney_disease.pkl ├── bank_generator_old.pkl └── class_default_model.pkl ├── requirements.txt ├── pyenv-installer.ps1 ├── grafana-provisioning ├── dashboards │ ├── automatic.yml │ ├── drift_summary_dashboard.json │ ├── roi_dashboard.json │ └── drift_detail_dashboard.json └── datasources │ └── automatic.yml ├── pyproject.toml ├── Dockerfile ├── docker-compose.yaml ├── datasets ├── kidney_test.csv ├── kidney_test_backup.csv ├── test_data_no_class.csv └── bank_num.csv ├── .gitignore ├── README.md └── dashboards └── metrics ├── drift_summary_metrics.json ├── roi_dashboard.json └── drift_detail_metrics.json /src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docker/influxdb/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM influxdb:1.5.3 2 | 3 | COPY influxdb.sh /docker-entrypoint-initdb.d/influxdb.sh -------------------------------------------------------------------------------- /images/pulsarml-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rocket-Science-Development/pulsar_demo/HEAD/images/pulsarml-logo.png -------------------------------------------------------------------------------- /class_Legendary_model.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rocket-Science-Development/pulsar_demo/HEAD/class_Legendary_model.pkl -------------------------------------------------------------------------------- /models/kidney_disease.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rocket-Science-Development/pulsar_demo/HEAD/models/kidney_disease.pkl -------------------------------------------------------------------------------- /models/bank_generator_old.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rocket-Science-Development/pulsar_demo/HEAD/models/bank_generator_old.pkl -------------------------------------------------------------------------------- /src/class_Legendary_model.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rocket-Science-Development/pulsar_demo/HEAD/src/class_Legendary_model.pkl -------------------------------------------------------------------------------- /images/pulsar-demo-workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rocket-Science-Development/pulsar_demo/HEAD/images/pulsar-demo-workflow.png -------------------------------------------------------------------------------- /models/class_default_model.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rocket-Science-Development/pulsar_demo/HEAD/models/class_default_model.pkl -------------------------------------------------------------------------------- /images/PulsarDemoFlowDiagram_v1.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rocket-Science-Development/pulsar_demo/HEAD/images/PulsarDemoFlowDiagram_v1.jpeg -------------------------------------------------------------------------------- /src/__pycache__/training_script.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rocket-Science-Development/pulsar_demo/HEAD/src/__pycache__/training_script.cpython-39.pyc -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | black 2 | pulsar-metrics 3 | pulsar-data-collection 4 | copulas 5 | catboost==1.2 6 | scikit-learn 7 | numpy==1.21.0 8 | pysqlite3==0.4.7 9 | pandas==1.5.3 -------------------------------------------------------------------------------- /src/__pycache__/gen_data_and_simulate_drift.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Rocket-Science-Development/pulsar_demo/HEAD/src/__pycache__/gen_data_and_simulate_drift.cpython-39.pyc -------------------------------------------------------------------------------- /docker/influxdb/influxdb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | influx -execute "CREATE USER ${INFLUXDB_USER} WITH PASSWORD '${INFLUXDB_USER_PASSWORD}' WITH ALL PRIVILEGES" 5 | influx -execute "CREATE DATABASE PULSAR_DATA_COLLECTION" 6 | 7 | 8 | cat >> /etc/influxdb/influxdb.conf <[http] 9 | auth-enabled = true 10 | EOD -------------------------------------------------------------------------------- /pyenv-installer.ps1: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Usage: curl https://pyenv.run | bash 4 | # 5 | # For more info, visit: https://github.com/pyenv/pyenv-installer 6 | # 7 | index_main() { 8 | set -e 9 | curl -s -S -L https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash 10 | } 11 | 12 | index_main 13 | -------------------------------------------------------------------------------- /grafana-provisioning/dashboards/automatic.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: "Influx DB" 5 | orgId: 1 6 | folder: "Metrics" 7 | type: file 8 | disableDeletion: false 9 | updateIntervalSeconds: 10 10 | editable: true 11 | allowUiUpdates: true 12 | options: 13 | path: /etc/grafana/dashboards 14 | foldersFromFilesStructure: true -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "pulsar-demo" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Your Name "] 6 | readme = "README.md" 7 | packages = [{include = "pulsar_demo"}] 8 | 9 | [tool.poetry.dependencies] 10 | python = "3.9.6" 11 | 12 | 13 | [tool.poetry.group.dev.dependencies] 14 | ipykernel = "^6.22.0" 15 | 16 | [build-system] 17 | requires = ["poetry-core"] 18 | build-backend = "poetry.core.masonry.api" 19 | -------------------------------------------------------------------------------- /src/crontab/crontab: -------------------------------------------------------------------------------- 1 | SHELL=/bin/bash 2 | PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin 3 | 4 | * * * * * root env > /var/log/cron_env.log 2>&1 5 | * * * * * root echo "Cron ran at $(date)" >> /var/log/cron_test.log 2>&1 6 | */4 * * * * root /usr/local/bin/python /app/compute_metrics_script.py >> /var/log/cron.log 2>&1 7 | */2 * * * * root /usr/local/bin/python /app/fake_data_script.py >> /var/log/cron.log 2>&1 8 | 9 | # Don't remove the empty line at the end of this file. It is required to run the cron job. -------------------------------------------------------------------------------- /grafana-provisioning/datasources/automatic.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | datasources: 4 | - name: MYINFLUXDB 5 | type: influxdb 6 | access: proxy 7 | url: http://influxdb:8086 8 | password: pass123 9 | user: admin 10 | database: PULSAR_DATA_COLLECTION 11 | basicAuth: false 12 | basicAuthUser: 13 | basicAuthPassword: 14 | withCredentials: 15 | isDefault: true 16 | jsonData: 17 | tlsAuth: false 18 | tlsAuthWithCACert: false 19 | secureJsonData: 20 | tlsCACert: "" 21 | tlsClientCert: "" 22 | tlsClientKey: "" 23 | version: 1 24 | editable: true -------------------------------------------------------------------------------- /src/model_training_script.py: -------------------------------------------------------------------------------- 1 | import pickle as pkl 2 | import os 3 | import pandas as pd 4 | from gen_data_and_simulate_drift import GenerateFakeData 5 | from training_script import Classifier 6 | 7 | # Load and train the classifier 8 | target = 'Legendary' 9 | reference_data = 'pokemon.csv' 10 | SAMPLE_SIZE = 1000 11 | 12 | if __name__ == '__main__': 13 | genertor_fake_data = GenerateFakeData(path_ref_data=reference_data, sample_size=SAMPLE_SIZE, target=target) 14 | sampled_data = genertor_fake_data.get_dataclass_sampling() 15 | 16 | pok_classifier = Classifier(df_train=sampled_data.train_data, 17 | num_features=sampled_data.list_num_col, 18 | cat_features=None, 19 | target=target, 20 | pkl_file_path=os.path.join('/app', f'class_{target}_model.pkl')) 21 | 22 | pok_classifier.train() 23 | pok_classifier.serialize() -------------------------------------------------------------------------------- /src/app.py: -------------------------------------------------------------------------------- 1 | from gen_data_and_simulate_drift import GenerateFakeData, DriftSimulator, DriftIntensity 2 | from training_script import Classifier 3 | 4 | 5 | if __name__ == '__main__': 6 | target = 'Legendary' 7 | genertor_fake_data = GenerateFakeData(path_ref_data='pokemon.csv', sample_size = 1000, target=target) 8 | sampled_data = genertor_fake_data.get_dataclass_sampling() 9 | 10 | # if the task is classification 11 | pok_classifier = Classifier(df_train=sampled_data.train_data, 12 | num_features=sampled_data.list_num_col, 13 | cat_features=None, 14 | target=target, 15 | pkl_file_path=f'class_{target}_model.pkl') 16 | pok_classifier.train() 17 | 18 | drift_sim_info = DriftSimulator(sampled_data, nb_cols_to_drift=1, drift_intensity=DriftIntensity.MODERATE) 19 | # to get test_data after drifting 20 | df_test_drifted = drift_sim_info.get_test_data_drifted() -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.9.6-slim-buster 2 | 3 | # Install dependencies 4 | RUN apt-get update && apt-get install -y git gcc cron procps 5 | 6 | # Set working directory 7 | WORKDIR /app 8 | 9 | # Copy application files 10 | COPY requirements.txt requirements.txt 11 | COPY src/compute_metrics_script.py compute_metrics_script.py 12 | COPY src/gen_data_and_simulate_drift.py gen_data_and_simulate_drift.py 13 | COPY src/fake_data_script.py fake_data_script.py 14 | COPY src/training_script.py training_script.py 15 | COPY src/roi.py roi.py 16 | COPY src/class_Legendary_model.pkl class_Legendary_model.pkl 17 | COPY datasets/ datasets/ 18 | COPY models/ models/ 19 | 20 | # Install Python packages 21 | RUN pip install --no-cache-dir -r requirements.txt 22 | 23 | # Copy and set up cron job 24 | COPY src/crontab/crontab /etc/cron.d/my-cron-job 25 | RUN sed -i 's/\r$//' /etc/cron.d/my-cron-job && \ 26 | chmod 0644 /etc/cron.d/my-cron-job 27 | 28 | # Create log files for cron output 29 | RUN touch /var/log/cron.log /var/log/cron_env.log 30 | 31 | # Ensure cron picks up the job file in /etc/cron.d/ 32 | # Start cron and follow the log file 33 | CMD ["cron", "-f"] -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | compute-metrics: 3 | build: . 4 | links: 5 | - influxdb 6 | influxdb: 7 | build: 8 | context: ./docker/influxdb 9 | ports: 10 | - "8086:8086" 11 | volumes: 12 | - influxdb-storage:/var/lib/influxdb 13 | environment: 14 | - INFLUXDB_ADMIN_USER=${DB_USER} 15 | - INFLUXDB_ADMIN_PASSWORD=${DB_PASSWORD} 16 | - INFLUXDB_USER=${DB_USER} 17 | - INFLUXDB_PASSWORD=${DB_PASSWORD} 18 | - INFLUXDB_HTTP_AUTH_ENABLED=false 19 | restart: always 20 | chronograf: 21 | image: chronograf:latest 22 | ports: 23 | - '8889:8888' 24 | volumes: 25 | - chronograf-storage:/var/lib/chronograf 26 | depends_on: 27 | - influxdb 28 | environment: 29 | - INFLUXDB_URL=http://influxdb:8086 30 | - INFLUXDB_USERNAME=${DB_USER} 31 | - INFLUXDB_PASSWORD=${DB_PASSWORD} 32 | grafana: 33 | image: grafana/grafana:latest 34 | ports: 35 | - '3000:3000' 36 | volumes: 37 | - grafana-storage:/var/lib/grafana 38 | - ./grafana-provisioning/:/etc/grafana/provisioning 39 | - ./dashboards/:/etc/grafana/dashboards 40 | depends_on: 41 | - influxdb 42 | environment: 43 | - GF_SECURITY_ADMIN_USER=${GRAFANA_USERNAME} 44 | - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD} 45 | volumes: 46 | influxdb-storage: 47 | chronograf-storage: 48 | grafana-storage: -------------------------------------------------------------------------------- /datasets/kidney_test.csv: -------------------------------------------------------------------------------- 1 | id,age,bp,sg,al,su,rbc,pc,pcc,ba,bgr,bu,sc,sod,pot,hemo,pcv,wbcc,rbcc,htn,dm,cad,appet,pe,ane,class,y_pred,clf_target,y_pred_proba,pred_timestamp 2 | 214,55.0,80.0,1.01,3.0,1,1,0,1,1,214.0,73.0,3.9,137.0,4.9,10.9,34.0,7400.0,3.7,1,1,0,0,1,0,1,1,0,1.00,2022-07-10 22:30:30 3 | 372,28.0,60.0,1.025,0.0,0,1,1,0,0,79.0,50.0,0.5,145.0,5.0,17.6,51.0,6500.0,5.0,0,0,0,0,0,0,0,1,0,1.00,2022-07-10 22:30:30 4 | 367,60.0,80.0,1.025,0.0,0,1,1,0,0,81.0,15.0,0.5,141.0,3.6,15.0,46.0,10500.0,5.3,0,0,0,0,0,0,0,0,0,0.99,2022-07-10 22:30:30 5 | 260,35.0,80.0,1.02,0.0,0,1,1,0,0,104.0,31.0,1.2,135.0,5.0,16.1,45.0,4300.0,5.2,0,0,0,0,0,0,0,0,0,0.98,2022-07-15 22:30:30 6 | 374,61.0,70.0,1.025,0.0,0,1,1,0,0,133.0,38.0,1.0,142.0,3.6,13.7,47.0,9200.0,4.9,0,0,0,0,0,0,0,1,0,0.91,2022-07-15 22:30:30 7 | 243,69.0,70.0,1.01,4.0,3,1,0,1,1,214.0,96.0,6.3,120.0,3.9,9.4,28.0,11500.0,3.3,1,1,1,0,1,1,1,1,1,0.88,2022-07-25 15:30:30 8 | 307,52.0,80.0,1.02,0.0,0,1,1,0,0,128.0,30.0,1.2,140.0,4.5,15.2,52.0,4300.0,5.7,0,0,0,0,0,0,0,1,0,0.75,2022-07-25 15:30:30 9 | 362,29.0,80.0,1.02,0.0,0,1,1,0,0,70.0,16.0,0.7,138.0,3.5,13.7,54.0,5400.0,5.8,0,0,0,0,0,0,0,0,0,0.77,2022-08-05 15:30:30 10 | 371,69.0,70.0,1.02,0.0,0,1,1,0,0,83.0,42.0,1.2,139.0,3.7,16.2,50.0,9300.0,5.4,0,0,0,0,0,0,0,1,0,0.55,2022-08-05 15:30:30 11 | 263,55.0,80.0,1.02,0.0,0,1,1,0,0,118.0,18.0,0.9,135.0,3.6,15.5,43.0,7200.0,5.4,0,0,0,0,0,0,0,0,0,0.55,2022-08-21 15:30:30 12 | 267,55.0,80.0,1.02,0.0,0,1,1,0,0,133.0,17.0,1.2,135.0,4.8,13.2,41.0,6800.0,5.3,0,0,0,0,0,0,0,0,0,0.31,2022-08-21 15:30:30 13 | 398,12.0,80.0,1.02,0.0,0,1,1,0,0,100.0,26.0,0.6,137.0,4.4,15.8,49.0,6600.0,5.4,0,0,0,0,0,0,0,0,0,0.21,2022-08-21 15:30:30 -------------------------------------------------------------------------------- /datasets/kidney_test_backup.csv: -------------------------------------------------------------------------------- 1 | id,age,bp,sg,al,su,rbc,pc,pcc,ba,bgr,bu,sc,sod,pot,hemo,pcv,wbcc,rbcc,htn,dm,cad,appet,pe,ane,class,y_pred,clf_target,y_pred_proba,pred_timestamp 2 | 214,55.0,80.0,1.01,3.0,1,1,0,1,1,214.0,73.0,3.9,137.0,4.9,10.9,34.0,7400.0,3.7,1,1,0,0,1,0,1,1,0,1.00,2022-07-10 22:30:30 3 | 372,28.0,60.0,1.025,0.0,0,1,1,0,0,79.0,50.0,0.5,145.0,5.0,17.6,51.0,6500.0,5.0,0,0,0,0,0,0,0,1,0,1.00,2022-07-10 22:30:30 4 | 367,60.0,80.0,1.025,0.0,0,1,1,0,0,81.0,15.0,0.5,141.0,3.6,15.0,46.0,10500.0,5.3,0,0,0,0,0,0,0,0,0,0.99,2022-07-10 22:30:30 5 | 260,35.0,80.0,1.02,0.0,0,1,1,0,0,104.0,31.0,1.2,135.0,5.0,16.1,45.0,4300.0,5.2,0,0,0,0,0,0,0,0,0,0.98,2022-07-15 22:30:30 6 | 374,61.0,70.0,1.025,0.0,0,1,1,0,0,133.0,38.0,1.0,142.0,3.6,13.7,47.0,9200.0,4.9,0,0,0,0,0,0,0,1,0,0.91,2022-07-15 22:30:30 7 | 243,69.0,70.0,1.01,4.0,3,1,0,1,1,214.0,96.0,6.3,120.0,3.9,9.4,28.0,11500.0,3.3,1,1,1,0,1,1,1,1,1,0.88,2022-07-25 15:30:30 8 | 307,52.0,80.0,1.02,0.0,0,1,1,0,0,128.0,30.0,1.2,140.0,4.5,15.2,52.0,4300.0,5.7,0,0,0,0,0,0,0,1,0,0.75,2022-07-25 15:30:30 9 | 362,29.0,80.0,1.02,0.0,0,1,1,0,0,70.0,16.0,0.7,138.0,3.5,13.7,54.0,5400.0,5.8,0,0,0,0,0,0,0,0,0,0.77,2022-08-05 15:30:30 10 | 371,69.0,70.0,1.02,0.0,0,1,1,0,0,83.0,42.0,1.2,139.0,3.7,16.2,50.0,9300.0,5.4,0,0,0,0,0,0,0,1,0,0.55,2022-08-05 15:30:30 11 | 263,55.0,80.0,1.02,0.0,0,1,1,0,0,118.0,18.0,0.9,135.0,3.6,15.5,43.0,7200.0,5.4,0,0,0,0,0,0,0,0,0,0.55,2022-08-21 15:30:30 12 | 267,55.0,80.0,1.02,0.0,0,1,1,0,0,133.0,17.0,1.2,135.0,4.8,13.2,41.0,6800.0,5.3,0,0,0,0,0,0,0,0,0,0.31,2022-08-21 15:30:30 13 | 398,12.0,80.0,1.02,0.0,0,1,1,0,0,100.0,26.0,0.6,137.0,4.4,15.8,49.0,6600.0,5.4,0,0,0,0,0,0,0,0,0,0.21,2022-08-21 15:30:30 -------------------------------------------------------------------------------- /src/roi.py: -------------------------------------------------------------------------------- 1 | # Author: Jananjoy Rajkumar 2 | import numpy as np 3 | from sklearn.metrics import confusion_matrix 4 | """ Implementation of ROI calculation for wrong or bad prediction """ 5 | 6 | class ReturnOfInvestment(): 7 | def __init__(self): 8 | '''Constructor of the ReturnOfInvestment class ''' 9 | 10 | def calculate_ROI( 11 | self, 12 | lebel_data, 13 | predicted_data, 14 | FPR_THRESHOLD = 0.5, 15 | FNR_THRESHOLD = 0.2, 16 | cost_per_FP = 100, 17 | cost_per_FN = 300 18 | ): 19 | cm = confusion_matrix(lebel_data, predicted_data) 20 | cm_param = {'tn': cm[0, 0], 'fp': cm[0, 1],'fn': cm[1, 0], 'tp': cm[1, 1]} 21 | print("confusion_matrix",cm_param) 22 | 23 | # total_positves = (FN+TP) being the total number of positives 24 | totalPositives = cm_param['fn'] + cm_param['tp'] 25 | #total_negatives = (FP+ TN) is the total number of ground truth negatives. 26 | totalNegatives = cm_param['fp'] + cm_param['tn'] 27 | 28 | # Caluclate the False Positive Rate(FPR) 29 | # The False Positive Rate FPR = FP / (FP+ TN) 30 | FPR_denominator = cm_param['fp'] + cm_param['tn'] 31 | FPR = cm_param['fp']/FPR_denominator 32 | print("False Positive Rate(FPR)",FPR) 33 | 34 | #Caluclate the False Negative Rate(FNR) 35 | #The False Negative Rate is FNR = FN / (FN + TP) 36 | FNR_denominator = cm_param['fn'] + cm_param['tp'] 37 | FNR = cm_param['fn']/FNR_denominator 38 | print("False Negative Rate(FNR)",FNR) 39 | 40 | #Caluclate the Return of Investment (ROI) for wrong prediction 41 | net_FPR = np.sign(FPR_THRESHOLD - FPR) * totalNegatives * cost_per_FP * FPR 42 | net_FNR = np.sign(FNR_THRESHOLD - FNR) * totalPositives * cost_per_FN * FNR 43 | 44 | print("predict_ROI_FP =>",net_FPR) 45 | print("predict_ROI_FN =>",net_FNR) 46 | 47 | ROI_gain = net_FPR + net_FNR 48 | return ROI_gain -------------------------------------------------------------------------------- /src/training_script.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | from typing import List 3 | 4 | import pandas as pd 5 | from catboost import CatBoostClassifier 6 | from sklearn.model_selection import train_test_split 7 | 8 | 9 | class Classifier: 10 | def __init__(self, df_train:pd.DataFrame, 11 | num_features:List, 12 | cat_features:List, 13 | target:str, 14 | pkl_file_path:str, 15 | iterations = 2, 16 | learning_rate = 1, 17 | depth = 2): 18 | self.df_train = df_train 19 | self.num_features = num_features 20 | self.target = target 21 | self.pkl_file_path= pkl_file_path 22 | if not cat_features: 23 | self.cat_features = list(set(self.df_train.columns) - set(num_features)- set([self.target]) ) 24 | self.iterations = iterations 25 | self.learning_rate = learning_rate 26 | self.depth = depth 27 | 28 | def serialize(self): 29 | with open(self.pkl_file_path, 'wb') as file: 30 | try: 31 | pickle.dump(self, file) 32 | except pickle.PickleError as e: 33 | # handle any PickleError exceptions that may occur during serialization 34 | print("Error occurred while pickling:", e) 35 | 36 | """ 37 | Trains a machine learning model using a dataframe (self. df_train and specified features [self.num_features + self.cat_features] 38 | and target (self.target) attributes. 39 | """ 40 | def train(self): 41 | self.model = CatBoostClassifier(iterations=self.iterations, 42 | learning_rate=self.learning_rate, 43 | depth=self.depth) 44 | self.model.fit(self.df_train[self.num_features + self.cat_features], self.df_train[self.target], cat_features=self.cat_features) 45 | 46 | 47 | 48 | def predict(self, df_test): 49 | #self.load_model() 50 | # Use the loaded model to make predictions 51 | return self.model.predict(df_test) 52 | 53 | def load_model(self): 54 | with open(self.pkl_file_path, 'rb') as file: 55 | loaded_classifier = pickle.load(file) 56 | self.model = loaded_classifier.model 57 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | # Byte-compiled / optimized / DLL files 3 | __pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | pip-wheel-metadata/ 25 | share/python-wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | MANIFEST 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .nox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | *.py,cover 52 | .hypothesis/ 53 | .pytest_cache/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | .python-version 87 | 88 | # pipenv 89 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 90 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 91 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 92 | # install all needed dependencies. 93 | #Pipfile.lock 94 | 95 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 96 | __pypackages__/ 97 | 98 | # Celery stuff 99 | celerybeat-schedule 100 | celerybeat.pid 101 | 102 | # SageMath parsed files 103 | *.sage.py 104 | 105 | # Environments 106 | .env 107 | .venv 108 | env/ 109 | venv/ 110 | ENV/ 111 | env.bak/ 112 | venv.bak/ 113 | 114 | # Spyder project settings 115 | .spyderproject 116 | .spyproject 117 | 118 | # Rope project settings 119 | .ropeproject 120 | 121 | # mkdocs documentation 122 | /site 123 | 124 | # mypy 125 | .mypy_cache/ 126 | .dmypy.json 127 | dmypy.json 128 | 129 | # Pyre type checker 130 | .pyre/ 131 | 132 | # MacOS special files 133 | .DS_Store 134 | .DS_Store? 135 | 136 | # Catboost folder 137 | catboost_info/ 138 | src/catboost_info/ 139 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pulsar.ML Local demo 2 | 3 | This repository contains an example use-case demonstrating how to utilize [pulsar-data-collection](https://github.com/Rocket-Science-Development/pulsar_data_collection) and [pulsar-metrics](https://github.com/Rocket-Science-Development/pulsar_metrics) to implement model monitoring and performance management. 4 | 5 | ## Quickstart 6 | 7 | 1. Make sure that you have the Docker and Docker compose installed and execute the following commands: 8 | 9 | ```bash 10 | export GRAFANA_USERNAME=admin; 11 | export GRAFANA_PASSWORD=pass123; 12 | export DB_USER=admin; 13 | export DB_PASSWORD=pass123; 14 | docker-compose up --build 15 | ``` 16 | 17 | 2. Open a web browser and go to localhost:3000. Enter the Grafana credentials set in the previous step. 18 | 19 | ## Pulsar.ML Architecture 20 | 21 | ![Architecture](images/PulsarDemoFlowDiagram_v1.jpeg) 22 | 23 | ## Demo Components 24 | 25 | ![Demo workflow](images/pulsar-demo-workflow.png) 26 | 27 | Here is a description of the steps within the workflow 28 | 29 | 1. Data is captured from an inference container or a notebook using [pulsar-data-collection](https://github.com/Rocket-Science-Development/pulsar_data_collection) 30 | 2. Collected data point, predictions, and other relevant configured metadata are written into an index inside influxdb 31 | 3. `compute-metrics` service will query the latest entries from the database then leverage [pulsar-metrics](https://github.com/Rocket-Science-Development/pulsar_metrics) in order to compute the different metrics. 32 | 4. All computed metrics are then written to another index in InfluxDB for display in Grafana. 33 | 34 | ## About [Pulsar.ML](https://pulsar.ml/) 35 | 36 | Pulsar.ML is an application designed to assist with monitoring your machine learning models and gaining powerful insights into their performance. 37 | 38 | We have released two open-source packages: 39 | 40 | - [pulsar-data-collection](https://github.com/Rocket-Science-Development/pulsar_data_collection) : A lightweight Python SDK for collecting features, predictions, and metadata from an ML model serving code or microservice. 41 | - [pulsar-metrics](https://github.com/Rocket-Science-Development/pulsar_metrics) : A library for evaluating and monitoring data and concept drift with an extensive set of metrics. It also provides the flexibility to use custom metrics defined by the user. 42 | 43 | We also created [pulsar demo](https://github.com/Rocket-Science-Development/pulsar_demo) to showcase an example use-case that demonstrates how to leverage both packages for model monitoring and performance management. 44 | 45 | For further interaction with our community, Please join our [slack channel](https://pulsarml.slack.com) 46 | 47 | Powered by [Rocket Science Development](https://rocketscience.one/) 48 | 49 | ## Contributing 50 | 51 | 1. Fork this repository, develop, and test your changes 52 | 2. open an issue 53 | 3. Submit a pull request with a reference to the issue 54 | -------------------------------------------------------------------------------- /datasets/test_data_no_class.csv: -------------------------------------------------------------------------------- 1 | id,age,bp,sg,al,su,rbc,pc,pcc,ba,bgr,bu,sc,sod,pot,hemo,pcv,wbcc,rbcc,htn,dm,cad,appet,pe,ane 2 | 214,55.0,80.0,1.01,3.0,1,1,0,1,1,214.0,73.0,3.9,137.0,4.9,10.9,34.0,7400.0,3.7,1,1,0,0,1,0 3 | 372,28.0,60.0,1.025,0.0,0,1,1,0,0,79.0,50.0,0.5,145.0,5.0,17.6,51.0,6500.0,5.0,0,0,0,0,0,0 4 | 367,60.0,80.0,1.025,0.0,0,1,1,0,0,81.0,15.0,0.5,141.0,3.6,15.0,46.0,10500.0,5.3,0,0,0,0,0,0 5 | 260,35.0,80.0,1.02,0.0,0,1,1,0,0,104.0,31.0,1.2,135.0,5.0,16.1,45.0,4300.0,5.2,0,0,0,0,0,0 6 | 374,61.0,70.0,1.025,0.0,0,1,1,0,0,133.0,38.0,1.0,142.0,3.6,13.7,47.0,9200.0,4.9,0,0,0,0,0,0 7 | 243,69.0,70.0,1.01,4.0,3,1,0,1,1,214.0,96.0,6.3,120.0,3.9,9.4,28.0,11500.0,3.3,1,1,1,0,1,1 8 | 307,52.0,80.0,1.02,0.0,0,1,1,0,0,128.0,30.0,1.2,140.0,4.5,15.2,52.0,4300.0,5.7,0,0,0,0,0,0 9 | 362,29.0,80.0,1.02,0.0,0,1,1,0,0,70.0,16.0,0.7,138.0,3.5,13.7,54.0,5400.0,5.8,0,0,0,0,0,0 10 | 371,69.0,70.0,1.02,0.0,0,1,1,0,0,83.0,42.0,1.2,139.0,3.7,16.2,50.0,9300.0,5.4,0,0,0,0,0,0 11 | 263,55.0,80.0,1.02,0.0,0,1,1,0,0,118.0,18.0,0.9,135.0,3.6,15.5,43.0,7200.0,5.4,0,0,0,0,0,0 12 | 267,55.0,80.0,1.02,0.0,0,1,1,0,0,133.0,17.0,1.2,135.0,4.8,13.2,41.0,6800.0,5.3,0,0,0,0,0,0 13 | 398,12.0,80.0,1.02,0.0,0,1,1,0,0,100.0,26.0,0.6,137.0,4.4,15.8,49.0,6600.0,5.4,0,0,0,0,0,0 14 | 213,40.0,70.0,1.015,3.0,4,1,1,0,0,253.0,150.0,11.9,132.0,5.6,10.9,31.0,8800.0,3.4,1,1,0,1,1,0 15 | 378,64.0,70.0,1.02,0.0,0,1,1,0,0,97.0,27.0,0.7,145.0,4.8,13.8,49.0,6400.0,4.8,0,0,0,0,0,0 16 | 345,64.0,60.0,1.02,0.0,0,1,1,0,0,106.0,27.0,0.7,150.0,3.3,14.4,42.0,8100.0,4.7,0,0,0,0,0,0 17 | 292,47.0,80.0,1.025,0.0,0,1,1,0,0,124.0,44.0,1.0,140.0,4.9,14.9,41.0,7000.0,5.7,0,0,0,0,0,0 18 | 388,15.0,80.0,1.025,0.0,0,1,1,0,0,93.0,17.0,0.9,136.0,3.9,16.7,50.0,6200.0,5.2,0,0,0,0,0,0 19 | 342,63.0,70.0,1.025,0.0,0,1,1,0,0,130.0,37.0,0.9,150.0,5.0,13.4,41.0,7300.0,4.7,0,0,0,0,0,0 20 | 387,46.0,70.0,1.025,0.0,0,1,1,0,0,100.0,47.0,0.5,142.0,3.5,16.4,43.0,5700.0,6.5,0,0,0,0,0,0 21 | 383,48.0,80.0,1.025,0.0,0,1,1,0,0,75.0,22.0,0.8,137.0,5.0,16.8,51.0,6000.0,6.5,0,0,0,0,0,0 22 | 92,56.0,70.0,1.015,4.0,1,0,1,0,0,210.0,26.0,1.7,136.0,3.8,16.1,52.0,12500.0,5.6,0,0,0,0,0,0 23 | 327,47.0,60.0,1.02,0.0,0,1,1,0,0,109.0,25.0,1.1,141.0,4.7,15.8,41.0,8300.0,5.2,0,0,0,0,0,0 24 | 293,30.0,80.0,1.02,0.0,0,1,1,0,0,89.0,42.0,0.5,139.0,5.0,16.7,52.0,10200.0,5.0,0,0,0,0,0,0 25 | 15,68.0,80.0,1.01,3.0,2,1,0,1,1,157.0,90.0,4.1,130.0,6.4,5.6,16.0,11000.0,2.6,1,1,1,1,1,0 26 | 365,73.0,80.0,1.025,0.0,0,1,1,0,0,118.0,44.0,0.7,137.0,3.5,14.8,45.0,9300.0,4.7,0,0,0,0,0,0 27 | 131,50.0,90.0,1.01,2.0,0,1,0,1,1,128.0,208.0,9.2,134.0,4.8,8.2,22.0,16300.0,2.7,0,0,0,1,1,1 28 | 400,58.0,80.0,1.025,0.0,0,1,1,0,0,131.0,18.0,1.1,141.0,3.5,15.8,53.0,6800.0,6.1,0,0,0,0,0,0 29 | 59,73.0,80.0,1.02,2.0,0,0,0,0,0,253.0,142.0,4.6,138.0,5.8,10.5,33.0,7200.0,4.3,1,1,1,0,0,0 30 | 399,17.0,60.0,1.025,0.0,0,1,1,0,0,114.0,50.0,1.0,135.0,4.9,14.2,51.0,7200.0,5.9,0,0,0,0,0,0 31 | 129,52.0,90.0,1.015,4.0,3,1,0,0,0,224.0,166.0,5.6,133.0,47.0,8.1,23.0,5000.0,2.9,1,1,0,0,0,1 32 | 85,59.0,70.0,1.01,3.0,0,1,0,0,0,76.0,186.0,15.0,135.0,7.6,7.1,22.0,3800.0,2.1,1,0,0,1,1,1 33 | 299,34.0,60.0,1.02,0.0,0,1,1,0,0,91.0,49.0,1.2,135.0,4.5,13.5,48.0,8600.0,4.9,0,0,0,0,0,0 34 | -------------------------------------------------------------------------------- /src/fake_data_script.py: -------------------------------------------------------------------------------- 1 | import pickle as pkl 2 | import os 3 | import pandas as pd 4 | import numpy as numpy 5 | from pulsar_data_collection.data_capture import ( 6 | DatabaseLogin, 7 | DataCapture, 8 | DataWithPrediction, 9 | ) 10 | 11 | from gen_data_and_simulate_drift import DriftIntensity, DriftSimulator, GenerateFakeData 12 | from training_script import Classifier 13 | 14 | 15 | """ 16 | 17 | :This script generates synthetic data using copulas and simulates drift in the dataset. 18 | 19 | """ 20 | 21 | # Below is the code for Kidney disease model 22 | # kidneydisease_test_data='/app/datasets/test_data_no_class.csv' 23 | # kidneydisease_model="/app/models/kidney_disease.pkl" 24 | # if __name__ == "__main__": 25 | # print("Pushing data") 26 | # data_test = pd.read_csv(kidneydisease_test_data) 27 | 28 | # model = pkl.load(open(kidneydisease_model, "rb")) 29 | 30 | # test_data = data_test.sample(frac=0.3, random_state=1) 31 | 32 | # prediction = model.predict(test_data) 33 | 34 | # Below is the code for Pokemon model 35 | 36 | with open("/var/log/test_script.log", "a") as f: 37 | f.write("Script started\n") 38 | 39 | pokemon_test_data='/app/datasets/bank_num.csv' 40 | SAMPLE_SIZE=1000 41 | if __name__ == '__main__': 42 | """ 43 | :Main execution point of the script. 44 | """ 45 | target = 'default' 46 | genertor_fake_data = GenerateFakeData(path_ref_data=pokemon_test_data, sample_size=SAMPLE_SIZE, target=target, model_name = 'bank_generator.pkl') 47 | sampled_data = genertor_fake_data.get_dataclass_sampling() 48 | 49 | # if the task is classification 50 | # pok_classifier = Classifier(df_train=sampled_data.train_data, 51 | # num_features=sampled_data.list_num_col, 52 | # cat_features=None, 53 | # target=target, 54 | # pkl_file_path=f'class_{target}_model.pkl') 55 | pok_classifier = Classifier(df_train=sampled_data.train_data, 56 | num_features=sampled_data.list_num_col, 57 | cat_features=None, 58 | target=target, 59 | pkl_file_path=os.path.join('/app', f'class_{target}_model.pkl') 60 | ,iterations = 200, depth = None, learning_rate = 0.03) 61 | 62 | pok_classifier.train() 63 | pok_classifier.serialize() 64 | 65 | # pok_classifier.load_model() 66 | 67 | drift_sim_info = DriftSimulator(sampled_data, nb_cols_to_drift=1, drift_intensity=DriftIntensity.MODERATE) 68 | # to get test_data after drifting 69 | 70 | df_test_drifted = drift_sim_info.get_test_data_drifted() 71 | # print('info:',df_test_drifted.dtypes) 72 | # df_test_drifted[target] = df_test_drifted[target].astype(int) 73 | 74 | prediction = pok_classifier.predict(df_test=df_test_drifted.drop(target, axis=1)) 75 | prediction_int = [1 if e=='yes' else 0 for e in prediction] 76 | 77 | conversion_dict = {'yes': 1, 'no': 0} 78 | df_test_drifted['default'] = df_test_drifted['default'].map(conversion_dict) 79 | # prediction = prediction.astype(int) 80 | # print('prediction: ', prediction) 81 | # print('prediction.type',prediction.dtypes) 82 | # print('prediction_int: ', prediction_int) 83 | # print('prediction_int_type',prediction_int.dtypes) 84 | 85 | # Adding the logic to Flip a % of values in our Prediction array before ingesting the data in DB so as to have FP & FN values generated successfully. 86 | # Introduce random coefficient 87 | # randomization_percentage = 0.2 # Can be adjusted. 0.2 here means 20% of the values will be flipped. Accuracy hence would be 80% in this scenario. 88 | 89 | # Calculate the number of values to be changed based on the coefficient 90 | # num_values_to_change = int(len(prediction_int) * randomization_percentage) 91 | 92 | # Generate random indices to select positions for randomization 93 | # random_indices = numpy.random.choice(len(prediction_int), size=num_values_to_change, replace=False) 94 | 95 | # Iterate through the random indices and flip the corresponding values in prediction_int 96 | # for index in random_indices: 97 | # prediction_int[index] = 1 - prediction_int[index] 98 | 99 | # print('prediction_int after: ', prediction_int) 100 | 101 | database_login = DatabaseLogin( 102 | db_host="influxdb", 103 | db_port=8086, 104 | db_user="admin", 105 | db_password="pass123", 106 | protocol="line" 107 | ) 108 | 109 | dat_capture = DataCapture( 110 | storage_engine="influxdb", 111 | model_id="1", 112 | model_version="2", 113 | data_id="FluxDB", 114 | y_name="y_pred", 115 | pred_name="clf_target", 116 | operation_type="INSERT_PREDICTION", 117 | login_url=database_login 118 | ) 119 | 120 | prediction_numpy = numpy.asarray(prediction_int) 121 | 122 | dat_predict = DataWithPrediction(prediction=prediction_numpy, data_points=df_test_drifted) 123 | 124 | dat_capture.push(dat_predict) 125 | -------------------------------------------------------------------------------- /src/compute_metrics_script.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | import pandas as pd 3 | from datetime import datetime 4 | from pulsar_data_collection.data_capture import DataCapture, DatabaseLogin 5 | from pulsar_metrics.analyzers.base import Analyzer 6 | from pulsar_metrics.metrics.base import CustomMetric 7 | from roi import ReturnOfInvestment 8 | 9 | FPR_THRESHOLD=0.5 10 | FNR_THRESHOLD=0.2 11 | cost_per_FP=100 12 | cost_per_FN=300 13 | 14 | @CustomMetric 15 | def test_roi(current, reference, multiple=3, **kwargs): 16 | roiInstance = ReturnOfInvestment() 17 | roi_x = roiInstance.calculate_ROI(reference, current, FPR_THRESHOLD, FNR_THRESHOLD, cost_per_FP, cost_per_FN) 18 | return roi_x 19 | 20 | if __name__ == "__main__": 21 | # Reading reference dataset 22 | df_result = pd.DataFrame() 23 | df_ref = pd.read_csv('/app/datasets/bank_num.csv') 24 | target = 'default' 25 | prediction = 'y_pred' 26 | 27 | df_ref['model_id'] = '1' 28 | df_ref['model_version'] = '2' 29 | df_ref['pred_timestamp'] = datetime.strptime('10/7/2022 22:30:30', '%d/%m/%Y %H:%M:%S') 30 | df_ref['period'] = 'reference' 31 | 32 | # Reading the newest prediction data 33 | database_login = DatabaseLogin(db_host="influxdb", db_port=8086, db_user="admin", db_password="pass123", protocol="line") 34 | 35 | dat_capture = DataCapture( 36 | storage_engine="influxdb", 37 | operation_type="METRICS", 38 | login_url=database_login 39 | ) 40 | 41 | # receiving the last period 42 | last_eval_timestamp = dat_capture.collect_eval_timestamp() 43 | if last_eval_timestamp: 44 | last_eval_timestamp_str = last_eval_timestamp.strftime('%Y-%m-%d %H:%M:%S') 45 | db_df = pd.DataFrame(dat_capture.collect({"time": f">= '{last_eval_timestamp_str}'"}).get("prediction")) 46 | else: 47 | db_df = pd.DataFrame(dat_capture.collect().get("prediction")) 48 | 49 | print(f"Dataframe collected, df length: {len(db_df)}") 50 | # print(f"Dataframe : {db_df.head()}") 51 | 52 | conversion_dict = {'yes': 1, 'no': 0} 53 | df_ref['default'] = df_ref['default'].map(conversion_dict) 54 | 55 | if len(db_df): 56 | # TODO: it should be changed, it's not clear why we cannot use datetime now 57 | db_df['pred_timestamp'] = datetime.strptime('20/8/2022 22:30:30', '%d/%m/%Y %H:%M:%S') 58 | 59 | analysis = Analyzer(name='Compute Script Analyzer', description='Analyzer for compute script usage', model_id='1', model_version='2') 60 | analysis.add_drift_metrics( 61 | metrics_list=['wasserstein', 'ttest', 'ks_2samp','kl','manwu','levene','bftest','CvM','psi'], 62 | # features_list=['age','al','ane','appet','ba','bgr','bp','bu','cad','dm','hemo','htn','id','pc','pcc','pcv','pe','pot','rbc','rbcc','sg','sc','sod','su'] 63 | # features_list=['#','Attack','Defense','Generation','HP','Sp. Atk','Sp. Def','Speed','Total'] 64 | features_list=['age','job','marital','education','balance','housing','loan','contact','day','month','duration','campaign','pdays','previous','poutcome','deposit'] 65 | ) 66 | analysis.add_performance_metrics(metrics_list=['accuracy','precision','recall'], y_name = target) 67 | 68 | analysis.run(reference=df_ref, current=db_df, options={'ttest': {'alpha': 0.05, 'equal_var': False}, 'wasserstein': {'threshold' : 0.2}}) 69 | # print("analysis: ", analysis) 70 | df_result_drift = analysis.results_to_pandas() 71 | 72 | # print("df_result",df_result_drift) 73 | 74 | # print("prediction:",db_df['y_pred']) 75 | # print("ground truth:",db_df['Legendary']) 76 | # print("Lists of Legendary: ", db_df['Legendary'].astype(int).to_list()) 77 | # print("Lists of Prediction: ", db_df['y_pred'].to_list()) 78 | 79 | # roiInstance = ReturnOfInvestment() 80 | # roi_x = roiInstance.calculate_ROI(db_df['Legendary'].astype(int).to_list(), db_df['y_pred'].to_list()) 81 | # print("roi return: ", roi_x) 82 | 83 | # conversion_dict = {'yes': 1, 'no': 0} 84 | # reference_data = db_df[target].map(conversion_dict).tolist() 85 | reference_data = db_df[target].astype(int).tolist() 86 | current_data = db_df[prediction].tolist() 87 | 88 | cus = test_roi(metric_name = 'roi') 89 | cus.evaluate(current = current_data, reference = reference_data) 90 | df_roi = cus.get_result() 91 | # print("df_roi: ", df_roi) 92 | 93 | df_roi_pd = pd.DataFrame(df_roi) 94 | # print("df_roi_pd: ", df_roi_pd) 95 | 96 | # df_reshaped = pd.DataFrame([df_roi_pd[1].to_list()], columns=df_roi_pd[0]) 97 | # df_reshaped = df_roi_pd.pivot(index=None, columns=df_roi_pd.columns[0], values=df_roi_pd.columns[1]) 98 | df_reshaped = df_roi_pd.set_index(0).T 99 | df_reshaped["period_start"] = db_df.pred_timestamp.min() 100 | df_reshaped["period_end"] = db_df.pred_timestamp.max() 101 | df_reshaped["eval_timestamp"] = datetime.now() 102 | df_reshaped["model_id"] = df_ref["model_id"] 103 | df_reshaped["model_version"] = df_ref["model_version"] 104 | 105 | # print("df_reshaped: ", df_reshaped) 106 | 107 | df_result_drift = df_result_drift.append(df_reshaped, ignore_index=True) 108 | # print("df_result_drift: ", df_result_drift) 109 | 110 | 111 | 112 | # @CustomMetric 113 | # def test_roi(current, reference, multiple=3, **kwargs): 114 | # return multiple*np.max(current - reference) 115 | 116 | # cus = test_roi(metric_name = 'test') 117 | # cus.evaluate(current = current_data, reference = reference_data, threshold = 1, multiple=0.5) 118 | # df_roi = cus.get_result() 119 | # print("df_roi: ", df_roi) 120 | 121 | 122 | df_result_drift.set_index("eval_timestamp", inplace=True) 123 | 124 | df_result_drift["uuid"] = [uuid.uuid4() for _ in range(len(df_result_drift.index))] 125 | df_result_drift["period_start"] = df_result_drift["period_start"].dt.strftime('%Y-%m-%dT%H:%M:%SZ') 126 | df_result_drift["period_end"] = df_result_drift["period_end"].dt.strftime('%Y-%m-%dT%H:%M:%SZ') 127 | 128 | dat_capture.push_metrics(df_result_drift) 129 | 130 | print(f"Metrics pushed to the db") 131 | 132 | # Add the last period to db after pushing 133 | eval_timestamp_df = pd.DataFrame({"uuid": uuid.uuid4(), "timestamp": datetime.utcnow(), 134 | "eval_timestamp": datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')}, index=[0, ]) 135 | eval_timestamp_df.set_index("timestamp", inplace=True) 136 | dat_capture.push_eval_timestamp(eval_timestamp_df) 137 | print(f"Eval timestamp is updated in the db. Timestamp is: {eval_timestamp_df}") 138 | 139 | 140 | def result_to_pandas(self): 141 | result = None 142 | if self._results is not None: 143 | results = pd.DataFrame.from_records([self._results[i].dict() for i in range(len(self._results))]) 144 | for key, value in self._metadata.items(): 145 | if key not in ["name", "description"]: 146 | results[key] = value 147 | 148 | return result 149 | 150 | 151 | -------------------------------------------------------------------------------- /src/gen_data_and_simulate_drift.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from copulas.multivariate import GaussianMultivariate 3 | from enum import Enum 4 | import warnings 5 | from dataclasses import dataclass 6 | from typing import List, Dict 7 | import logging 8 | import random 9 | from sklearn.model_selection import train_test_split 10 | import numpy as np 11 | import pickle 12 | import os 13 | from pandas.api.types import is_numeric_dtype, is_object_dtype, is_bool_dtype 14 | PATH_REF_DATA_GDRIVE='https://drive.google.com/file/d/11rMPmaE_T-LFCNcb281PG611g38syCkI/view?usp=sharing' 15 | PATH_REF_DATA_GDRIVE='https://drive.google.com/uc?id=' + PATH_REF_DATA_GDRIVE.split('/')[-2] 16 | DATA_PATH = '/datasets/' 17 | MODELS_PATH = '/app/models/' 18 | # logging.getLogger().setLevel(logging.INFO) 19 | 20 | 21 | ''' 22 | :This module contains code for generating synthetic data and simulating data drift. 23 | ''' 24 | 25 | warnings.filterwarnings("ignore") 26 | 27 | class SamplingMethod(Enum): 28 | COPULAS_GAUSS_MULT = 1 29 | 30 | class DriftIntensity(Enum): 31 | MODERATE = 1, 32 | EXTREME = 2 33 | 34 | @dataclass 35 | class SampledData: 36 | df_ref_data: pd.DataFrame 37 | df_sampled: pd.DataFrame 38 | train_data:pd.DataFrame 39 | test_data:pd.DataFrame 40 | list_num_col: List 41 | used_distribution: SamplingMethod 42 | dict_col_type: Dict 43 | 44 | 45 | def create_dict_type_for_df(df_ref:pd.DataFrame): 46 | ''' 47 | :param df_ref: dataframe contains the data reference (original) 48 | :return: a dictionary (key = column name; value = type of column) 49 | ''' 50 | dict_col_type= {} 51 | for col in df_ref.columns: 52 | dict_col_type[col] = df_ref[col].dtype 53 | return dict_col_type 54 | 55 | def get_shift_coef(drift_intensity:DriftIntensity=DriftIntensity.MODERATE): 56 | ''' 57 | :param drift_intensity: enum with the level of intensity to be used 58 | :return: the new mean and stdev; used to drift features 59 | ''' 60 | 61 | if drift_intensity is DriftIntensity.MODERATE: 62 | mean_coef = random.uniform(1.2, 1.7) 63 | std_coef = random.uniform(1.5, 2) 64 | elif drift_intensity is DriftIntensity.EXTREME: 65 | mean_coef = random.uniform(1.5, 2.7) 66 | std_coef = random.uniform(2, 5) 67 | else: 68 | print('Unrecognized drift intensity in get_shift_coef()') 69 | 70 | return mean_coef, std_coef 71 | 72 | 73 | class GenerateFakeData(): 74 | def __init__(self, path_ref_data:str='pokemon.csv', 75 | sample_size = 1000, 76 | sampling_method= SamplingMethod.COPULAS_GAUSS_MULT, 77 | target = 'Legendary', 78 | model_name:str= ''): 79 | ''' 80 | 81 | :param path_ref_data: the dataset (reference) 82 | :param sample_size: number of samples to generate 83 | :param sampling_method: COPULAS method to be used to generate new data 84 | :param target: target of the dataset (we assume a supervised task) 85 | ''' 86 | self.df_ref = None 87 | self.num_cols = [] 88 | self.df_samples= pd.DataFrame({}) 89 | self.data_ref_target= target 90 | self.sampling_method = sampling_method 91 | self.sample_size = sample_size 92 | self.train_set = pd.DataFrame({}) 93 | self.test_set = pd.DataFrame({}) 94 | self.model_path = MODELS_PATH + model_name 95 | self.model_name = model_name 96 | 97 | if os.path.exists(self.model_path) and (self.model_path != MODELS_PATH): 98 | with open(self.model_path, 'rb') as f: 99 | self.model = pickle.load(f) 100 | else: 101 | self.model = {} 102 | 103 | self.path_ref_data=path_ref_data 104 | # logging.info('Read reference data...') 105 | self.read_data_reference() 106 | # logging.info('Keep numerical columns...') 107 | self.keep_numerical_col() 108 | self.dict_col_type = self.df_ref.dtypes 109 | 110 | # logging.info('Generate fake/synthetic data...') 111 | if self.sampling_method == SamplingMethod.COPULAS_GAUSS_MULT: 112 | self.generate_fake_data_using_copulas() 113 | # TODO add more distributions 114 | 115 | def keep_numerical_col(self): 116 | ''' 117 | keep only the numerical columns and target 118 | :return: the data ref with num cols and target 119 | ''' 120 | # logging.info(f'target:: {self.data_ref_target}') 121 | target_col = self.df_ref[self.data_ref_target] 122 | self.df_ref = self.df_ref.select_dtypes(['number']) 123 | self.num_cols = list(self.df_ref.columns) 124 | if self.data_ref_target in self.num_cols: 125 | self.num_cols.remove(self.data_ref_target) 126 | self.df_ref[self.data_ref_target]= target_col 127 | 128 | def read_data_reference(self): 129 | # logging.info('Inside reference data...') 130 | # logging.info(f'Trying to read {self.path_ref_data}') 131 | try: 132 | self.df_ref = pd.read_csv(self.path_ref_data) 133 | # pd_string=self.df_ref.head().to_string(index=False) 134 | # logging.info(f'dataframe:: {pd_string}') 135 | except FileNotFoundError: 136 | print("Wrong file or file path") 137 | 138 | def generate_fake_data_using_copulas(self): 139 | dist= None 140 | 141 | # generate new data using COPULAS_GAUSS_MULT method 142 | # TODO add more distribution option 143 | if self.sampling_method == SamplingMethod.COPULAS_GAUSS_MULT: 144 | dist = GaussianMultivariate(random_state = 1) 145 | 146 | # logging.info(f'Target for dataframe:: {self.df_ref.dtypes}') 147 | # logging.info(f'Target type is {self.df_ref[self.data_ref_target].dtypes}') 148 | # If target is numeric 149 | if (not is_object_dtype(self.df_ref[self.data_ref_target])) & (not is_bool_dtype(self.df_ref[self.data_ref_target])): 150 | # logging.info('numerical copulas...') 151 | if (len(self.model) == 0): 152 | dist.fit(self.df_ref) 153 | self.model = dist 154 | else: 155 | dist = self.model 156 | self.df_samples = dist.sample(self.sample_size) 157 | else: 158 | # logging.info('numerical copulas 2...') 159 | target_unique = self.df_ref[self.data_ref_target].unique() 160 | for a_target in target_unique: 161 | ## Only fit if model does not exists already 162 | if (len(self.model) == 0) or (a_target not in self.model.keys()): 163 | dist.fit(self.df_ref[self.df_ref[self.data_ref_target] == a_target].drop([self.data_ref_target], axis=1)) 164 | self.model[a_target] = dist 165 | else: 166 | dist = self.model[a_target] 167 | sampled = dist.sample(self.sample_size//len(target_unique)) 168 | sampled[self.data_ref_target] = a_target 169 | self.df_samples = self.df_samples.append(sampled) 170 | # Setting the right data types 171 | 172 | self.df_samples = self.df_samples.astype(self.dict_col_type) 173 | 174 | # logging.info(f'model path is {self.model_path}') 175 | if (len(self.model) !=0) & (self.model_name != ''): 176 | with open (self.model_path, 'wb') as f: 177 | pickle.dump(self.model, f) 178 | 179 | def get_dataclass_sampling(self): 180 | ''' 181 | 182 | :return: data class with the relevant information/data 183 | ''' 184 | # logging.info('sampling...') 185 | df_train, df_test = train_test_split(self.df_samples, test_size=0.2, stratify = self.df_samples[self.data_ref_target]) 186 | return SampledData(df_ref_data= self.df_ref, 187 | df_sampled=self.df_samples, 188 | list_num_col=self.num_cols, 189 | used_distribution=self.sampling_method, 190 | train_data=df_train, 191 | test_data=df_test, 192 | dict_col_type=self.dict_col_type 193 | ) 194 | 195 | 196 | class DriftSimulator(): 197 | ''' 198 | class used to drift the 'selected'(random or specified) columns. 199 | 200 | there is two options: 201 | 1) give the number N of columns to be drifted; and then select randomly these N columns. 202 | 2) give a list of the columns to be drifted 203 | 204 | # TODO specify for each column the intensity of the drift (MEDIUM or EXTREME) 205 | ''' 206 | def __init__(self, sampled_data:SampledData, 207 | nb_cols_to_drift:int, 208 | drift_intensity:DriftIntensity = DriftIntensity.MODERATE, 209 | selected_columns_to_drift = []): 210 | self.input_data = sampled_data 211 | self.selected_columns_to_drift= selected_columns_to_drift 212 | self.nb_col_to_drift = nb_cols_to_drift 213 | self.drift_intensity = drift_intensity 214 | self.test_data_drifted = self.input_data.test_data.copy() 215 | 216 | self.nb_col_to_drift = min(len(self.input_data.list_num_col), nb_cols_to_drift) 217 | print(f'number of columns to drift is : {self.nb_col_to_drift}') 218 | 219 | if len(self.selected_columns_to_drift) == 0: 220 | print('select random column to drift ...') 221 | self.selected_columns_to_drift = random.sample(self.input_data.list_num_col, 222 | self.nb_col_to_drift) 223 | 224 | self.run_drifting() 225 | 226 | 227 | def run_drifting(self): 228 | 229 | size_test_data = self.test_data_drifted.shape[0] 230 | for col in self.selected_columns_to_drift: 231 | print(f'Drifting column {col}') 232 | mean = self.test_data_drifted[col].mean() 233 | std = self.test_data_drifted[col].std() 234 | # get the coefs for the to be applied for the mean and the std for the actual distribution 235 | # new_mean = actual_mean * coef_mean (obtained randomly, based on the intensity) 236 | # new_std = actual_std * coef_std (obtained randomly, based on the intensity) 237 | coef_mean, coef_std = get_shift_coef(self.drift_intensity) 238 | 239 | # apply drifting to the test set using the new distribution 240 | # TODO add more distribution option 241 | drifted_data = np.random.normal(mean*coef_mean, std*coef_std, size_test_data) 242 | self.test_data_drifted[col] = drifted_data 243 | 244 | #Added for ROI 245 | #self.test_data_drifted[f'Ground Truth_{col}'] = self.input_data.test_data[col] 246 | 247 | def get_test_data_drifted(self): 248 | self.test_data_drifted = self.test_data_drifted.astype(self.input_data.test_data.dtypes) 249 | return self.test_data_drifted 250 | 251 | if __name__ == '__main__': 252 | ''' 253 | Example how to simulate drift 254 | pokemon dataset is used per default 255 | ''' 256 | genertor_fake_data = GenerateFakeData() 257 | samplet_data = genertor_fake_data.get_dataclass_sampling() 258 | ds = DriftSimulator(samplet_data, nb_cols_to_drift=1, drift_intensity=DriftIntensity.MODERATE) 259 | a = ds.get_test_data_drifted() 260 | 261 | 262 | -------------------------------------------------------------------------------- /dashboards/metrics/drift_summary_metrics.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "grafana", 8 | "uid": "-- Grafana --" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "editable": true, 25 | "fiscalYearStartMonth": 0, 26 | "graphTooltip": 0, 27 | "id": 4, 28 | "links": [], 29 | "liveNow": false, 30 | "panels": [ 31 | { 32 | "datasource": { 33 | "type": "influxdb", 34 | "uid": "PB6B4F2F1C736D27A" 35 | }, 36 | "gridPos": { 37 | "h": 8, 38 | "w": 4, 39 | "x": 0, 40 | "y": 0 41 | }, 42 | "id": 4, 43 | "options": { 44 | "code": { 45 | "language": "plaintext", 46 | "showLineNumbers": false, 47 | "showMiniMap": false 48 | }, 49 | "content": "", 50 | "mode": "html" 51 | }, 52 | "pluginVersion": "9.3.2", 53 | "transparent": true, 54 | "type": "text" 55 | }, 56 | { 57 | "datasource": { 58 | "type": "influxdb", 59 | "uid": "PB6B4F2F1C736D27A" 60 | }, 61 | "fieldConfig": { 62 | "defaults": { 63 | "color": { 64 | "mode": "thresholds" 65 | }, 66 | "custom": { 67 | "align": "auto", 68 | "displayMode": "auto", 69 | "inspect": false 70 | }, 71 | "mappings": [], 72 | "thresholds": { 73 | "mode": "absolute", 74 | "steps": [ 75 | { 76 | "color": "green", 77 | "value": null 78 | }, 79 | { 80 | "color": "red", 81 | "value": 80 82 | } 83 | ] 84 | } 85 | }, 86 | "overrides": [] 87 | }, 88 | "gridPos": { 89 | "h": 23, 90 | "w": 20, 91 | "x": 4, 92 | "y": 0 93 | }, 94 | "id": 2, 95 | "options": { 96 | "footer": { 97 | "enablePagination": true, 98 | "fields": "", 99 | "reducer": [ 100 | "sum" 101 | ], 102 | "show": false 103 | }, 104 | "frameIndex": 1, 105 | "showHeader": true 106 | }, 107 | "pluginVersion": "9.3.2", 108 | "targets": [ 109 | { 110 | "datasource": { 111 | "type": "influxdb", 112 | "uid": "PB6B4F2F1C736D27A" 113 | }, 114 | "hide": false, 115 | "query": "SELECT feature, value FROM metrics where type = 'drift' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version'", 116 | "rawQuery": true, 117 | "refId": "A", 118 | "resultFormat": "table" 119 | } 120 | ], 121 | "title": "Feature Summary", 122 | "transformations": [ 123 | { 124 | "id": "extractFields", 125 | "options": { 126 | "format": "auto", 127 | "replace": false, 128 | "source": "value" 129 | } 130 | }, 131 | { 132 | "id": "groupBy", 133 | "options": { 134 | "fields": { 135 | "feature": { 136 | "aggregations": [], 137 | "operation": "groupby" 138 | }, 139 | "value": { 140 | "aggregations": [ 141 | "max", 142 | "min", 143 | "range", 144 | "distinctCount", 145 | "changeCount" 146 | ], 147 | "operation": "aggregate" 148 | } 149 | } 150 | } 151 | }, 152 | { 153 | "id": "calculateField", 154 | "options": { 155 | "alias": "ChangeAmt", 156 | "binary": { 157 | "left": "value (range)", 158 | "operator": "/", 159 | "reducer": "sum", 160 | "right": "value (max)" 161 | }, 162 | "mode": "binary", 163 | "reduce": { 164 | "reducer": "sum" 165 | } 166 | } 167 | }, 168 | { 169 | "id": "calculateField", 170 | "options": { 171 | "alias": "FeatureChange%", 172 | "binary": { 173 | "left": "ChangeAmt", 174 | "operator": "*", 175 | "reducer": "sum", 176 | "right": "100" 177 | }, 178 | "mode": "binary", 179 | "reduce": { 180 | "reducer": "sum" 181 | } 182 | } 183 | }, 184 | { 185 | "id": "organize", 186 | "options": { 187 | "excludeByName": { 188 | "ChangeAmt": true, 189 | "value (range)": true 190 | }, 191 | "indexByName": { 192 | "ChangeAmt": 7, 193 | "FeatureChange%": 4, 194 | "feature": 0, 195 | "value (changeCount)": 6, 196 | "value (distinctCount)": 5, 197 | "value (max)": 1, 198 | "value (min)": 2, 199 | "value (range)": 3 200 | }, 201 | "renameByName": { 202 | "feature": "Feature", 203 | "value (changeCount)": "ChangeCount", 204 | "value (distinctCount)": "DistinctCount", 205 | "value (max)": "Max", 206 | "value (min)": "Min" 207 | } 208 | } 209 | } 210 | ], 211 | "transparent": true, 212 | "type": "table" 213 | } 214 | ], 215 | "refresh": false, 216 | "schemaVersion": 37, 217 | "style": "dark", 218 | "tags": [], 219 | "templating": { 220 | "list": [ 221 | { 222 | "current": { 223 | "selected": false, 224 | "text": "CvM", 225 | "value": "CvM" 226 | }, 227 | "datasource": { 228 | "type": "influxdb", 229 | "uid": "PB6B4F2F1C736D27A" 230 | }, 231 | "definition": "SELECT distinct(\"metric_name\") FROM metrics where type = '$type'", 232 | "hide": 0, 233 | "includeAll": false, 234 | "multi": false, 235 | "name": "metric_name", 236 | "options": [], 237 | "query": "SELECT distinct(\"metric_name\") FROM metrics where type = '$type'", 238 | "refresh": 1, 239 | "regex": "", 240 | "skipUrlSync": false, 241 | "sort": 0, 242 | "type": "query" 243 | }, 244 | { 245 | "current": { 246 | "selected": false, 247 | "text": "1", 248 | "value": "1" 249 | }, 250 | "datasource": { 251 | "type": "influxdb", 252 | "uid": "PB6B4F2F1C736D27A" 253 | }, 254 | "definition": "SELECT distinct(\"model_id\") FROM metrics where type = '$type'", 255 | "hide": 0, 256 | "includeAll": false, 257 | "multi": false, 258 | "name": "model_id", 259 | "options": [], 260 | "query": "SELECT distinct(\"model_id\") FROM metrics where type = '$type'", 261 | "refresh": 1, 262 | "regex": "", 263 | "skipUrlSync": false, 264 | "sort": 0, 265 | "type": "query" 266 | }, 267 | { 268 | "current": { 269 | "selected": false, 270 | "text": "2", 271 | "value": "2" 272 | }, 273 | "datasource": { 274 | "type": "influxdb", 275 | "uid": "PB6B4F2F1C736D27A" 276 | }, 277 | "definition": "SELECT distinct(\"model_version\") FROM metrics where type = '$type'", 278 | "hide": 0, 279 | "includeAll": false, 280 | "multi": false, 281 | "name": "model_version", 282 | "options": [], 283 | "query": "SELECT distinct(\"model_version\") FROM metrics where type = '$type'", 284 | "refresh": 1, 285 | "regex": "", 286 | "skipUrlSync": false, 287 | "sort": 0, 288 | "type": "query" 289 | }, 290 | { 291 | "hide": 2, 292 | "name": "type", 293 | "query": "drift", 294 | "skipUrlSync": false, 295 | "type": "constant" 296 | } 297 | ] 298 | }, 299 | "time": { 300 | "from": "now-2d", 301 | "to": "now" 302 | }, 303 | "timepicker": {}, 304 | "timezone": "", 305 | "title": "DriftSummaryDashboard", 306 | "uid": "663ynapVk", 307 | "version": 2, 308 | "weekStart": "" 309 | } -------------------------------------------------------------------------------- /grafana-provisioning/dashboards/drift_summary_dashboard.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "grafana", 8 | "uid": "-- Grafana --" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "editable": true, 25 | "fiscalYearStartMonth": 0, 26 | "graphTooltip": 0, 27 | "id": 4, 28 | "links": [], 29 | "liveNow": false, 30 | "panels": [ 31 | { 32 | "datasource": { 33 | "type": "influxdb", 34 | "uid": "PB6B4F2F1C736D27A" 35 | }, 36 | "gridPos": { 37 | "h": 8, 38 | "w": 4, 39 | "x": 0, 40 | "y": 0 41 | }, 42 | "id": 4, 43 | "options": { 44 | "code": { 45 | "language": "plaintext", 46 | "showLineNumbers": false, 47 | "showMiniMap": false 48 | }, 49 | "content": "", 50 | "mode": "html" 51 | }, 52 | "pluginVersion": "9.3.2", 53 | "transparent": true, 54 | "type": "text" 55 | }, 56 | { 57 | "datasource": { 58 | "type": "influxdb", 59 | "uid": "PB6B4F2F1C736D27A" 60 | }, 61 | "fieldConfig": { 62 | "defaults": { 63 | "color": { 64 | "mode": "thresholds" 65 | }, 66 | "custom": { 67 | "align": "auto", 68 | "displayMode": "auto", 69 | "inspect": false 70 | }, 71 | "mappings": [], 72 | "thresholds": { 73 | "mode": "absolute", 74 | "steps": [ 75 | { 76 | "color": "green", 77 | "value": null 78 | }, 79 | { 80 | "color": "red", 81 | "value": 80 82 | } 83 | ] 84 | } 85 | }, 86 | "overrides": [] 87 | }, 88 | "gridPos": { 89 | "h": 23, 90 | "w": 20, 91 | "x": 4, 92 | "y": 0 93 | }, 94 | "id": 2, 95 | "options": { 96 | "footer": { 97 | "enablePagination": true, 98 | "fields": "", 99 | "reducer": [ 100 | "sum" 101 | ], 102 | "show": false 103 | }, 104 | "frameIndex": 1, 105 | "showHeader": true 106 | }, 107 | "pluginVersion": "9.3.2", 108 | "targets": [ 109 | { 110 | "datasource": { 111 | "type": "influxdb", 112 | "uid": "PB6B4F2F1C736D27A" 113 | }, 114 | "hide": false, 115 | "query": "SELECT feature, value FROM metrics where type = 'drift' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version'", 116 | "rawQuery": true, 117 | "refId": "A", 118 | "resultFormat": "table" 119 | } 120 | ], 121 | "title": "Feature Summary", 122 | "transformations": [ 123 | { 124 | "id": "extractFields", 125 | "options": { 126 | "format": "auto", 127 | "replace": false, 128 | "source": "value" 129 | } 130 | }, 131 | { 132 | "id": "groupBy", 133 | "options": { 134 | "fields": { 135 | "feature": { 136 | "aggregations": [], 137 | "operation": "groupby" 138 | }, 139 | "value": { 140 | "aggregations": [ 141 | "max", 142 | "min", 143 | "range", 144 | "distinctCount", 145 | "changeCount" 146 | ], 147 | "operation": "aggregate" 148 | } 149 | } 150 | } 151 | }, 152 | { 153 | "id": "calculateField", 154 | "options": { 155 | "alias": "ChangeAmt", 156 | "binary": { 157 | "left": "value (range)", 158 | "operator": "/", 159 | "reducer": "sum", 160 | "right": "value (max)" 161 | }, 162 | "mode": "binary", 163 | "reduce": { 164 | "reducer": "sum" 165 | } 166 | } 167 | }, 168 | { 169 | "id": "calculateField", 170 | "options": { 171 | "alias": "FeatureChange%", 172 | "binary": { 173 | "left": "ChangeAmt", 174 | "operator": "*", 175 | "reducer": "sum", 176 | "right": "100" 177 | }, 178 | "mode": "binary", 179 | "reduce": { 180 | "reducer": "sum" 181 | } 182 | } 183 | }, 184 | { 185 | "id": "organize", 186 | "options": { 187 | "excludeByName": { 188 | "ChangeAmt": true, 189 | "value (range)": true 190 | }, 191 | "indexByName": { 192 | "ChangeAmt": 7, 193 | "FeatureChange%": 4, 194 | "feature": 0, 195 | "value (changeCount)": 6, 196 | "value (distinctCount)": 5, 197 | "value (max)": 1, 198 | "value (min)": 2, 199 | "value (range)": 3 200 | }, 201 | "renameByName": { 202 | "feature": "Feature", 203 | "value (changeCount)": "ChangeCount", 204 | "value (distinctCount)": "DistinctCount", 205 | "value (max)": "Max", 206 | "value (min)": "Min" 207 | } 208 | } 209 | } 210 | ], 211 | "transparent": true, 212 | "type": "table" 213 | } 214 | ], 215 | "refresh": false, 216 | "schemaVersion": 37, 217 | "style": "dark", 218 | "tags": [], 219 | "templating": { 220 | "list": [ 221 | { 222 | "current": { 223 | "selected": false, 224 | "text": "CvM", 225 | "value": "CvM" 226 | }, 227 | "datasource": { 228 | "type": "influxdb", 229 | "uid": "PB6B4F2F1C736D27A" 230 | }, 231 | "definition": "SELECT distinct(\"metric_name\") FROM metrics where type = '$type'", 232 | "hide": 0, 233 | "includeAll": false, 234 | "multi": false, 235 | "name": "metric_name", 236 | "options": [], 237 | "query": "SELECT distinct(\"metric_name\") FROM metrics where type = '$type'", 238 | "refresh": 1, 239 | "regex": "", 240 | "skipUrlSync": false, 241 | "sort": 0, 242 | "type": "query" 243 | }, 244 | { 245 | "current": { 246 | "selected": false, 247 | "text": "1", 248 | "value": "1" 249 | }, 250 | "datasource": { 251 | "type": "influxdb", 252 | "uid": "PB6B4F2F1C736D27A" 253 | }, 254 | "definition": "SELECT distinct(\"model_id\") FROM metrics where type = '$type'", 255 | "hide": 0, 256 | "includeAll": false, 257 | "multi": false, 258 | "name": "model_id", 259 | "options": [], 260 | "query": "SELECT distinct(\"model_id\") FROM metrics where type = '$type'", 261 | "refresh": 1, 262 | "regex": "", 263 | "skipUrlSync": false, 264 | "sort": 0, 265 | "type": "query" 266 | }, 267 | { 268 | "current": { 269 | "selected": false, 270 | "text": "2", 271 | "value": "2" 272 | }, 273 | "datasource": { 274 | "type": "influxdb", 275 | "uid": "PB6B4F2F1C736D27A" 276 | }, 277 | "definition": "SELECT distinct(\"model_version\") FROM metrics where type = '$type'", 278 | "hide": 0, 279 | "includeAll": false, 280 | "multi": false, 281 | "name": "model_version", 282 | "options": [], 283 | "query": "SELECT distinct(\"model_version\") FROM metrics where type = '$type'", 284 | "refresh": 1, 285 | "regex": "", 286 | "skipUrlSync": false, 287 | "sort": 0, 288 | "type": "query" 289 | }, 290 | { 291 | "hide": 2, 292 | "name": "type", 293 | "query": "drift", 294 | "skipUrlSync": false, 295 | "type": "constant" 296 | } 297 | ] 298 | }, 299 | "time": { 300 | "from": "now-2d", 301 | "to": "now" 302 | }, 303 | "timepicker": {}, 304 | "timezone": "", 305 | "title": "DriftSummaryDashboard", 306 | "uid": "663ynapVk", 307 | "version": 2, 308 | "weekStart": "" 309 | } -------------------------------------------------------------------------------- /dashboards/metrics/roi_dashboard.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "grafana", 8 | "uid": "-- Grafana --" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "type": "dashboard" 15 | } 16 | ] 17 | }, 18 | "description": "Dashboard for ROI display", 19 | "editable": true, 20 | "fiscalYearStartMonth": 0, 21 | "graphTooltip": 0, 22 | "id": 5, 23 | "links": [], 24 | "liveNow": false, 25 | "panels": [ 26 | { 27 | "datasource": { 28 | "type": "influxdb", 29 | "uid": "PB6B4F2F1C736D27A" 30 | }, 31 | "gridPos": { 32 | "h": 7, 33 | "w": 4, 34 | "x": 0, 35 | "y": 0 36 | }, 37 | "id": 1, 38 | "options": { 39 | "code": { 40 | "language": "plaintext", 41 | "showLineNumbers": false, 42 | "showMiniMap": false 43 | }, 44 | "content": "", 45 | "mode": "html" 46 | }, 47 | "pluginVersion": "10.1.0", 48 | "targets": [ 49 | { 50 | "datasource": { 51 | "type": "influxdb", 52 | "uid": "PB6B4F2F1C736D27A" 53 | }, 54 | "query": "SELECT value FROM metrics where type = '$type' and feature = '$feature_name' and metric_name = 'CvM' and model_id = '$model_id' and model_version = '$model_version'", 55 | "rawQuery": true, 56 | "refId": "A", 57 | "resultFormat": "time_series" 58 | } 59 | ], 60 | "transparent": true, 61 | "type": "text" 62 | }, 63 | { 64 | "datasource": { 65 | "type": "influxdb", 66 | "uid": "PB6B4F2F1C736D27A" 67 | }, 68 | "description": "ROI metric", 69 | "fieldConfig": { 70 | "defaults": { 71 | "color": { 72 | "fixedColor": "purple", 73 | "mode": "fixed" 74 | }, 75 | "custom": { 76 | "axisCenteredZero": false, 77 | "axisColorMode": "text", 78 | "axisLabel": "", 79 | "axisPlacement": "auto", 80 | "fillOpacity": 80, 81 | "gradientMode": "none", 82 | "hideFrom": { 83 | "legend": false, 84 | "tooltip": false, 85 | "viz": false 86 | }, 87 | "lineWidth": 1, 88 | "scaleDistribution": { 89 | "type": "linear" 90 | }, 91 | "thresholdsStyle": { 92 | "mode": "off" 93 | } 94 | }, 95 | "links": [], 96 | "mappings": [], 97 | "thresholds": { 98 | "mode": "absolute", 99 | "steps": [ 100 | { 101 | "color": "green", 102 | "value": null 103 | }, 104 | { 105 | "color": "red", 106 | "value": 80 107 | } 108 | ] 109 | }, 110 | "unit": "currencyUSD" 111 | }, 112 | "overrides": [] 113 | }, 114 | "gridPos": { 115 | "h": 12, 116 | "w": 20, 117 | "x": 4, 118 | "y": 0 119 | }, 120 | "id": 2, 121 | "options": { 122 | "barRadius": 0.2, 123 | "barWidth": 0.96, 124 | "fullHighlight": false, 125 | "groupWidth": 0.7, 126 | "legend": { 127 | "calcs": [], 128 | "displayMode": "list", 129 | "placement": "bottom", 130 | "showLegend": true 131 | }, 132 | "orientation": "auto", 133 | "showValue": "auto", 134 | "stacking": "none", 135 | "tooltip": { 136 | "mode": "single", 137 | "sort": "none" 138 | }, 139 | "xTickLabelRotation": 0, 140 | "xTickLabelSpacing": 0 141 | }, 142 | "pluginVersion": "9.1.6", 143 | "targets": [ 144 | { 145 | "alias": "Return of Investment (in $)", 146 | "datasource": { 147 | "type": "influxdb", 148 | "uid": "PB6B4F2F1C736D27A" 149 | }, 150 | "query": "SELECT metric_value FROM metrics where metric_name = 'roi' and model_id = '$model_id' and model_version = '$model_version' and $timeFilter", 151 | "rawQuery": true, 152 | "refId": "A", 153 | "resultFormat": "time_series" 154 | } 155 | ], 156 | "title": "ROI", 157 | "transparent": true, 158 | "type": "barchart" 159 | }, 160 | { 161 | "datasource": { 162 | "type": "influxdb", 163 | "uid": "PB6B4F2F1C736D27A" 164 | }, 165 | "description": "Confusion matrix showing the breakdown of TP, TN, FP, FN values", 166 | "fieldConfig": { 167 | "defaults": { 168 | "color": { 169 | "mode": "palette-classic" 170 | }, 171 | "custom": { 172 | "hideFrom": { 173 | "legend": false, 174 | "tooltip": false, 175 | "viz": false 176 | } 177 | }, 178 | "mappings": [] 179 | }, 180 | "overrides": [] 181 | }, 182 | "gridPos": { 183 | "h": 10, 184 | "w": 12, 185 | "x": 0, 186 | "y": 12 187 | }, 188 | "id": 3, 189 | "options": { 190 | "displayLabels": [ 191 | "percent" 192 | ], 193 | "legend": { 194 | "displayMode": "list", 195 | "placement": "right", 196 | "showLegend": true, 197 | "values": [] 198 | }, 199 | "pieType": "pie", 200 | "reduceOptions": { 201 | "calcs": [ 202 | "lastNotNull" 203 | ], 204 | "fields": "", 205 | "values": false 206 | }, 207 | "tooltip": { 208 | "mode": "single", 209 | "sort": "none" 210 | } 211 | }, 212 | "targets": [ 213 | { 214 | "alias": "True Positive", 215 | "datasource": { 216 | "type": "influxdb", 217 | "uid": "PB6B4F2F1C736D27A" 218 | }, 219 | "groupBy": [ 220 | { 221 | "params": [ 222 | "$__interval" 223 | ], 224 | "type": "time" 225 | }, 226 | { 227 | "params": [ 228 | "null" 229 | ], 230 | "type": "fill" 231 | } 232 | ], 233 | "orderByTime": "ASC", 234 | "policy": "autogen", 235 | "query": "SELECT count(\"y_pred\") FROM prediction where \"y_pred\"=1 and \"default\"=1 and $timeFilter", 236 | "rawQuery": true, 237 | "refId": "A", 238 | "resultFormat": "time_series", 239 | "select": [ 240 | [ 241 | { 242 | "params": [ 243 | "value" 244 | ], 245 | "type": "field" 246 | }, 247 | { 248 | "params": [], 249 | "type": "mean" 250 | } 251 | ] 252 | ], 253 | "tags": [] 254 | }, 255 | { 256 | "alias": "True Negative", 257 | "datasource": { 258 | "type": "influxdb", 259 | "uid": "PB6B4F2F1C736D27A" 260 | }, 261 | "groupBy": [ 262 | { 263 | "params": [ 264 | "$__interval" 265 | ], 266 | "type": "time" 267 | }, 268 | { 269 | "params": [ 270 | "null" 271 | ], 272 | "type": "fill" 273 | } 274 | ], 275 | "hide": false, 276 | "orderByTime": "ASC", 277 | "policy": "autogen", 278 | "query": "SELECT count(\"y_pred\") FROM prediction where \"y_pred\"=0 and \"default\"=0 and $timeFilter", 279 | "rawQuery": true, 280 | "refId": "B", 281 | "resultFormat": "time_series", 282 | "select": [ 283 | [ 284 | { 285 | "params": [ 286 | "value" 287 | ], 288 | "type": "field" 289 | }, 290 | { 291 | "params": [], 292 | "type": "mean" 293 | } 294 | ] 295 | ], 296 | "tags": [] 297 | }, 298 | { 299 | "alias": "False Positive", 300 | "datasource": { 301 | "type": "influxdb", 302 | "uid": "PB6B4F2F1C736D27A" 303 | }, 304 | "groupBy": [ 305 | { 306 | "params": [ 307 | "$__interval" 308 | ], 309 | "type": "time" 310 | }, 311 | { 312 | "params": [ 313 | "null" 314 | ], 315 | "type": "fill" 316 | } 317 | ], 318 | "hide": false, 319 | "orderByTime": "ASC", 320 | "policy": "autogen", 321 | "query": "SELECT count(\"y_pred\") FROM prediction where \"y_pred\"=1 and \"default\"=0 and $timeFilter", 322 | "rawQuery": true, 323 | "refId": "C", 324 | "resultFormat": "time_series", 325 | "select": [ 326 | [ 327 | { 328 | "params": [ 329 | "value" 330 | ], 331 | "type": "field" 332 | }, 333 | { 334 | "params": [], 335 | "type": "mean" 336 | } 337 | ] 338 | ], 339 | "tags": [] 340 | }, 341 | { 342 | "alias": "False Negative", 343 | "datasource": { 344 | "type": "influxdb", 345 | "uid": "PB6B4F2F1C736D27A" 346 | }, 347 | "groupBy": [ 348 | { 349 | "params": [ 350 | "$__interval" 351 | ], 352 | "type": "time" 353 | }, 354 | { 355 | "params": [ 356 | "null" 357 | ], 358 | "type": "fill" 359 | } 360 | ], 361 | "hide": false, 362 | "orderByTime": "ASC", 363 | "policy": "autogen", 364 | "query": "SELECT count(\"y_pred\") FROM prediction where \"y_pred\"=0 and \"default\"=1 and $timeFilter", 365 | "rawQuery": true, 366 | "refId": "D", 367 | "resultFormat": "time_series", 368 | "select": [ 369 | [ 370 | { 371 | "params": [ 372 | "value" 373 | ], 374 | "type": "field" 375 | }, 376 | { 377 | "params": [], 378 | "type": "mean" 379 | } 380 | ] 381 | ], 382 | "tags": [] 383 | } 384 | ], 385 | "title": "Confusion Matrix", 386 | "transparent": true, 387 | "type": "piechart" 388 | }, 389 | { 390 | "datasource": { 391 | "type": "influxdb", 392 | "uid": "PB6B4F2F1C736D27A" 393 | }, 394 | "description": "Performance matrices for getting a better understanding of how ROI is co-related with different parameters.", 395 | "fieldConfig": { 396 | "defaults": { 397 | "color": { 398 | "mode": "palette-classic" 399 | }, 400 | "custom": { 401 | "axisCenteredZero": false, 402 | "axisColorMode": "text", 403 | "axisLabel": "", 404 | "axisPlacement": "auto", 405 | "barAlignment": 0, 406 | "drawStyle": "line", 407 | "fillOpacity": 15, 408 | "gradientMode": "hue", 409 | "hideFrom": { 410 | "legend": false, 411 | "tooltip": false, 412 | "viz": false 413 | }, 414 | "insertNulls": false, 415 | "lineInterpolation": "smooth", 416 | "lineWidth": 2, 417 | "pointSize": 5, 418 | "scaleDistribution": { 419 | "type": "linear" 420 | }, 421 | "showPoints": "auto", 422 | "spanNulls": false, 423 | "stacking": { 424 | "group": "A", 425 | "mode": "none" 426 | }, 427 | "thresholdsStyle": { 428 | "mode": "off" 429 | } 430 | }, 431 | "mappings": [], 432 | "thresholds": { 433 | "mode": "absolute", 434 | "steps": [ 435 | { 436 | "color": "green", 437 | "value": null 438 | }, 439 | { 440 | "color": "red", 441 | "value": 80 442 | } 443 | ] 444 | }, 445 | "unit": "percentunit" 446 | }, 447 | "overrides": [] 448 | }, 449 | "gridPos": { 450 | "h": 10, 451 | "w": 12, 452 | "x": 12, 453 | "y": 12 454 | }, 455 | "id": 4, 456 | "options": { 457 | "legend": { 458 | "calcs": [], 459 | "displayMode": "list", 460 | "placement": "bottom", 461 | "showLegend": true 462 | }, 463 | "tooltip": { 464 | "mode": "single", 465 | "sort": "none" 466 | } 467 | }, 468 | "targets": [ 469 | { 470 | "alias": "Accuracy", 471 | "datasource": { 472 | "type": "influxdb", 473 | "uid": "PB6B4F2F1C736D27A" 474 | }, 475 | "groupBy": [ 476 | { 477 | "params": [ 478 | "$__interval" 479 | ], 480 | "type": "time" 481 | }, 482 | { 483 | "params": [ 484 | "null" 485 | ], 486 | "type": "fill" 487 | } 488 | ], 489 | "orderByTime": "ASC", 490 | "policy": "autogen", 491 | "query": "SELECT metric_value FROM metrics where metric_name = 'accuracy' and model_id = '$model_id' and model_version = '$model_version'", 492 | "rawQuery": true, 493 | "refId": "A", 494 | "resultFormat": "time_series", 495 | "select": [ 496 | [ 497 | { 498 | "params": [ 499 | "value" 500 | ], 501 | "type": "field" 502 | }, 503 | { 504 | "params": [], 505 | "type": "mean" 506 | } 507 | ] 508 | ], 509 | "tags": [] 510 | }, 511 | { 512 | "alias": "Precision", 513 | "datasource": { 514 | "type": "influxdb", 515 | "uid": "PB6B4F2F1C736D27A" 516 | }, 517 | "groupBy": [ 518 | { 519 | "params": [ 520 | "$__interval" 521 | ], 522 | "type": "time" 523 | }, 524 | { 525 | "params": [ 526 | "null" 527 | ], 528 | "type": "fill" 529 | } 530 | ], 531 | "hide": false, 532 | "orderByTime": "ASC", 533 | "policy": "autogen", 534 | "query": "SELECT metric_value FROM metrics where metric_name = 'precision' and model_id = '$model_id' and model_version = '$model_version'", 535 | "rawQuery": true, 536 | "refId": "B", 537 | "resultFormat": "time_series", 538 | "select": [ 539 | [ 540 | { 541 | "params": [ 542 | "value" 543 | ], 544 | "type": "field" 545 | }, 546 | { 547 | "params": [], 548 | "type": "mean" 549 | } 550 | ] 551 | ], 552 | "tags": [] 553 | }, 554 | { 555 | "alias": "Recall", 556 | "datasource": { 557 | "type": "influxdb", 558 | "uid": "PB6B4F2F1C736D27A" 559 | }, 560 | "groupBy": [ 561 | { 562 | "params": [ 563 | "$__interval" 564 | ], 565 | "type": "time" 566 | }, 567 | { 568 | "params": [ 569 | "null" 570 | ], 571 | "type": "fill" 572 | } 573 | ], 574 | "hide": false, 575 | "orderByTime": "ASC", 576 | "policy": "autogen", 577 | "query": "SELECT metric_value FROM metrics where metric_name = 'recall' and model_id = '$model_id' and model_version = '$model_version'", 578 | "rawQuery": true, 579 | "refId": "C", 580 | "resultFormat": "time_series", 581 | "select": [ 582 | [ 583 | { 584 | "params": [ 585 | "value" 586 | ], 587 | "type": "field" 588 | }, 589 | { 590 | "params": [], 591 | "type": "mean" 592 | } 593 | ] 594 | ], 595 | "tags": [] 596 | } 597 | ], 598 | "title": "Performance Metrics", 599 | "transparent": true, 600 | "type": "timeseries" 601 | } 602 | ], 603 | "refresh": false, 604 | "schemaVersion": 38, 605 | "style": "dark", 606 | "tags": [], 607 | "templating": { 608 | "list": [ 609 | { 610 | "current": { 611 | "selected": false, 612 | "text": "custom", 613 | "value": "custom" 614 | }, 615 | "hide": 0, 616 | "includeAll": false, 617 | "label": "metric type", 618 | "multi": false, 619 | "name": "metric_type", 620 | "options": [ 621 | { 622 | "selected": true, 623 | "text": "custom", 624 | "value": "custom" 625 | } 626 | ], 627 | "query": "custom", 628 | "skipUrlSync": false, 629 | "type": "custom" 630 | }, 631 | { 632 | "current": { 633 | "selected": false, 634 | "text": "1", 635 | "value": "1" 636 | }, 637 | "datasource": { 638 | "type": "influxdb", 639 | "uid": "PB6B4F2F1C736D27A" 640 | }, 641 | "definition": "SELECT distinct(\"model_id\") FROM metrics where metric_type = '$metric_type'", 642 | "hide": 0, 643 | "includeAll": false, 644 | "label": "model id", 645 | "multi": false, 646 | "name": "model_id", 647 | "options": [], 648 | "query": "SELECT distinct(\"model_id\") FROM metrics where metric_type = '$metric_type'", 649 | "refresh": 1, 650 | "regex": "", 651 | "skipUrlSync": false, 652 | "sort": 0, 653 | "type": "query" 654 | }, 655 | { 656 | "current": { 657 | "selected": false, 658 | "text": "2", 659 | "value": "2" 660 | }, 661 | "datasource": { 662 | "type": "influxdb", 663 | "uid": "PB6B4F2F1C736D27A" 664 | }, 665 | "definition": "SELECT distinct(\"model_version\") FROM metrics where metric_type = '$metric_type'", 666 | "hide": 0, 667 | "includeAll": false, 668 | "label": "model version", 669 | "multi": false, 670 | "name": "model_version", 671 | "options": [], 672 | "query": "SELECT distinct(\"model_version\") FROM metrics where metric_type = '$metric_type'", 673 | "refresh": 1, 674 | "regex": "", 675 | "skipUrlSync": false, 676 | "sort": 0, 677 | "type": "query" 678 | } 679 | ] 680 | }, 681 | "time": { 682 | "from": "2023-08-29T16:21:59.212Z", 683 | "to": "2023-08-30T16:21:59.212Z" 684 | }, 685 | "timepicker": {}, 686 | "timezone": "", 687 | "title": "ROI-dashboard", 688 | "uid": "b93ee885-457d-4298-b2d6-563bcb2efb42", 689 | "version": 6, 690 | "weekStart": "" 691 | } -------------------------------------------------------------------------------- /grafana-provisioning/dashboards/roi_dashboard.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "grafana", 8 | "uid": "-- Grafana --" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "type": "dashboard" 15 | } 16 | ] 17 | }, 18 | "description": "Dashboard for ROI display", 19 | "editable": true, 20 | "fiscalYearStartMonth": 0, 21 | "graphTooltip": 0, 22 | "id": 5, 23 | "links": [], 24 | "liveNow": false, 25 | "panels": [ 26 | { 27 | "datasource": { 28 | "type": "influxdb", 29 | "uid": "PB6B4F2F1C736D27A" 30 | }, 31 | "gridPos": { 32 | "h": 7, 33 | "w": 4, 34 | "x": 0, 35 | "y": 0 36 | }, 37 | "id": 1, 38 | "options": { 39 | "code": { 40 | "language": "plaintext", 41 | "showLineNumbers": false, 42 | "showMiniMap": false 43 | }, 44 | "content": "", 45 | "mode": "html" 46 | }, 47 | "pluginVersion": "10.1.0", 48 | "targets": [ 49 | { 50 | "datasource": { 51 | "type": "influxdb", 52 | "uid": "PB6B4F2F1C736D27A" 53 | }, 54 | "query": "SELECT value FROM metrics where type = '$type' and feature = '$feature_name' and metric_name = 'CvM' and model_id = '$model_id' and model_version = '$model_version'", 55 | "rawQuery": true, 56 | "refId": "A", 57 | "resultFormat": "time_series" 58 | } 59 | ], 60 | "transparent": true, 61 | "type": "text" 62 | }, 63 | { 64 | "datasource": { 65 | "type": "influxdb", 66 | "uid": "PB6B4F2F1C736D27A" 67 | }, 68 | "description": "ROI metric", 69 | "fieldConfig": { 70 | "defaults": { 71 | "color": { 72 | "fixedColor": "purple", 73 | "mode": "fixed" 74 | }, 75 | "custom": { 76 | "axisCenteredZero": false, 77 | "axisColorMode": "text", 78 | "axisLabel": "", 79 | "axisPlacement": "auto", 80 | "fillOpacity": 80, 81 | "gradientMode": "none", 82 | "hideFrom": { 83 | "legend": false, 84 | "tooltip": false, 85 | "viz": false 86 | }, 87 | "lineWidth": 1, 88 | "scaleDistribution": { 89 | "type": "linear" 90 | }, 91 | "thresholdsStyle": { 92 | "mode": "off" 93 | } 94 | }, 95 | "links": [], 96 | "mappings": [], 97 | "thresholds": { 98 | "mode": "absolute", 99 | "steps": [ 100 | { 101 | "color": "green", 102 | "value": null 103 | }, 104 | { 105 | "color": "red", 106 | "value": 80 107 | } 108 | ] 109 | }, 110 | "unit": "currencyUSD" 111 | }, 112 | "overrides": [] 113 | }, 114 | "gridPos": { 115 | "h": 12, 116 | "w": 20, 117 | "x": 4, 118 | "y": 0 119 | }, 120 | "id": 2, 121 | "options": { 122 | "barRadius": 0.2, 123 | "barWidth": 0.96, 124 | "fullHighlight": false, 125 | "groupWidth": 0.7, 126 | "legend": { 127 | "calcs": [], 128 | "displayMode": "list", 129 | "placement": "bottom", 130 | "showLegend": true 131 | }, 132 | "orientation": "auto", 133 | "showValue": "auto", 134 | "stacking": "none", 135 | "tooltip": { 136 | "mode": "single", 137 | "sort": "none" 138 | }, 139 | "xTickLabelRotation": 0, 140 | "xTickLabelSpacing": 0 141 | }, 142 | "pluginVersion": "9.1.6", 143 | "targets": [ 144 | { 145 | "alias": "Return of Investment (in $)", 146 | "datasource": { 147 | "type": "influxdb", 148 | "uid": "PB6B4F2F1C736D27A" 149 | }, 150 | "query": "SELECT metric_value FROM metrics where metric_name = 'roi' and model_id = '$model_id' and model_version = '$model_version' and $timeFilter", 151 | "rawQuery": true, 152 | "refId": "A", 153 | "resultFormat": "time_series" 154 | } 155 | ], 156 | "title": "ROI", 157 | "transparent": true, 158 | "type": "barchart" 159 | }, 160 | { 161 | "datasource": { 162 | "type": "influxdb", 163 | "uid": "PB6B4F2F1C736D27A" 164 | }, 165 | "description": "Confusion matrix showing the breakdown of TP, TN, FP, FN values", 166 | "fieldConfig": { 167 | "defaults": { 168 | "color": { 169 | "mode": "palette-classic" 170 | }, 171 | "custom": { 172 | "hideFrom": { 173 | "legend": false, 174 | "tooltip": false, 175 | "viz": false 176 | } 177 | }, 178 | "mappings": [] 179 | }, 180 | "overrides": [] 181 | }, 182 | "gridPos": { 183 | "h": 10, 184 | "w": 12, 185 | "x": 0, 186 | "y": 12 187 | }, 188 | "id": 3, 189 | "options": { 190 | "displayLabels": [ 191 | "percent" 192 | ], 193 | "legend": { 194 | "displayMode": "list", 195 | "placement": "right", 196 | "showLegend": true, 197 | "values": [] 198 | }, 199 | "pieType": "pie", 200 | "reduceOptions": { 201 | "calcs": [ 202 | "lastNotNull" 203 | ], 204 | "fields": "", 205 | "values": false 206 | }, 207 | "tooltip": { 208 | "mode": "single", 209 | "sort": "none" 210 | } 211 | }, 212 | "targets": [ 213 | { 214 | "alias": "True Positive", 215 | "datasource": { 216 | "type": "influxdb", 217 | "uid": "PB6B4F2F1C736D27A" 218 | }, 219 | "groupBy": [ 220 | { 221 | "params": [ 222 | "$__interval" 223 | ], 224 | "type": "time" 225 | }, 226 | { 227 | "params": [ 228 | "null" 229 | ], 230 | "type": "fill" 231 | } 232 | ], 233 | "orderByTime": "ASC", 234 | "policy": "autogen", 235 | "query": "SELECT count(\"y_pred\") FROM prediction where \"y_pred\"=1 and \"default\"=1 and $timeFilter", 236 | "rawQuery": true, 237 | "refId": "A", 238 | "resultFormat": "time_series", 239 | "select": [ 240 | [ 241 | { 242 | "params": [ 243 | "value" 244 | ], 245 | "type": "field" 246 | }, 247 | { 248 | "params": [], 249 | "type": "mean" 250 | } 251 | ] 252 | ], 253 | "tags": [] 254 | }, 255 | { 256 | "alias": "True Negative", 257 | "datasource": { 258 | "type": "influxdb", 259 | "uid": "PB6B4F2F1C736D27A" 260 | }, 261 | "groupBy": [ 262 | { 263 | "params": [ 264 | "$__interval" 265 | ], 266 | "type": "time" 267 | }, 268 | { 269 | "params": [ 270 | "null" 271 | ], 272 | "type": "fill" 273 | } 274 | ], 275 | "hide": false, 276 | "orderByTime": "ASC", 277 | "policy": "autogen", 278 | "query": "SELECT count(\"y_pred\") FROM prediction where \"y_pred\"=0 and \"default\"=0 and $timeFilter", 279 | "rawQuery": true, 280 | "refId": "B", 281 | "resultFormat": "time_series", 282 | "select": [ 283 | [ 284 | { 285 | "params": [ 286 | "value" 287 | ], 288 | "type": "field" 289 | }, 290 | { 291 | "params": [], 292 | "type": "mean" 293 | } 294 | ] 295 | ], 296 | "tags": [] 297 | }, 298 | { 299 | "alias": "False Positive", 300 | "datasource": { 301 | "type": "influxdb", 302 | "uid": "PB6B4F2F1C736D27A" 303 | }, 304 | "groupBy": [ 305 | { 306 | "params": [ 307 | "$__interval" 308 | ], 309 | "type": "time" 310 | }, 311 | { 312 | "params": [ 313 | "null" 314 | ], 315 | "type": "fill" 316 | } 317 | ], 318 | "hide": false, 319 | "orderByTime": "ASC", 320 | "policy": "autogen", 321 | "query": "SELECT count(\"y_pred\") FROM prediction where \"y_pred\"=1 and \"default\"=0 and $timeFilter", 322 | "rawQuery": true, 323 | "refId": "C", 324 | "resultFormat": "time_series", 325 | "select": [ 326 | [ 327 | { 328 | "params": [ 329 | "value" 330 | ], 331 | "type": "field" 332 | }, 333 | { 334 | "params": [], 335 | "type": "mean" 336 | } 337 | ] 338 | ], 339 | "tags": [] 340 | }, 341 | { 342 | "alias": "False Negative", 343 | "datasource": { 344 | "type": "influxdb", 345 | "uid": "PB6B4F2F1C736D27A" 346 | }, 347 | "groupBy": [ 348 | { 349 | "params": [ 350 | "$__interval" 351 | ], 352 | "type": "time" 353 | }, 354 | { 355 | "params": [ 356 | "null" 357 | ], 358 | "type": "fill" 359 | } 360 | ], 361 | "hide": false, 362 | "orderByTime": "ASC", 363 | "policy": "autogen", 364 | "query": "SELECT count(\"y_pred\") FROM prediction where \"y_pred\"=0 and \"default\"=1 and $timeFilter", 365 | "rawQuery": true, 366 | "refId": "D", 367 | "resultFormat": "time_series", 368 | "select": [ 369 | [ 370 | { 371 | "params": [ 372 | "value" 373 | ], 374 | "type": "field" 375 | }, 376 | { 377 | "params": [], 378 | "type": "mean" 379 | } 380 | ] 381 | ], 382 | "tags": [] 383 | } 384 | ], 385 | "title": "Confusion Matrix", 386 | "transparent": true, 387 | "type": "piechart" 388 | }, 389 | { 390 | "datasource": { 391 | "type": "influxdb", 392 | "uid": "PB6B4F2F1C736D27A" 393 | }, 394 | "description": "Performance matrices for getting a better understanding of how ROI is co-related with different parameters.", 395 | "fieldConfig": { 396 | "defaults": { 397 | "color": { 398 | "mode": "palette-classic" 399 | }, 400 | "custom": { 401 | "axisCenteredZero": false, 402 | "axisColorMode": "text", 403 | "axisLabel": "", 404 | "axisPlacement": "auto", 405 | "barAlignment": 0, 406 | "drawStyle": "line", 407 | "fillOpacity": 15, 408 | "gradientMode": "hue", 409 | "hideFrom": { 410 | "legend": false, 411 | "tooltip": false, 412 | "viz": false 413 | }, 414 | "insertNulls": false, 415 | "lineInterpolation": "smooth", 416 | "lineWidth": 2, 417 | "pointSize": 5, 418 | "scaleDistribution": { 419 | "type": "linear" 420 | }, 421 | "showPoints": "auto", 422 | "spanNulls": false, 423 | "stacking": { 424 | "group": "A", 425 | "mode": "none" 426 | }, 427 | "thresholdsStyle": { 428 | "mode": "off" 429 | } 430 | }, 431 | "mappings": [], 432 | "thresholds": { 433 | "mode": "absolute", 434 | "steps": [ 435 | { 436 | "color": "green", 437 | "value": null 438 | }, 439 | { 440 | "color": "red", 441 | "value": 80 442 | } 443 | ] 444 | }, 445 | "unit": "percentunit" 446 | }, 447 | "overrides": [] 448 | }, 449 | "gridPos": { 450 | "h": 10, 451 | "w": 12, 452 | "x": 12, 453 | "y": 12 454 | }, 455 | "id": 4, 456 | "options": { 457 | "legend": { 458 | "calcs": [], 459 | "displayMode": "list", 460 | "placement": "bottom", 461 | "showLegend": true 462 | }, 463 | "tooltip": { 464 | "mode": "single", 465 | "sort": "none" 466 | } 467 | }, 468 | "targets": [ 469 | { 470 | "alias": "Accuracy", 471 | "datasource": { 472 | "type": "influxdb", 473 | "uid": "PB6B4F2F1C736D27A" 474 | }, 475 | "groupBy": [ 476 | { 477 | "params": [ 478 | "$__interval" 479 | ], 480 | "type": "time" 481 | }, 482 | { 483 | "params": [ 484 | "null" 485 | ], 486 | "type": "fill" 487 | } 488 | ], 489 | "orderByTime": "ASC", 490 | "policy": "autogen", 491 | "query": "SELECT metric_value FROM metrics where metric_name = 'accuracy' and model_id = '$model_id' and model_version = '$model_version'", 492 | "rawQuery": true, 493 | "refId": "A", 494 | "resultFormat": "time_series", 495 | "select": [ 496 | [ 497 | { 498 | "params": [ 499 | "value" 500 | ], 501 | "type": "field" 502 | }, 503 | { 504 | "params": [], 505 | "type": "mean" 506 | } 507 | ] 508 | ], 509 | "tags": [] 510 | }, 511 | { 512 | "alias": "Precision", 513 | "datasource": { 514 | "type": "influxdb", 515 | "uid": "PB6B4F2F1C736D27A" 516 | }, 517 | "groupBy": [ 518 | { 519 | "params": [ 520 | "$__interval" 521 | ], 522 | "type": "time" 523 | }, 524 | { 525 | "params": [ 526 | "null" 527 | ], 528 | "type": "fill" 529 | } 530 | ], 531 | "hide": false, 532 | "orderByTime": "ASC", 533 | "policy": "autogen", 534 | "query": "SELECT metric_value FROM metrics where metric_name = 'precision' and model_id = '$model_id' and model_version = '$model_version'", 535 | "rawQuery": true, 536 | "refId": "B", 537 | "resultFormat": "time_series", 538 | "select": [ 539 | [ 540 | { 541 | "params": [ 542 | "value" 543 | ], 544 | "type": "field" 545 | }, 546 | { 547 | "params": [], 548 | "type": "mean" 549 | } 550 | ] 551 | ], 552 | "tags": [] 553 | }, 554 | { 555 | "alias": "Recall", 556 | "datasource": { 557 | "type": "influxdb", 558 | "uid": "PB6B4F2F1C736D27A" 559 | }, 560 | "groupBy": [ 561 | { 562 | "params": [ 563 | "$__interval" 564 | ], 565 | "type": "time" 566 | }, 567 | { 568 | "params": [ 569 | "null" 570 | ], 571 | "type": "fill" 572 | } 573 | ], 574 | "hide": false, 575 | "orderByTime": "ASC", 576 | "policy": "autogen", 577 | "query": "SELECT metric_value FROM metrics where metric_name = 'recall' and model_id = '$model_id' and model_version = '$model_version'", 578 | "rawQuery": true, 579 | "refId": "C", 580 | "resultFormat": "time_series", 581 | "select": [ 582 | [ 583 | { 584 | "params": [ 585 | "value" 586 | ], 587 | "type": "field" 588 | }, 589 | { 590 | "params": [], 591 | "type": "mean" 592 | } 593 | ] 594 | ], 595 | "tags": [] 596 | } 597 | ], 598 | "title": "Performance Metrics", 599 | "transparent": true, 600 | "type": "timeseries" 601 | } 602 | ], 603 | "refresh": false, 604 | "schemaVersion": 38, 605 | "style": "dark", 606 | "tags": [], 607 | "templating": { 608 | "list": [ 609 | { 610 | "current": { 611 | "selected": false, 612 | "text": "custom", 613 | "value": "custom" 614 | }, 615 | "hide": 0, 616 | "includeAll": false, 617 | "label": "metric type", 618 | "multi": false, 619 | "name": "metric_type", 620 | "options": [ 621 | { 622 | "selected": true, 623 | "text": "custom", 624 | "value": "custom" 625 | } 626 | ], 627 | "query": "custom", 628 | "skipUrlSync": false, 629 | "type": "custom" 630 | }, 631 | { 632 | "current": { 633 | "selected": false, 634 | "text": "1", 635 | "value": "1" 636 | }, 637 | "datasource": { 638 | "type": "influxdb", 639 | "uid": "PB6B4F2F1C736D27A" 640 | }, 641 | "definition": "SELECT distinct(\"model_id\") FROM metrics where metric_type = '$metric_type'", 642 | "hide": 0, 643 | "includeAll": false, 644 | "label": "model id", 645 | "multi": false, 646 | "name": "model_id", 647 | "options": [], 648 | "query": "SELECT distinct(\"model_id\") FROM metrics where metric_type = '$metric_type'", 649 | "refresh": 1, 650 | "regex": "", 651 | "skipUrlSync": false, 652 | "sort": 0, 653 | "type": "query" 654 | }, 655 | { 656 | "current": { 657 | "selected": false, 658 | "text": "2", 659 | "value": "2" 660 | }, 661 | "datasource": { 662 | "type": "influxdb", 663 | "uid": "PB6B4F2F1C736D27A" 664 | }, 665 | "definition": "SELECT distinct(\"model_version\") FROM metrics where metric_type = '$metric_type'", 666 | "hide": 0, 667 | "includeAll": false, 668 | "label": "model version", 669 | "multi": false, 670 | "name": "model_version", 671 | "options": [], 672 | "query": "SELECT distinct(\"model_version\") FROM metrics where metric_type = '$metric_type'", 673 | "refresh": 1, 674 | "regex": "", 675 | "skipUrlSync": false, 676 | "sort": 0, 677 | "type": "query" 678 | } 679 | ] 680 | }, 681 | "time": { 682 | "from": "2023-08-29T16:21:59.212Z", 683 | "to": "2023-08-30T16:21:59.212Z" 684 | }, 685 | "timepicker": {}, 686 | "timezone": "", 687 | "title": "ROI-dashboard", 688 | "uid": "b93ee885-457d-4298-b2d6-563bcb2efb42", 689 | "version": 6, 690 | "weekStart": "" 691 | } -------------------------------------------------------------------------------- /dashboards/metrics/drift_detail_metrics.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "grafana", 8 | "uid": "-- Grafana --" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "editable": true, 25 | "fiscalYearStartMonth": 0, 26 | "graphTooltip": 0, 27 | "id": 2, 28 | "links": [], 29 | "liveNow": false, 30 | "panels": [ 31 | { 32 | "datasource": { 33 | "type": "influxdb", 34 | "uid": "PB6B4F2F1C736D27A" 35 | }, 36 | "gridPos": { 37 | "h": 7, 38 | "w": 4, 39 | "x": 0, 40 | "y": 0 41 | }, 42 | "id": 4, 43 | "options": { 44 | "code": { 45 | "language": "plaintext", 46 | "showLineNumbers": false, 47 | "showMiniMap": false 48 | }, 49 | "content": "", 50 | "mode": "html" 51 | }, 52 | "pluginVersion": "9.5.3", 53 | "transparent": true, 54 | "type": "text" 55 | }, 56 | { 57 | "datasource": { 58 | "type": "influxdb", 59 | "uid": "PB6B4F2F1C736D27A" 60 | }, 61 | "description": "", 62 | "fieldConfig": { 63 | "defaults": { 64 | "color": { 65 | "fixedColor": "#9d0bde", 66 | "mode": "fixed" 67 | }, 68 | "decimals": 3, 69 | "displayName": "Max:", 70 | "mappings": [], 71 | "thresholds": { 72 | "mode": "absolute", 73 | "steps": [ 74 | { 75 | "color": "green", 76 | "value": null 77 | }, 78 | { 79 | "color": "red", 80 | "value": 80 81 | } 82 | ] 83 | }, 84 | "unit": "short" 85 | }, 86 | "overrides": [] 87 | }, 88 | "gridPos": { 89 | "h": 4, 90 | "w": 4, 91 | "x": 4, 92 | "y": 0 93 | }, 94 | "id": 6, 95 | "options": { 96 | "colorMode": "background", 97 | "graphMode": "none", 98 | "justifyMode": "center", 99 | "orientation": "auto", 100 | "reduceOptions": { 101 | "calcs": [ 102 | "max" 103 | ], 104 | "fields": "", 105 | "values": false 106 | }, 107 | "text": { 108 | "titleSize": 20, 109 | "valueSize": 45 110 | }, 111 | "textMode": "auto" 112 | }, 113 | "pluginVersion": "9.5.3", 114 | "targets": [ 115 | { 116 | "alias": "", 117 | "datasource": { 118 | "type": "influxdb", 119 | "uid": "PB6B4F2F1C736D27A" 120 | }, 121 | "query": "SELECT metric_value FROM metrics where metric_type = '$type' and feature_name = '$feature_name' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version' ", 122 | "rawQuery": true, 123 | "refId": "A", 124 | "resultFormat": "time_series" 125 | } 126 | ], 127 | "transparent": true, 128 | "type": "stat" 129 | }, 130 | { 131 | "datasource": { 132 | "type": "influxdb", 133 | "uid": "PB6B4F2F1C736D27A" 134 | }, 135 | "description": "", 136 | "fieldConfig": { 137 | "defaults": { 138 | "color": { 139 | "fixedColor": "purple", 140 | "mode": "fixed" 141 | }, 142 | "decimals": 3, 143 | "displayName": "Min:", 144 | "mappings": [], 145 | "thresholds": { 146 | "mode": "absolute", 147 | "steps": [ 148 | { 149 | "color": "green", 150 | "value": null 151 | }, 152 | { 153 | "color": "red", 154 | "value": 80 155 | } 156 | ] 157 | }, 158 | "unit": "short" 159 | }, 160 | "overrides": [] 161 | }, 162 | "gridPos": { 163 | "h": 4, 164 | "w": 4, 165 | "x": 8, 166 | "y": 0 167 | }, 168 | "id": 7, 169 | "options": { 170 | "colorMode": "background", 171 | "graphMode": "none", 172 | "justifyMode": "center", 173 | "orientation": "auto", 174 | "reduceOptions": { 175 | "calcs": [ 176 | "min" 177 | ], 178 | "fields": "", 179 | "values": false 180 | }, 181 | "text": { 182 | "titleSize": 20, 183 | "valueSize": 45 184 | }, 185 | "textMode": "auto" 186 | }, 187 | "pluginVersion": "9.5.3", 188 | "targets": [ 189 | { 190 | "alias": "", 191 | "datasource": { 192 | "type": "influxdb", 193 | "uid": "PB6B4F2F1C736D27A" 194 | }, 195 | "query": "SELECT metric_value FROM metrics where metric_type = '$type' and feature_name = '$feature_name' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version' ", 196 | "rawQuery": true, 197 | "refId": "A", 198 | "resultFormat": "time_series" 199 | } 200 | ], 201 | "transparent": true, 202 | "type": "stat" 203 | }, 204 | { 205 | "datasource": { 206 | "type": "influxdb", 207 | "uid": "PB6B4F2F1C736D27A" 208 | }, 209 | "description": "", 210 | "fieldConfig": { 211 | "defaults": { 212 | "color": { 213 | "fixedColor": "super-light-purple", 214 | "mode": "fixed" 215 | }, 216 | "displayName": "#Count:", 217 | "mappings": [], 218 | "thresholds": { 219 | "mode": "absolute", 220 | "steps": [ 221 | { 222 | "color": "green", 223 | "value": null 224 | }, 225 | { 226 | "color": "red", 227 | "value": 80 228 | } 229 | ] 230 | }, 231 | "unit": "short" 232 | }, 233 | "overrides": [] 234 | }, 235 | "gridPos": { 236 | "h": 4, 237 | "w": 4, 238 | "x": 12, 239 | "y": 0 240 | }, 241 | "id": 10, 242 | "options": { 243 | "colorMode": "background", 244 | "graphMode": "none", 245 | "justifyMode": "center", 246 | "orientation": "auto", 247 | "reduceOptions": { 248 | "calcs": [ 249 | "min" 250 | ], 251 | "fields": "", 252 | "values": true 253 | }, 254 | "text": { 255 | "titleSize": 20, 256 | "valueSize": 45 257 | }, 258 | "textMode": "auto" 259 | }, 260 | "pluginVersion": "9.5.3", 261 | "targets": [ 262 | { 263 | "alias": "", 264 | "datasource": { 265 | "type": "influxdb", 266 | "uid": "PB6B4F2F1C736D27A" 267 | }, 268 | "query": "SELECT COUNT(metric_value) FROM metrics where metric_type = '$type' and feature_name = '$feature_name' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version' and $timeFilter", 269 | "rawQuery": true, 270 | "refId": "A", 271 | "resultFormat": "time_series" 272 | } 273 | ], 274 | "transparent": true, 275 | "type": "stat" 276 | }, 277 | { 278 | "datasource": { 279 | "type": "influxdb", 280 | "uid": "PB6B4F2F1C736D27A" 281 | }, 282 | "description": "", 283 | "fieldConfig": { 284 | "defaults": { 285 | "color": { 286 | "mode": "thresholds" 287 | }, 288 | "mappings": [], 289 | "thresholds": { 290 | "mode": "absolute", 291 | "steps": [ 292 | { 293 | "color": "green", 294 | "value": null 295 | }, 296 | { 297 | "color": "light-purple", 298 | "value": 80 299 | } 300 | ] 301 | } 302 | }, 303 | "overrides": [] 304 | }, 305 | "gridPos": { 306 | "h": 3, 307 | "w": 4, 308 | "x": 8, 309 | "y": 4 310 | }, 311 | "id": 5, 312 | "options": { 313 | "colorMode": "value", 314 | "graphMode": "area", 315 | "justifyMode": "auto", 316 | "orientation": "auto", 317 | "reduceOptions": { 318 | "calcs": [ 319 | "lastNotNull" 320 | ], 321 | "fields": "/^Time$/", 322 | "values": false 323 | }, 324 | "text": { 325 | "titleSize": 20, 326 | "valueSize": 20 327 | }, 328 | "textMode": "auto" 329 | }, 330 | "pluginVersion": "9.5.3", 331 | "targets": [ 332 | { 333 | "alias": "", 334 | "datasource": { 335 | "type": "influxdb", 336 | "uid": "PB6B4F2F1C736D27A" 337 | }, 338 | "query": "SELECT time, metric_value FROM metrics where metric_type = '$type' and feature_name = '$feature_name' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version' ORDER BY time DESC LIMIT 1", 339 | "rawQuery": true, 340 | "refId": "A", 341 | "resultFormat": "time_series" 342 | } 343 | ], 344 | "title": "Last Value Inserted at", 345 | "transparent": true, 346 | "type": "stat" 347 | }, 348 | { 349 | "datasource": { 350 | "type": "influxdb", 351 | "uid": "PB6B4F2F1C736D27A" 352 | }, 353 | "description": "", 354 | "fieldConfig": { 355 | "defaults": { 356 | "color": { 357 | "fixedColor": "#b0aaf6", 358 | "mode": "fixed" 359 | }, 360 | "displayName": "Change", 361 | "mappings": [], 362 | "thresholds": { 363 | "mode": "percentage", 364 | "steps": [ 365 | { 366 | "color": "green", 367 | "value": null 368 | }, 369 | { 370 | "color": "orange", 371 | "value": 70 372 | }, 373 | { 374 | "color": "red", 375 | "value": 85 376 | } 377 | ] 378 | }, 379 | "unit": "percent" 380 | }, 381 | "overrides": [] 382 | }, 383 | "gridPos": { 384 | "h": 4, 385 | "w": 4, 386 | "x": 12, 387 | "y": 4 388 | }, 389 | "id": 8, 390 | "options": { 391 | "orientation": "auto", 392 | "reduceOptions": { 393 | "calcs": [ 394 | "lastNotNull" 395 | ], 396 | "fields": "", 397 | "values": false 398 | }, 399 | "showThresholdLabels": false, 400 | "showThresholdMarkers": true, 401 | "text": { 402 | "titleSize": 17, 403 | "valueSize": 40 404 | } 405 | }, 406 | "pluginVersion": "9.5.3", 407 | "targets": [ 408 | { 409 | "alias": "", 410 | "datasource": { 411 | "type": "influxdb", 412 | "uid": "PB6B4F2F1C736D27A" 413 | }, 414 | "query": "SELECT metric_value FROM metrics where metric_type = '$type' and feature_name = '$feature_name' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version' ", 415 | "rawQuery": true, 416 | "refId": "A", 417 | "resultFormat": "time_series" 418 | } 419 | ], 420 | "transformations": [ 421 | { 422 | "id": "reduce", 423 | "options": { 424 | "includeTimeField": false, 425 | "labelsToFields": false, 426 | "mode": "seriesToRows", 427 | "reducers": [ 428 | "max", 429 | "range" 430 | ] 431 | } 432 | }, 433 | { 434 | "id": "calculateField", 435 | "options": { 436 | "alias": "RangeDiv", 437 | "binary": { 438 | "left": "Range", 439 | "operator": "/", 440 | "reducer": "sum", 441 | "right": "Max" 442 | }, 443 | "mode": "binary", 444 | "reduce": { 445 | "reducer": "sum" 446 | } 447 | } 448 | }, 449 | { 450 | "id": "calculateField", 451 | "options": { 452 | "alias": "Change%", 453 | "binary": { 454 | "left": "RangeDiv", 455 | "operator": "*", 456 | "reducer": "sum", 457 | "right": "100" 458 | }, 459 | "mode": "binary", 460 | "reduce": { 461 | "reducer": "sum" 462 | }, 463 | "replaceFields": true 464 | } 465 | } 466 | ], 467 | "transparent": true, 468 | "type": "gauge" 469 | }, 470 | { 471 | "datasource": { 472 | "type": "influxdb", 473 | "uid": "PB6B4F2F1C736D27A" 474 | }, 475 | "description": "", 476 | "fieldConfig": { 477 | "defaults": { 478 | "custom": { 479 | "hideFrom": { 480 | "legend": false, 481 | "tooltip": false, 482 | "viz": false 483 | }, 484 | "scaleDistribution": { 485 | "type": "linear" 486 | } 487 | } 488 | }, 489 | "overrides": [] 490 | }, 491 | "gridPos": { 492 | "h": 16, 493 | "w": 9, 494 | "x": 0, 495 | "y": 7 496 | }, 497 | "id": 9, 498 | "options": { 499 | "calculate": true, 500 | "calculation": { 501 | "xBuckets": { 502 | "mode": "size" 503 | }, 504 | "yBuckets": { 505 | "mode": "size" 506 | } 507 | }, 508 | "cellGap": 1, 509 | "color": { 510 | "exponent": 0.5, 511 | "fill": "dark-orange", 512 | "mode": "scheme", 513 | "reverse": false, 514 | "scale": "exponential", 515 | "scheme": "Greens", 516 | "steps": 64 517 | }, 518 | "exemplars": { 519 | "color": "rgba(255,0,255,0.7)" 520 | }, 521 | "filterValues": { 522 | "le": 1e-9 523 | }, 524 | "legend": { 525 | "show": true 526 | }, 527 | "rowsFrame": { 528 | "layout": "auto" 529 | }, 530 | "tooltip": { 531 | "show": true, 532 | "yHistogram": false 533 | }, 534 | "yAxis": { 535 | "axisPlacement": "left", 536 | "reverse": false 537 | } 538 | }, 539 | "pluginVersion": "9.5.3", 540 | "targets": [ 541 | { 542 | "datasource": { 543 | "type": "influxdb", 544 | "uid": "PB6B4F2F1C736D27A" 545 | }, 546 | "query": "SELECT time, metric_value FROM metrics where metric_type = '$type' and feature_name = '$feature_name' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version' and $timeFilter", 547 | "rawQuery": true, 548 | "refId": "A", 549 | "resultFormat": "time_series" 550 | } 551 | ], 552 | "title": "Heatmap for Metrics vs Time", 553 | "transparent": true, 554 | "type": "heatmap" 555 | }, 556 | { 557 | "datasource": { 558 | "type": "influxdb", 559 | "uid": "PB6B4F2F1C736D27A" 560 | }, 561 | "description": "", 562 | "fieldConfig": { 563 | "defaults": { 564 | "color": { 565 | "fixedColor": "#8d82ff", 566 | "mode": "fixed" 567 | }, 568 | "custom": { 569 | "axisCenteredZero": false, 570 | "axisColorMode": "text", 571 | "axisLabel": "", 572 | "axisPlacement": "right", 573 | "barAlignment": 0, 574 | "drawStyle": "line", 575 | "fillOpacity": 25, 576 | "gradientMode": "none", 577 | "hideFrom": { 578 | "legend": false, 579 | "tooltip": false, 580 | "viz": false 581 | }, 582 | "lineInterpolation": "smooth", 583 | "lineStyle": { 584 | "dash": [ 585 | 10, 586 | 10 587 | ], 588 | "fill": "dash" 589 | }, 590 | "lineWidth": 4, 591 | "pointSize": 6, 592 | "scaleDistribution": { 593 | "type": "linear" 594 | }, 595 | "showPoints": "auto", 596 | "spanNulls": true, 597 | "stacking": { 598 | "group": "A", 599 | "mode": "none" 600 | }, 601 | "thresholdsStyle": { 602 | "mode": "off" 603 | } 604 | }, 605 | "decimals": 2, 606 | "mappings": [], 607 | "thresholds": { 608 | "mode": "absolute", 609 | "steps": [ 610 | { 611 | "color": "green", 612 | "value": null 613 | }, 614 | { 615 | "color": "red", 616 | "value": 80 617 | } 618 | ] 619 | } 620 | }, 621 | "overrides": [ 622 | { 623 | "matcher": { 624 | "id": "byName", 625 | "options": "threshold" 626 | }, 627 | "properties": [ 628 | { 629 | "id": "color", 630 | "value": { 631 | "fixedColor": "blue", 632 | "mode": "fixed" 633 | } 634 | }, 635 | { 636 | "id": "custom.axisPlacement", 637 | "value": "left" 638 | }, 639 | { 640 | "id": "unit", 641 | "value": "percentunit" 642 | } 643 | ] 644 | } 645 | ] 646 | }, 647 | "gridPos": { 648 | "h": 15, 649 | "w": 15, 650 | "x": 9, 651 | "y": 8 652 | }, 653 | "id": 2, 654 | "options": { 655 | "legend": { 656 | "calcs": [], 657 | "displayMode": "list", 658 | "placement": "bottom", 659 | "showLegend": true 660 | }, 661 | "tooltip": { 662 | "mode": "single", 663 | "sort": "none" 664 | } 665 | }, 666 | "targets": [ 667 | { 668 | "alias": "$metric_name value", 669 | "datasource": { 670 | "type": "influxdb", 671 | "uid": "PB6B4F2F1C736D27A" 672 | }, 673 | "query": "SELECT metric_value FROM metrics where metric_type = '$type' and feature_name = '$feature_name' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version'", 674 | "rawQuery": true, 675 | "refId": "A", 676 | "resultFormat": "time_series" 677 | }, 678 | { 679 | "alias": "threshold", 680 | "datasource": { 681 | "type": "influxdb", 682 | "uid": "PB6B4F2F1C736D27A" 683 | }, 684 | "hide": false, 685 | "query": "SELECT threshold FROM metrics where metric_type = '$type' and feature_name = '$feature_name' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version'", 686 | "rawQuery": true, 687 | "refId": "B", 688 | "resultFormat": "time_series" 689 | } 690 | ], 691 | "title": "Drift Detail for Feature - $feature_name and Metric - $metric_name", 692 | "transparent": true, 693 | "type": "timeseries" 694 | } 695 | ], 696 | "refresh": "", 697 | "revision": 1, 698 | "schemaVersion": 38, 699 | "style": "dark", 700 | "tags": [], 701 | "templating": { 702 | "list": [ 703 | { 704 | "current": { 705 | "selected": false, 706 | "text": "drift", 707 | "value": "drift" 708 | }, 709 | "hide": 0, 710 | "includeAll": false, 711 | "multi": false, 712 | "name": "type", 713 | "options": [ 714 | { 715 | "selected": true, 716 | "text": "drift", 717 | "value": "drift" 718 | } 719 | ], 720 | "query": "drift", 721 | "queryValue": "", 722 | "skipUrlSync": false, 723 | "type": "custom" 724 | }, 725 | { 726 | "current": { 727 | "selected": false, 728 | "text": "1", 729 | "value": "1" 730 | }, 731 | "datasource": { 732 | "type": "influxdb", 733 | "uid": "PB6B4F2F1C736D27A" 734 | }, 735 | "definition": "SELECT distinct(\"model_id\") FROM metrics where metric_type = '$type'", 736 | "hide": 0, 737 | "includeAll": false, 738 | "multi": false, 739 | "name": "model_id", 740 | "options": [], 741 | "query": "SELECT distinct(\"model_id\") FROM metrics where metric_type = '$type'", 742 | "refresh": 1, 743 | "regex": "", 744 | "skipUrlSync": false, 745 | "sort": 0, 746 | "type": "query" 747 | }, 748 | { 749 | "current": { 750 | "selected": false, 751 | "text": "2", 752 | "value": "2" 753 | }, 754 | "datasource": { 755 | "type": "influxdb", 756 | "uid": "PB6B4F2F1C736D27A" 757 | }, 758 | "definition": "SELECT distinct(\"model_version\") FROM metrics where metric_type = '$type'", 759 | "hide": 0, 760 | "includeAll": false, 761 | "multi": false, 762 | "name": "model_version", 763 | "options": [], 764 | "query": "SELECT distinct(\"model_version\") FROM metrics where metric_type = '$type'", 765 | "refresh": 1, 766 | "regex": "", 767 | "skipUrlSync": false, 768 | "sort": 0, 769 | "type": "query" 770 | }, 771 | { 772 | "current": { 773 | "selected": true, 774 | "text": "HP", 775 | "value": "HP" 776 | }, 777 | "datasource": { 778 | "type": "influxdb", 779 | "uid": "PB6B4F2F1C736D27A" 780 | }, 781 | "definition": "SELECT distinct(\"feature_name\") FROM metrics where metric_type = '$type'", 782 | "hide": 0, 783 | "includeAll": false, 784 | "multi": false, 785 | "name": "feature_name", 786 | "options": [], 787 | "query": "SELECT distinct(\"feature_name\") FROM metrics where metric_type = '$type'", 788 | "refresh": 1, 789 | "regex": "", 790 | "skipUrlSync": false, 791 | "sort": 0, 792 | "type": "query" 793 | }, 794 | { 795 | "current": { 796 | "selected": false, 797 | "text": "manwu", 798 | "value": "manwu" 799 | }, 800 | "datasource": { 801 | "type": "influxdb", 802 | "uid": "PB6B4F2F1C736D27A" 803 | }, 804 | "definition": "SELECT distinct(\"metric_name\") FROM metrics where metric_type = '$type'", 805 | "hide": 0, 806 | "includeAll": false, 807 | "multi": false, 808 | "name": "metric_name", 809 | "options": [], 810 | "query": "SELECT distinct(\"metric_name\") FROM metrics where metric_type = '$type'", 811 | "refresh": 1, 812 | "regex": "", 813 | "skipUrlSync": false, 814 | "sort": 0, 815 | "type": "query" 816 | } 817 | ] 818 | }, 819 | "time": { 820 | "from": "now-7d", 821 | "to": "now" 822 | }, 823 | "timepicker": {}, 824 | "timezone": "", 825 | "title": "DriftDetailDashboard", 826 | "uid": "Zntdh-tVz", 827 | "version": 8, 828 | "weekStart": "" 829 | } -------------------------------------------------------------------------------- /grafana-provisioning/dashboards/drift_detail_dashboard.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": { 7 | "type": "grafana", 8 | "uid": "-- Grafana --" 9 | }, 10 | "enable": true, 11 | "hide": true, 12 | "iconColor": "rgba(0, 211, 255, 1)", 13 | "name": "Annotations & Alerts", 14 | "target": { 15 | "limit": 100, 16 | "matchAny": false, 17 | "tags": [], 18 | "type": "dashboard" 19 | }, 20 | "type": "dashboard" 21 | } 22 | ] 23 | }, 24 | "editable": true, 25 | "fiscalYearStartMonth": 0, 26 | "graphTooltip": 0, 27 | "id": 2, 28 | "links": [], 29 | "liveNow": false, 30 | "panels": [ 31 | { 32 | "datasource": { 33 | "type": "influxdb", 34 | "uid": "PB6B4F2F1C736D27A" 35 | }, 36 | "gridPos": { 37 | "h": 7, 38 | "w": 4, 39 | "x": 0, 40 | "y": 0 41 | }, 42 | "id": 4, 43 | "options": { 44 | "code": { 45 | "language": "plaintext", 46 | "showLineNumbers": false, 47 | "showMiniMap": false 48 | }, 49 | "content": "", 50 | "mode": "html" 51 | }, 52 | "pluginVersion": "9.3.6", 53 | "transparent": true, 54 | "type": "text" 55 | }, 56 | { 57 | "datasource": { 58 | "type": "influxdb", 59 | "uid": "PB6B4F2F1C736D27A" 60 | }, 61 | "description": "", 62 | "fieldConfig": { 63 | "defaults": { 64 | "color": { 65 | "fixedColor": "#9d0bde", 66 | "mode": "fixed" 67 | }, 68 | "decimals": 3, 69 | "displayName": "MAX:", 70 | "mappings": [], 71 | "thresholds": { 72 | "mode": "absolute", 73 | "steps": [ 74 | { 75 | "color": "green", 76 | "value": null 77 | }, 78 | { 79 | "color": "red", 80 | "value": 80 81 | } 82 | ] 83 | }, 84 | "unit": "short" 85 | }, 86 | "overrides": [] 87 | }, 88 | "gridPos": { 89 | "h": 3, 90 | "w": 4, 91 | "x": 4, 92 | "y": 0 93 | }, 94 | "id": 6, 95 | "options": { 96 | "colorMode": "background", 97 | "graphMode": "none", 98 | "justifyMode": "center", 99 | "orientation": "auto", 100 | "reduceOptions": { 101 | "calcs": [ 102 | "max" 103 | ], 104 | "fields": "", 105 | "values": false 106 | }, 107 | "text": { 108 | "titleSize": 20, 109 | "valueSize": 45 110 | }, 111 | "textMode": "auto" 112 | }, 113 | "pluginVersion": "9.3.6", 114 | "targets": [ 115 | { 116 | "alias": "", 117 | "datasource": { 118 | "type": "influxdb", 119 | "uid": "PB6B4F2F1C736D27A" 120 | }, 121 | "query": "SELECT value FROM metrics where type = '$type' and feature = '$feature_name' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version' ", 122 | "rawQuery": true, 123 | "refId": "A", 124 | "resultFormat": "time_series" 125 | } 126 | ], 127 | "transparent": true, 128 | "type": "stat" 129 | }, 130 | { 131 | "datasource": { 132 | "type": "influxdb", 133 | "uid": "PB6B4F2F1C736D27A" 134 | }, 135 | "description": "", 136 | "fieldConfig": { 137 | "defaults": { 138 | "color": { 139 | "fixedColor": "#ffffff", 140 | "mode": "fixed" 141 | }, 142 | "displayName": "Change%", 143 | "mappings": [], 144 | "thresholds": { 145 | "mode": "percentage", 146 | "steps": [ 147 | { 148 | "color": "green", 149 | "value": null 150 | }, 151 | { 152 | "color": "orange", 153 | "value": 70 154 | }, 155 | { 156 | "color": "red", 157 | "value": 85 158 | } 159 | ] 160 | }, 161 | "unit": "percent" 162 | }, 163 | "overrides": [] 164 | }, 165 | "gridPos": { 166 | "h": 5, 167 | "w": 4, 168 | "x": 10, 169 | "y": 0 170 | }, 171 | "id": 8, 172 | "options": { 173 | "orientation": "auto", 174 | "reduceOptions": { 175 | "calcs": [ 176 | "lastNotNull" 177 | ], 178 | "fields": "", 179 | "values": false 180 | }, 181 | "showThresholdLabels": false, 182 | "showThresholdMarkers": true 183 | }, 184 | "pluginVersion": "9.3.6", 185 | "targets": [ 186 | { 187 | "alias": "", 188 | "datasource": { 189 | "type": "influxdb", 190 | "uid": "PB6B4F2F1C736D27A" 191 | }, 192 | "query": "SELECT value FROM metrics where type = '$type' and feature = '$feature_name' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version' ", 193 | "rawQuery": true, 194 | "refId": "A", 195 | "resultFormat": "time_series" 196 | } 197 | ], 198 | "transformations": [ 199 | { 200 | "id": "reduce", 201 | "options": { 202 | "includeTimeField": false, 203 | "labelsToFields": false, 204 | "mode": "seriesToRows", 205 | "reducers": [ 206 | "max", 207 | "range" 208 | ] 209 | } 210 | }, 211 | { 212 | "id": "calculateField", 213 | "options": { 214 | "alias": "RangeDiv", 215 | "binary": { 216 | "left": "Range", 217 | "operator": "/", 218 | "reducer": "sum", 219 | "right": "Max" 220 | }, 221 | "mode": "binary", 222 | "reduce": { 223 | "reducer": "sum" 224 | } 225 | } 226 | }, 227 | { 228 | "id": "calculateField", 229 | "options": { 230 | "alias": "Change%", 231 | "binary": { 232 | "left": "RangeDiv", 233 | "operator": "*", 234 | "reducer": "sum", 235 | "right": "100" 236 | }, 237 | "mode": "binary", 238 | "reduce": { 239 | "reducer": "sum" 240 | }, 241 | "replaceFields": true 242 | } 243 | } 244 | ], 245 | "transparent": true, 246 | "type": "gauge" 247 | }, 248 | { 249 | "datasource": { 250 | "type": "influxdb", 251 | "uid": "PB6B4F2F1C736D27A" 252 | }, 253 | "description": "", 254 | "fieldConfig": { 255 | "defaults": { 256 | "color": { 257 | "mode": "thresholds" 258 | }, 259 | "mappings": [], 260 | "thresholds": { 261 | "mode": "absolute", 262 | "steps": [ 263 | { 264 | "color": "green", 265 | "value": null 266 | }, 267 | { 268 | "color": "red", 269 | "value": 80 270 | } 271 | ] 272 | } 273 | }, 274 | "overrides": [] 275 | }, 276 | "gridPos": { 277 | "h": 2, 278 | "w": 5, 279 | "x": 16, 280 | "y": 0 281 | }, 282 | "id": 5, 283 | "options": { 284 | "colorMode": "value", 285 | "graphMode": "area", 286 | "justifyMode": "auto", 287 | "orientation": "auto", 288 | "reduceOptions": { 289 | "calcs": [ 290 | "lastNotNull" 291 | ], 292 | "fields": "/^Time$/", 293 | "values": false 294 | }, 295 | "textMode": "auto" 296 | }, 297 | "pluginVersion": "9.3.6", 298 | "targets": [ 299 | { 300 | "alias": "", 301 | "datasource": { 302 | "type": "influxdb", 303 | "uid": "PB6B4F2F1C736D27A" 304 | }, 305 | "query": "SELECT time, value FROM metrics where type = '$type' and feature = '$feature_name' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version' ORDER BY time DESC LIMIT 1", 306 | "rawQuery": true, 307 | "refId": "A", 308 | "resultFormat": "time_series" 309 | } 310 | ], 311 | "title": "Last Value Inserted at", 312 | "transparent": true, 313 | "type": "stat" 314 | }, 315 | { 316 | "datasource": { 317 | "type": "influxdb", 318 | "uid": "PB6B4F2F1C736D27A" 319 | }, 320 | "description": "", 321 | "fieldConfig": { 322 | "defaults": { 323 | "color": { 324 | "fixedColor": "purple", 325 | "mode": "fixed" 326 | }, 327 | "decimals": 3, 328 | "displayName": "MIN:", 329 | "mappings": [], 330 | "thresholds": { 331 | "mode": "absolute", 332 | "steps": [ 333 | { 334 | "color": "green", 335 | "value": null 336 | }, 337 | { 338 | "color": "red", 339 | "value": 80 340 | } 341 | ] 342 | }, 343 | "unit": "short" 344 | }, 345 | "overrides": [] 346 | }, 347 | "gridPos": { 348 | "h": 3, 349 | "w": 4, 350 | "x": 4, 351 | "y": 3 352 | }, 353 | "id": 7, 354 | "options": { 355 | "colorMode": "background", 356 | "graphMode": "none", 357 | "justifyMode": "center", 358 | "orientation": "auto", 359 | "reduceOptions": { 360 | "calcs": [ 361 | "min" 362 | ], 363 | "fields": "", 364 | "values": false 365 | }, 366 | "text": { 367 | "titleSize": 20, 368 | "valueSize": 45 369 | }, 370 | "textMode": "auto" 371 | }, 372 | "pluginVersion": "9.3.6", 373 | "targets": [ 374 | { 375 | "alias": "", 376 | "datasource": { 377 | "type": "influxdb", 378 | "uid": "PB6B4F2F1C736D27A" 379 | }, 380 | "query": "SELECT value FROM metrics where type = '$type' and feature = '$feature_name' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version' ", 381 | "rawQuery": true, 382 | "refId": "A", 383 | "resultFormat": "time_series" 384 | } 385 | ], 386 | "transparent": true, 387 | "type": "stat" 388 | }, 389 | { 390 | "datasource": { 391 | "type": "influxdb", 392 | "uid": "PB6B4F2F1C736D27A" 393 | }, 394 | "description": "", 395 | "fieldConfig": { 396 | "defaults": { 397 | "color": { 398 | "fixedColor": "semi-dark-purple", 399 | "mode": "fixed" 400 | }, 401 | "custom": { 402 | "axisCenteredZero": false, 403 | "axisColorMode": "text", 404 | "axisLabel": "", 405 | "axisPlacement": "right", 406 | "barAlignment": 0, 407 | "drawStyle": "line", 408 | "fillOpacity": 25, 409 | "gradientMode": "none", 410 | "hideFrom": { 411 | "legend": false, 412 | "tooltip": false, 413 | "viz": false 414 | }, 415 | "lineInterpolation": "smooth", 416 | "lineStyle": { 417 | "dash": [ 418 | 10, 419 | 10 420 | ], 421 | "fill": "dash" 422 | }, 423 | "lineWidth": 4, 424 | "pointSize": 6, 425 | "scaleDistribution": { 426 | "type": "linear" 427 | }, 428 | "showPoints": "auto", 429 | "spanNulls": true, 430 | "stacking": { 431 | "group": "A", 432 | "mode": "none" 433 | }, 434 | "thresholdsStyle": { 435 | "mode": "off" 436 | } 437 | }, 438 | "decimals": 2, 439 | "mappings": [], 440 | "thresholds": { 441 | "mode": "absolute", 442 | "steps": [ 443 | { 444 | "color": "green", 445 | "value": null 446 | }, 447 | { 448 | "color": "red", 449 | "value": 80 450 | } 451 | ] 452 | } 453 | }, 454 | "overrides": [ 455 | { 456 | "matcher": { 457 | "id": "byName", 458 | "options": "threshold" 459 | }, 460 | "properties": [ 461 | { 462 | "id": "color", 463 | "value": { 464 | "mode": "palette-classic" 465 | } 466 | } 467 | ] 468 | } 469 | ] 470 | }, 471 | "gridPos": { 472 | "h": 17, 473 | "w": 15, 474 | "x": 9, 475 | "y": 5 476 | }, 477 | "id": 2, 478 | "options": { 479 | "legend": { 480 | "calcs": [], 481 | "displayMode": "list", 482 | "placement": "bottom", 483 | "showLegend": true 484 | }, 485 | "tooltip": { 486 | "mode": "single", 487 | "sort": "none" 488 | } 489 | }, 490 | "targets": [ 491 | { 492 | "alias": "$metric_name value", 493 | "datasource": { 494 | "type": "influxdb", 495 | "uid": "PB6B4F2F1C736D27A" 496 | }, 497 | "query": "SELECT value FROM metrics where type = '$type' and feature = '$feature_name' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version'", 498 | "rawQuery": true, 499 | "refId": "A", 500 | "resultFormat": "time_series" 501 | }, 502 | { 503 | "alias": "threshold", 504 | "datasource": { 505 | "type": "influxdb", 506 | "uid": "PB6B4F2F1C736D27A" 507 | }, 508 | "hide": false, 509 | "query": "SELECT threshold FROM metrics where type = '$type' and feature = '$feature_name' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version'", 510 | "rawQuery": true, 511 | "refId": "B", 512 | "resultFormat": "time_series" 513 | } 514 | ], 515 | "title": "Drift Detail for Feature - $feature_name and Metric - $metric_name", 516 | "transparent": true, 517 | "type": "timeseries" 518 | }, 519 | { 520 | "datasource": { 521 | "type": "influxdb", 522 | "uid": "PB6B4F2F1C736D27A" 523 | }, 524 | "description": "", 525 | "fieldConfig": { 526 | "defaults": { 527 | "color": { 528 | "fixedColor": "super-light-purple", 529 | "mode": "fixed" 530 | }, 531 | "displayName": "#Count:", 532 | "mappings": [], 533 | "thresholds": { 534 | "mode": "absolute", 535 | "steps": [ 536 | { 537 | "color": "green", 538 | "value": null 539 | }, 540 | { 541 | "color": "red", 542 | "value": 80 543 | } 544 | ] 545 | }, 546 | "unit": "short" 547 | }, 548 | "overrides": [] 549 | }, 550 | "gridPos": { 551 | "h": 3, 552 | "w": 4, 553 | "x": 4, 554 | "y": 6 555 | }, 556 | "id": 10, 557 | "options": { 558 | "colorMode": "background", 559 | "graphMode": "none", 560 | "justifyMode": "center", 561 | "orientation": "auto", 562 | "reduceOptions": { 563 | "calcs": [ 564 | "min" 565 | ], 566 | "fields": "", 567 | "values": true 568 | }, 569 | "text": { 570 | "titleSize": 20, 571 | "valueSize": 45 572 | }, 573 | "textMode": "auto" 574 | }, 575 | "pluginVersion": "9.3.6", 576 | "targets": [ 577 | { 578 | "alias": "", 579 | "datasource": { 580 | "type": "influxdb", 581 | "uid": "PB6B4F2F1C736D27A" 582 | }, 583 | "query": "SELECT COUNT(value) FROM metrics where type = '$type' and feature = '$feature_name' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version' and $timeFilter", 584 | "rawQuery": true, 585 | "refId": "A", 586 | "resultFormat": "time_series" 587 | } 588 | ], 589 | "transparent": true, 590 | "type": "stat" 591 | }, 592 | { 593 | "datasource": { 594 | "type": "influxdb", 595 | "uid": "PB6B4F2F1C736D27A" 596 | }, 597 | "description": "", 598 | "fieldConfig": { 599 | "defaults": { 600 | "custom": { 601 | "hideFrom": { 602 | "legend": false, 603 | "tooltip": false, 604 | "viz": false 605 | }, 606 | "scaleDistribution": { 607 | "type": "linear" 608 | } 609 | } 610 | }, 611 | "overrides": [] 612 | }, 613 | "gridPos": { 614 | "h": 13, 615 | "w": 9, 616 | "x": 0, 617 | "y": 9 618 | }, 619 | "id": 9, 620 | "options": { 621 | "calculate": true, 622 | "calculation": { 623 | "xBuckets": { 624 | "mode": "size" 625 | }, 626 | "yBuckets": { 627 | "mode": "size" 628 | } 629 | }, 630 | "cellGap": 1, 631 | "color": { 632 | "exponent": 0.5, 633 | "fill": "dark-orange", 634 | "mode": "scheme", 635 | "reverse": false, 636 | "scale": "exponential", 637 | "scheme": "Oranges", 638 | "steps": 64 639 | }, 640 | "exemplars": { 641 | "color": "rgba(255,0,255,0.7)" 642 | }, 643 | "filterValues": { 644 | "le": 1e-9 645 | }, 646 | "legend": { 647 | "show": true 648 | }, 649 | "rowsFrame": { 650 | "layout": "auto" 651 | }, 652 | "tooltip": { 653 | "show": true, 654 | "yHistogram": false 655 | }, 656 | "yAxis": { 657 | "axisPlacement": "left", 658 | "reverse": false 659 | } 660 | }, 661 | "pluginVersion": "9.3.6", 662 | "targets": [ 663 | { 664 | "datasource": { 665 | "type": "influxdb", 666 | "uid": "PB6B4F2F1C736D27A" 667 | }, 668 | "query": "SELECT time, value FROM metrics where type = '$type' and feature = '$feature_name' and metric_name = '$metric_name' and model_id = '$model_id' and model_version = '$model_version' and $timeFilter", 669 | "rawQuery": true, 670 | "refId": "A", 671 | "resultFormat": "time_series" 672 | } 673 | ], 674 | "title": "Heatmap for Metrics vs Time", 675 | "transparent": true, 676 | "type": "heatmap" 677 | } 678 | ], 679 | "refresh": false, 680 | "schemaVersion": 37, 681 | "style": "dark", 682 | "tags": [], 683 | "templating": { 684 | "list": [ 685 | { 686 | "current": { 687 | "selected": false, 688 | "text": "drift", 689 | "value": "drift" 690 | }, 691 | "hide": 0, 692 | "includeAll": false, 693 | "multi": false, 694 | "name": "type", 695 | "options": [ 696 | { 697 | "selected": true, 698 | "text": "drift", 699 | "value": "drift" 700 | } 701 | ], 702 | "query": "drift", 703 | "queryValue": "", 704 | "skipUrlSync": false, 705 | "type": "custom" 706 | }, 707 | { 708 | "current": { 709 | "selected": false, 710 | "text": "1", 711 | "value": "1" 712 | }, 713 | "datasource": { 714 | "type": "influxdb", 715 | "uid": "PB6B4F2F1C736D27A" 716 | }, 717 | "definition": "SELECT distinct(\"model_id\") FROM metrics where type = '$type'", 718 | "hide": 0, 719 | "includeAll": false, 720 | "multi": false, 721 | "name": "model_id", 722 | "options": [], 723 | "query": "SELECT distinct(\"model_id\") FROM metrics where type = '$type'", 724 | "refresh": 1, 725 | "regex": "", 726 | "skipUrlSync": false, 727 | "sort": 0, 728 | "type": "query" 729 | }, 730 | { 731 | "current": { 732 | "selected": false, 733 | "text": "2", 734 | "value": "2" 735 | }, 736 | "datasource": { 737 | "type": "influxdb", 738 | "uid": "PB6B4F2F1C736D27A" 739 | }, 740 | "definition": "SELECT distinct(\"model_version\") FROM metrics where type = '$type'", 741 | "hide": 0, 742 | "includeAll": false, 743 | "multi": false, 744 | "name": "model_version", 745 | "options": [], 746 | "query": "SELECT distinct(\"model_version\") FROM metrics where type = '$type'", 747 | "refresh": 1, 748 | "regex": "", 749 | "skipUrlSync": false, 750 | "sort": 0, 751 | "type": "query" 752 | }, 753 | { 754 | "current": { 755 | "selected": false, 756 | "text": "bp", 757 | "value": "bp" 758 | }, 759 | "datasource": { 760 | "type": "influxdb", 761 | "uid": "PB6B4F2F1C736D27A" 762 | }, 763 | "definition": "SELECT distinct(\"feature\") FROM metrics where type = '$type'", 764 | "hide": 0, 765 | "includeAll": false, 766 | "multi": false, 767 | "name": "feature_name", 768 | "options": [], 769 | "query": "SELECT distinct(\"feature\") FROM metrics where type = '$type'", 770 | "refresh": 1, 771 | "regex": "", 772 | "skipUrlSync": false, 773 | "sort": 0, 774 | "type": "query" 775 | }, 776 | { 777 | "current": { 778 | "selected": false, 779 | "text": "bftest", 780 | "value": "bftest" 781 | }, 782 | "datasource": { 783 | "type": "influxdb", 784 | "uid": "PB6B4F2F1C736D27A" 785 | }, 786 | "definition": "SELECT distinct(\"metric_name\") FROM metrics where type = '$type'", 787 | "hide": 0, 788 | "includeAll": false, 789 | "multi": false, 790 | "name": "metric_name", 791 | "options": [], 792 | "query": "SELECT distinct(\"metric_name\") FROM metrics where type = '$type'", 793 | "refresh": 1, 794 | "regex": "", 795 | "skipUrlSync": false, 796 | "sort": 0, 797 | "type": "query" 798 | } 799 | ] 800 | }, 801 | "time": { 802 | "from": "now-6h", 803 | "to": "now" 804 | }, 805 | "timepicker": {}, 806 | "timezone": "", 807 | "title": "DriftDetailDashboard", 808 | "uid": "Zntdh-tVz", 809 | "version": 6, 810 | "weekStart": "" 811 | } -------------------------------------------------------------------------------- /datasets/bank_num.csv: -------------------------------------------------------------------------------- 1 | age,job,marital,education,default,balance,housing,loan,contact,day,month,duration,campaign,pdays,previous,poutcome,deposit 2 | 59,0,1,1,no,2343,1,0,2,5,8,1042,1,-1,0,3,1 3 | 56,0,1,1,no,45,0,0,2,5,8,1467,1,-1,0,3,1 4 | 41,9,1,1,no,1270,1,0,2,5,8,1389,1,-1,0,3,1 5 | 55,7,1,1,no,2476,1,0,2,5,8,579,1,-1,0,3,1 6 | 54,0,1,2,no,184,0,0,2,5,8,673,2,-1,0,3,1 7 | 42,4,2,2,no,0,1,1,2,5,8,562,2,-1,0,3,1 8 | 56,4,1,2,no,830,1,1,2,6,8,1201,1,-1,0,3,1 9 | 60,5,0,1,no,545,1,0,2,6,8,1030,1,-1,0,3,1 10 | 37,9,1,1,no,1,1,0,2,6,8,608,1,-1,0,3,1 11 | 28,7,2,1,no,5090,1,0,2,6,8,1297,3,-1,0,3,1 12 | 38,0,2,1,no,100,1,0,2,7,8,786,1,-1,0,3,1 13 | 30,1,1,1,no,309,1,0,2,7,8,1574,2,-1,0,3,1 14 | 29,4,1,2,no,199,1,1,2,7,8,1689,4,-1,0,3,1 15 | 46,1,2,2,no,460,1,0,2,7,8,1102,2,-1,0,3,1 16 | 31,9,2,2,no,703,1,0,2,8,8,943,2,-1,0,3,1 17 | 35,4,0,2,no,3837,1,0,2,8,8,1084,1,-1,0,3,1 18 | 32,1,2,0,no,611,1,0,2,8,8,541,3,-1,0,3,1 19 | 49,7,1,1,no,-8,1,0,2,8,8,1119,1,-1,0,3,1 20 | 41,0,1,1,no,55,1,0,2,8,8,1120,2,-1,0,3,1 21 | 49,0,0,1,no,168,1,1,2,8,8,513,1,-1,0,3,1 22 | 28,0,0,1,no,785,1,0,2,8,8,442,2,-1,0,3,1 23 | 43,4,2,2,no,2067,1,0,2,8,8,756,1,-1,0,3,1 24 | 43,4,0,2,no,388,1,0,2,8,8,2087,2,-1,0,3,1 25 | 43,1,1,0,no,-192,1,0,2,8,8,1120,2,-1,0,3,1 26 | 37,10,2,1,no,381,1,0,2,8,8,985,2,-1,0,3,1 27 | 35,1,2,1,no,40,1,0,2,9,8,617,4,-1,0,3,1 28 | 31,9,2,2,no,22,1,0,2,9,8,483,3,-1,0,3,1 29 | 43,1,2,1,no,3,1,0,2,9,8,929,3,-1,0,3,1 30 | 31,0,1,1,no,307,1,0,2,9,8,538,1,-1,0,3,1 31 | 28,1,2,1,no,759,1,0,2,9,8,710,1,-1,0,3,1 32 | 32,1,1,1,yes,-1,1,0,2,9,8,653,1,-1,0,3,1 33 | 60,9,1,0,no,65,1,0,2,9,8,1028,2,-1,0,3,1 34 | 26,1,2,1,no,82,1,0,2,9,8,654,1,-1,0,3,1 35 | 40,1,1,1,no,10,1,0,2,9,8,1692,2,-1,0,3,1 36 | 33,1,0,0,no,390,1,0,2,9,8,665,2,-1,0,3,1 37 | 32,4,2,2,no,311,0,0,2,12,8,757,2,-1,0,3,1 38 | 35,1,1,0,no,414,0,0,2,13,8,504,4,-1,0,3,1 39 | 33,4,0,1,no,5,1,0,2,13,8,1346,1,-1,0,3,1 40 | 38,4,1,2,no,119,1,0,2,13,8,568,4,-1,0,3,1 41 | 23,2,2,0,no,4,1,0,2,13,8,395,2,-1,0,3,1 42 | 60,1,1,0,no,1262,1,1,2,13,8,1015,1,-1,0,3,1 43 | 48,4,1,2,no,1949,1,0,2,13,8,683,2,-1,0,3,1 44 | 45,2,0,2,no,-395,1,0,2,13,8,470,1,-1,0,3,1 45 | 36,1,1,1,no,1165,1,0,2,13,8,1001,4,-1,0,3,1 46 | 52,4,1,3,no,2240,1,0,2,13,8,845,1,-1,0,3,1 47 | 35,1,0,0,no,300,1,0,2,13,8,945,2,-1,0,3,1 48 | 43,9,1,1,no,3285,1,0,2,13,8,1721,2,-1,0,3,1 49 | 52,3,1,1,no,3923,1,0,2,14,8,942,3,-1,0,3,1 50 | 53,9,0,0,no,1443,1,0,2,14,8,476,1,-1,0,3,1 51 | 48,1,0,0,no,24,1,0,2,14,8,832,1,-1,0,3,1 52 | 41,1,2,0,no,1618,1,0,2,14,8,1553,1,-1,0,3,1 53 | 39,4,0,2,no,517,1,1,2,14,8,1328,1,-1,0,3,1 54 | 59,5,1,1,no,1521,1,1,2,14,8,1125,2,-1,0,3,1 55 | 41,1,1,1,no,2823,1,0,2,14,8,858,1,-1,0,3,1 56 | 48,1,1,1,no,1405,1,0,2,14,8,629,2,-1,0,3,1 57 | 40,0,1,1,no,1535,1,0,2,14,8,704,1,-1,0,3,1 58 | 48,1,1,1,no,1596,1,0,2,14,8,760,1,-1,0,3,1 59 | 60,5,1,0,no,1542,1,0,2,14,8,930,1,-1,0,3,1 60 | 40,9,2,3,no,3652,1,0,2,14,8,1028,2,-1,0,3,1 61 | 57,9,1,2,no,-1,0,0,2,14,8,850,2,-1,0,3,1 62 | 51,1,1,1,no,7180,1,0,2,14,8,927,1,-1,0,3,1 63 | 41,1,0,1,no,5291,1,0,2,14,8,1423,3,-1,0,3,1 64 | 41,1,1,1,no,1384,1,0,2,15,8,1162,4,-1,0,3,1 65 | 52,1,0,0,no,-191,1,0,2,15,8,755,1,-1,0,3,1 66 | 59,1,1,0,no,320,1,0,2,15,8,695,1,-1,0,3,1 67 | 44,1,1,3,no,146,1,0,2,15,8,483,1,-1,0,3,1 68 | 49,11,1,0,no,341,1,1,2,15,8,520,2,-1,0,3,1 69 | 40,7,1,0,no,-9,1,0,2,15,8,920,2,-1,0,3,1 70 | 41,0,1,0,no,-306,1,0,2,15,8,500,1,-1,0,3,1 71 | 44,9,1,2,no,4580,1,0,2,15,8,911,2,-1,0,3,1 72 | 60,7,2,0,no,313,1,0,2,15,8,920,2,-1,0,3,1 73 | 29,4,1,2,no,10576,0,0,2,15,8,1224,2,-1,0,3,1 74 | 41,5,0,0,no,-233,1,0,2,15,8,1156,2,-1,0,3,1 75 | 41,1,1,0,no,2453,1,0,2,15,8,1052,1,-1,0,3,1 76 | 42,9,2,1,no,1364,1,0,2,15,8,1867,6,-1,0,3,1 77 | 36,7,2,3,no,281,1,0,2,16,8,515,1,-1,0,3,1 78 | 39,9,1,2,no,94,1,1,2,16,8,813,1,-1,0,3,1 79 | 31,6,2,2,no,144,1,0,2,16,8,676,1,-1,0,3,1 80 | 26,1,1,1,no,246,1,0,2,16,8,1143,3,-1,0,3,1 81 | 31,1,1,1,no,92,1,0,2,16,8,688,1,-1,0,3,1 82 | 29,0,2,1,no,163,1,1,2,16,8,803,1,-1,0,3,1 83 | 37,7,0,1,no,49,1,0,2,16,8,619,2,-1,0,3,1 84 | 33,1,2,1,no,-416,1,0,2,16,8,767,2,-1,0,3,1 85 | 24,9,2,1,no,409,1,0,2,16,8,912,1,-1,0,3,1 86 | 50,0,0,1,no,363,1,0,2,16,8,1340,1,-1,0,3,1 87 | 27,9,2,2,no,3706,1,0,2,16,8,897,1,-1,0,3,1 88 | 54,4,2,2,no,4393,1,0,2,16,8,1297,3,-1,0,3,1 89 | 30,9,2,1,no,863,1,0,2,16,8,1193,5,-1,0,3,1 90 | 34,1,1,0,no,695,1,1,2,16,8,1064,4,-1,0,3,1 91 | 37,4,2,1,no,792,1,0,2,16,8,1187,4,-1,0,3,1 92 | 41,0,2,1,no,1020,1,0,2,19,8,882,1,-1,0,3,1 93 | 47,1,1,1,no,863,1,0,2,19,8,943,2,-1,0,3,1 94 | 31,0,1,1,no,97,1,0,2,19,8,607,1,-1,0,3,1 95 | 31,1,1,1,no,754,1,0,2,19,8,1022,4,-1,0,3,1 96 | 43,7,0,1,no,1040,1,0,2,19,8,552,3,-1,0,3,1 97 | 37,4,0,2,no,122,1,0,2,19,8,1622,2,-1,0,3,1 98 | 30,9,2,2,no,880,1,1,2,19,8,967,6,-1,0,3,1 99 | 41,9,1,1,no,501,1,0,2,19,8,579,5,-1,0,3,1 100 | 36,1,1,1,no,4438,1,0,2,20,8,446,1,-1,0,3,1 101 | 35,1,0,1,no,0,1,0,2,20,8,1205,1,-1,0,3,1 102 | 29,7,2,1,no,271,1,0,2,20,8,1882,2,-1,0,3,1 103 | 35,1,1,0,no,102,1,0,2,20,8,1334,1,-1,0,3,1 104 | 31,1,1,1,no,2,1,0,2,20,8,182,2,-1,0,3,1 105 | 35,7,1,1,no,4170,1,0,2,20,8,1777,1,-1,0,3,1 106 | 34,9,2,1,no,85,1,0,2,20,8,1182,4,-1,0,3,1 107 | 31,0,2,1,no,431,1,1,2,20,8,1045,1,-1,0,3,1 108 | 44,9,0,1,no,982,1,0,2,20,8,650,3,-1,0,3,1 109 | 36,1,1,0,no,408,1,1,2,20,8,1063,2,-1,0,3,1 110 | 35,1,1,3,no,4822,1,0,2,20,8,843,2,-1,0,3,1 111 | 41,1,1,0,no,1250,1,0,2,20,8,1392,2,-1,0,3,1 112 | 31,1,2,0,no,216,1,0,2,21,8,565,1,-1,0,3,1 113 | 34,1,1,1,no,1207,1,0,2,21,8,905,1,-1,0,3,1 114 | 32,9,1,1,no,791,1,0,2,21,8,783,2,-1,0,3,1 115 | 36,4,1,2,no,849,1,1,2,21,8,958,2,-1,0,3,1 116 | 30,1,2,1,yes,239,1,0,2,21,8,412,1,-1,0,3,1 117 | 37,9,2,1,no,1211,1,0,2,21,8,795,1,-1,0,3,1 118 | 36,1,1,0,no,599,1,0,2,21,8,649,2,-1,0,3,1 119 | 31,4,2,2,no,825,1,0,2,21,8,506,2,-1,0,3,1 120 | 27,9,1,1,no,2183,1,1,2,21,8,857,1,-1,0,3,1 121 | 34,5,1,0,no,4499,0,0,2,21,8,1681,1,-1,0,3,1 122 | 30,9,2,2,no,1289,1,0,2,21,8,1697,2,-1,0,3,1 123 | 32,9,2,2,no,4665,1,0,2,21,8,860,2,-1,0,3,1 124 | 37,9,2,1,no,3326,1,0,2,21,8,799,1,-1,0,3,1 125 | 32,1,1,1,no,783,1,0,2,21,8,923,3,-1,0,3,1 126 | 33,1,1,1,no,0,1,0,2,21,8,521,3,-1,0,3,1 127 | 35,1,1,0,no,994,1,0,2,23,8,1349,8,-1,0,3,1 128 | 35,6,0,2,no,1354,1,0,2,23,8,736,2,-1,0,3,1 129 | 30,1,2,1,no,239,1,0,2,23,8,785,1,-1,0,3,1 130 | 45,1,0,0,no,-311,1,0,2,23,8,1030,1,-1,0,3,1 131 | 45,0,1,1,no,149,1,0,2,23,8,893,3,-1,0,3,1 132 | 33,1,1,1,no,1464,1,0,2,23,8,588,3,-1,0,3,1 133 | 44,4,0,2,no,5773,0,0,2,23,8,597,6,-1,0,3,1 134 | 40,1,1,0,no,278,1,0,2,23,8,1015,3,-1,0,3,1 135 | 31,1,1,1,no,2910,1,0,2,23,8,1392,2,-1,0,3,1 136 | 30,9,1,2,no,541,1,0,2,26,8,414,2,-1,0,3,1 137 | 44,9,0,1,no,1262,1,0,2,26,8,788,11,-1,0,3,1 138 | 34,7,1,1,no,-538,1,0,2,26,8,682,1,-1,0,3,1 139 | 37,1,1,1,no,125,0,0,2,26,8,679,2,-1,0,3,1 140 | 28,6,2,2,no,0,1,0,2,26,8,1560,2,-1,0,3,1 141 | 32,0,2,1,no,620,1,0,2,26,8,1234,3,-1,0,3,1 142 | 37,1,0,1,no,316,1,0,2,26,8,642,2,-1,0,3,1 143 | 34,4,1,2,no,2287,1,0,2,26,8,895,4,-1,0,3,1 144 | 38,0,0,1,no,198,1,0,2,26,8,431,2,-1,0,3,1 145 | 28,0,2,1,no,460,1,0,2,26,8,741,6,-1,0,3,1 146 | 31,4,0,2,no,1145,1,0,2,26,8,1272,4,-1,0,3,1 147 | 59,7,1,1,no,-22,1,0,2,26,8,748,3,-1,0,3,1 148 | 24,1,1,1,no,685,1,0,2,26,8,896,3,-1,0,3,1 149 | 26,7,2,1,no,901,1,0,2,26,8,764,2,-1,0,3,1 150 | 59,5,2,1,no,351,1,0,2,27,8,1063,5,-1,0,3,1 151 | 30,2,2,0,no,0,1,1,2,27,8,1051,1,-1,0,3,1 152 | 37,0,1,1,no,274,1,0,2,27,8,731,3,-1,0,3,1 153 | 30,0,2,1,no,-213,1,1,2,27,8,751,1,-1,0,3,1 154 | 25,1,1,1,no,97,1,0,2,27,8,709,1,-1,0,3,1 155 | 44,7,0,1,no,51,1,1,2,27,8,3094,2,-1,0,3,1 156 | 29,4,1,2,no,314,0,0,2,28,8,938,2,-1,0,3,1 157 | 26,4,2,2,no,6840,1,0,2,28,8,1560,2,-1,0,3,1 158 | 26,9,2,1,no,668,1,0,2,28,8,576,3,-1,0,3,1 159 | 27,7,2,1,no,54,1,0,2,28,8,543,2,-1,0,3,1 160 | 25,6,2,1,no,1242,1,0,2,28,8,615,2,-1,0,3,1 161 | 28,4,1,1,no,292,1,0,2,28,8,507,1,-1,0,3,1 162 | 23,7,2,1,no,665,1,0,2,28,8,1183,1,-1,0,3,1 163 | 34,4,2,2,no,1058,1,0,2,28,8,864,2,-1,0,3,1 164 | 28,8,2,1,no,949,1,0,2,28,8,1730,1,-1,0,3,1 165 | 51,2,1,3,no,606,1,0,2,28,8,560,9,-1,0,3,1 166 | 35,1,1,1,no,0,1,1,2,29,8,543,1,-1,0,3,1 167 | 35,0,1,1,no,404,1,0,2,29,8,539,1,-1,0,3,1 168 | 56,1,1,1,no,249,1,0,2,29,8,791,2,-1,0,3,1 169 | 45,10,0,1,no,3354,1,0,2,29,8,746,1,-1,0,3,1 170 | 60,1,0,1,no,0,1,0,2,29,8,1207,1,-1,0,3,1 171 | 56,9,1,1,no,589,1,0,2,29,8,535,2,-1,0,3,1 172 | 27,4,2,2,no,12956,1,0,2,29,8,789,3,-1,0,3,1 173 | 30,0,1,1,no,873,1,0,2,29,8,792,3,-1,0,3,1 174 | 45,1,0,0,no,594,1,0,2,29,8,833,2,-1,0,3,1 175 | 29,1,2,1,no,-20,1,0,2,29,8,814,2,-1,0,3,1 176 | 38,4,0,2,no,4692,1,0,2,29,8,1363,2,-1,0,3,1 177 | 30,1,1,1,no,486,1,0,2,29,8,1109,3,-1,0,3,1 178 | 44,1,2,0,no,1593,1,0,2,29,8,828,3,-1,0,3,1 179 | 36,4,2,2,no,7606,1,0,2,30,8,917,1,-1,0,3,1 180 | 38,9,1,1,no,226,1,0,2,30,8,762,3,-1,0,3,1 181 | 36,1,2,1,no,366,1,1,2,30,8,1133,2,-1,0,3,1 182 | 37,4,0,1,no,16,1,0,2,30,8,638,1,-1,0,3,1 183 | 35,4,2,2,no,565,1,0,2,30,8,763,1,-1,0,3,1 184 | 38,0,0,1,no,19,1,0,2,30,8,604,2,-1,0,3,1 185 | 40,7,1,0,no,1082,1,0,2,30,8,854,2,-1,0,3,1 186 | 42,10,0,1,no,1713,1,0,2,30,8,855,2,-1,0,3,1 187 | 46,4,0,2,no,14481,1,0,2,30,8,1269,1,-1,0,3,1 188 | 35,9,0,1,no,5724,1,0,2,30,8,691,10,-1,0,3,1 189 | 34,9,0,1,no,1451,1,0,2,30,8,1097,15,-1,0,3,1 190 | 32,7,1,1,no,-34,1,0,2,30,8,1236,2,-1,0,3,1 191 | 50,10,1,1,no,3674,1,0,2,30,8,886,3,-1,0,3,1 192 | 35,1,1,1,no,698,1,1,2,30,8,1343,3,-1,0,3,1 193 | 36,9,2,1,no,4136,1,0,2,30,8,812,3,-1,0,3,1 194 | 53,1,0,0,no,2656,1,0,2,30,8,1980,2,-1,0,3,1 195 | 46,4,2,2,no,2904,1,0,2,30,8,984,2,-1,0,3,1 196 | 26,4,2,2,no,1004,1,0,2,30,8,228,2,-1,0,3,1 197 | 45,9,2,1,no,410,1,0,2,30,8,891,4,-1,0,3,1 198 | 31,10,0,0,no,0,1,0,2,30,8,745,12,-1,0,3,1 199 | 29,4,2,2,no,0,1,0,2,30,8,539,12,-1,0,3,1 200 | 53,2,2,1,no,62,1,0,2,30,8,1044,5,-1,0,3,1 201 | 27,1,1,0,no,416,1,0,2,30,8,1193,2,-1,0,3,1 202 | 39,4,1,2,no,2763,1,0,2,2,6,526,2,-1,0,3,1 203 | 36,7,1,1,no,2984,1,0,2,2,6,394,1,-1,0,3,1 204 | 41,1,1,0,no,143,1,1,2,2,6,659,2,-1,0,3,1 205 | 36,7,1,0,no,0,1,0,2,2,6,1036,1,-1,0,3,1 206 | 32,9,1,1,no,696,1,0,2,2,6,815,1,-1,0,3,1 207 | 35,7,1,1,no,152,1,0,2,2,6,563,1,-1,0,3,1 208 | 36,4,2,2,no,-78,1,0,2,2,6,1068,1,-1,0,3,1 209 | 35,4,1,2,no,867,1,0,2,2,6,230,3,-1,0,3,1 210 | 30,1,2,1,no,953,1,0,2,2,6,747,2,-1,0,3,1 211 | 42,1,1,1,no,443,1,0,2,2,6,671,5,-1,0,3,1 212 | 42,9,2,1,yes,-1129,1,0,2,2,6,555,5,-1,0,3,1 213 | 32,1,1,0,no,415,1,0,2,2,6,777,4,-1,0,3,1 214 | 29,1,2,1,no,260,1,0,2,2,6,707,14,-1,0,3,1 215 | 44,4,0,2,yes,-411,1,0,2,2,6,432,4,-1,0,3,1 216 | 32,9,2,1,no,1279,1,0,2,3,6,1173,3,-1,0,3,1 217 | 31,10,1,1,no,86,1,1,2,3,6,963,2,-1,0,3,1 218 | 40,6,1,1,yes,-754,1,1,2,3,6,941,3,-1,0,3,1 219 | 43,2,1,1,no,0,0,0,2,3,6,1025,7,-1,0,3,1 220 | 27,4,2,1,no,178,1,0,2,3,6,732,2,-1,0,3,1 221 | 37,7,2,1,no,278,1,0,2,3,6,1045,3,-1,0,3,1 222 | 23,7,2,1,no,425,1,0,2,3,6,768,1,-1,0,3,1 223 | 25,4,1,0,no,168,1,0,2,3,6,801,1,-1,0,3,1 224 | 26,1,2,1,yes,-36,1,0,2,3,6,482,2,-1,0,3,1 225 | 47,1,0,0,no,126,1,0,2,3,6,2456,2,-1,0,3,1 226 | 35,0,1,1,no,-87,1,0,2,3,6,1363,7,-1,0,3,1 227 | 32,9,2,2,no,6281,1,0,2,3,6,1336,4,-1,0,3,1 228 | 45,4,0,1,no,644,1,0,2,4,6,633,1,-1,0,3,1 229 | 30,0,0,1,no,377,1,0,2,4,6,524,15,-1,0,3,1 230 | 29,7,2,1,no,-271,1,0,2,4,6,578,1,-1,0,3,1 231 | 38,1,1,0,no,5839,1,0,2,4,6,984,2,-1,0,3,1 232 | 23,1,2,1,no,492,1,0,2,4,6,638,1,-1,0,3,1 233 | 27,4,2,2,no,99,1,0,2,4,6,767,2,-1,0,3,1 234 | 27,4,2,2,yes,-713,1,0,2,4,6,525,2,-1,0,3,1 235 | 29,7,2,1,no,32,1,0,2,4,6,563,1,-1,0,3,1 236 | 30,6,1,2,no,543,1,1,2,4,6,1449,6,-1,0,3,1 237 | 36,1,1,0,no,403,1,0,2,4,6,920,1,-1,0,3,1 238 | 35,0,0,1,no,437,1,0,2,4,6,908,1,-1,0,3,1 239 | 36,1,1,1,no,1772,0,0,2,5,6,208,1,-1,0,3,1 240 | 59,5,0,0,no,-88,1,0,2,5,6,910,2,-1,0,3,1 241 | 47,2,1,1,no,0,0,0,2,5,6,1446,1,-1,0,3,1 242 | 58,2,1,2,no,853,0,0,2,5,6,1149,2,-1,0,3,1 243 | 37,4,2,2,no,240,0,0,2,5,6,1123,1,-1,0,3,1 244 | 46,0,2,1,no,849,1,0,2,5,6,508,2,-1,0,3,1 245 | 43,1,2,1,no,318,1,0,2,5,6,736,2,-1,0,3,1 246 | 27,4,2,2,no,703,1,0,2,5,6,590,3,-1,0,3,1 247 | 38,7,0,1,no,61,0,0,2,5,6,636,1,-1,0,3,1 248 | 53,3,1,0,no,104,0,1,2,5,6,701,1,-1,0,3,1 249 | 47,1,1,1,no,1996,0,0,2,5,6,761,2,-1,0,3,1 250 | 56,3,0,0,yes,1238,0,0,2,5,6,1558,1,-1,0,3,1 251 | 41,3,1,1,no,879,0,0,2,5,6,1053,2,-1,0,3,1 252 | 29,6,1,1,no,425,1,0,2,5,6,562,15,-1,0,3,1 253 | 50,9,1,1,no,416,1,0,2,5,6,494,5,-1,0,3,1 254 | 45,1,0,0,no,844,0,0,2,5,6,1018,3,-1,0,3,1 255 | 35,1,1,1,no,860,1,0,2,5,6,884,4,-1,0,3,1 256 | 26,9,2,1,no,639,1,0,2,6,6,709,5,-1,0,3,1 257 | 26,0,1,1,no,710,1,1,2,6,6,1276,2,-1,0,3,1 258 | 37,1,1,3,no,1508,0,1,2,6,6,381,2,-1,0,3,1 259 | 30,0,2,1,no,223,1,0,2,6,6,862,11,-1,0,3,1 260 | 38,4,2,2,no,619,1,0,2,6,6,460,4,-1,0,3,1 261 | 37,7,0,1,no,319,1,0,2,6,6,467,3,-1,0,3,1 262 | 53,9,1,1,no,480,0,0,2,6,6,648,2,-1,0,3,1 263 | 29,1,2,0,no,213,1,0,2,6,6,2653,3,-1,0,3,1 264 | 35,4,2,2,no,68,0,0,2,6,6,1085,1,-1,0,3,1 265 | 28,1,1,1,no,2788,1,0,2,6,6,369,2,-1,0,3,1 266 | 55,1,1,3,no,7561,0,0,2,6,6,685,3,-1,0,3,1 267 | 49,4,0,1,no,-46,1,0,2,9,6,1055,10,-1,0,3,1 268 | 50,7,1,1,no,1595,0,0,2,9,6,882,1,-1,0,3,1 269 | 43,4,2,2,no,1046,0,0,2,9,6,901,1,-1,0,3,1 270 | 58,6,0,2,no,1382,0,0,2,9,6,700,1,-1,0,3,1 271 | 38,9,1,0,no,217,1,0,2,9,6,491,6,-1,0,3,1 272 | 48,1,0,0,no,183,0,0,2,9,6,940,1,-1,0,3,1 273 | 59,4,1,1,no,1321,0,0,2,9,6,3881,3,-1,0,3,1 274 | 58,1,1,1,no,25,0,0,2,9,6,486,3,-1,0,3,1 275 | 36,6,2,2,no,4844,0,0,2,9,6,1137,3,-1,0,3,1 276 | 42,2,1,2,no,5345,0,0,2,11,6,878,3,-1,0,3,1 277 | 41,0,2,1,no,812,0,1,2,11,6,583,4,-1,0,3,1 278 | 42,0,1,1,no,105,0,0,2,11,6,1159,4,-1,0,3,1 279 | 39,4,1,2,no,-190,0,1,2,11,6,893,8,-1,0,3,1 280 | 57,3,1,0,no,625,0,1,2,11,6,867,1,-1,0,3,1 281 | 52,5,1,3,no,293,0,0,2,11,6,706,1,-1,0,3,1 282 | 59,5,1,3,no,1033,0,0,2,11,6,1199,1,-1,0,3,1 283 | 56,1,1,1,no,1210,0,0,2,11,6,935,1,-1,0,3,1 284 | 49,4,1,2,no,64,0,0,2,11,6,586,1,-1,0,3,1 285 | 35,1,0,3,no,164,0,0,2,11,6,967,2,-1,0,3,1 286 | 36,10,1,1,no,1766,0,0,2,11,6,595,2,-1,0,3,1 287 | 40,0,2,2,no,1248,0,0,2,11,6,1290,2,-1,0,3,1 288 | 55,1,1,1,no,0,0,0,2,12,6,614,2,-1,0,3,1 289 | 58,0,1,1,no,826,0,0,2,12,6,885,1,-1,0,3,1 290 | 39,6,1,2,no,2630,0,0,2,12,6,651,5,-1,0,3,1 291 | 47,3,2,2,no,3727,0,0,2,12,6,993,6,-1,0,3,1 292 | 40,4,1,1,no,406,1,0,2,12,6,577,2,-1,0,3,1 293 | 35,1,1,2,no,2999,0,0,2,12,6,1141,6,-1,0,3,1 294 | 48,1,1,1,no,1000,1,0,2,12,6,1268,2,-1,0,3,1 295 | 48,4,0,2,no,503,0,1,2,12,6,1243,7,-1,0,3,1 296 | 24,1,2,1,yes,-203,1,0,2,12,6,465,3,-1,0,3,1 297 | 49,1,1,0,no,361,1,0,2,16,6,513,2,-1,0,3,1 298 | 36,1,1,1,no,274,1,0,2,16,6,686,11,-1,0,3,1 299 | 52,5,0,2,no,3687,1,1,2,16,6,741,4,-1,0,3,1 300 | 42,2,2,3,no,323,0,0,2,16,6,617,1,-1,0,3,1 301 | 36,7,1,1,no,981,0,0,2,16,6,1093,1,-1,0,3,1 302 | 36,1,0,1,no,638,1,0,2,16,6,1395,2,-1,0,3,1 303 | 48,10,2,2,no,3229,0,0,2,16,6,1089,1,-1,0,3,1 304 | 51,4,2,1,no,1610,1,0,2,16,6,1248,3,-1,0,3,1 305 | 51,1,1,1,no,33,0,1,2,16,6,721,3,-1,0,3,1 306 | 46,9,1,1,no,536,1,0,2,16,6,750,2,-1,0,3,1 307 | 37,4,1,2,no,0,0,0,2,16,6,637,6,-1,0,3,1 308 | 36,2,1,2,no,3057,0,0,2,16,6,2769,4,-1,0,3,1 309 | 47,4,1,1,no,2892,0,0,2,17,6,556,1,-1,0,3,1 310 | 40,9,0,2,no,0,0,0,2,17,6,522,1,-1,0,3,1 311 | 38,8,2,2,no,3316,0,0,2,17,6,1345,3,-1,0,3,1 312 | 40,4,1,1,no,106,0,0,2,17,6,676,2,-1,0,3,1 313 | 39,1,1,0,no,2303,1,0,2,17,6,775,3,-1,0,3,1 314 | 60,4,1,2,no,3301,0,0,2,17,6,2621,3,-1,0,3,1 315 | 39,6,2,2,no,426,0,0,2,18,6,1029,1,-1,0,3,1 316 | 51,7,0,1,no,9,0,0,2,18,6,371,2,-1,0,3,1 317 | 27,0,1,1,no,-97,1,0,2,18,6,1528,2,-1,0,3,1 318 | 41,4,2,2,no,11008,0,0,2,18,6,1540,1,-1,0,3,1 319 | 42,1,2,0,no,4930,0,0,2,18,6,973,1,-1,0,3,1 320 | 41,2,1,1,no,707,0,0,2,18,6,707,1,-1,0,3,1 321 | 33,9,1,1,no,4,1,1,2,18,6,625,2,-1,0,3,1 322 | 46,0,1,1,no,126,0,0,2,18,6,1255,1,-1,0,3,1 323 | 31,1,1,1,no,671,1,0,2,18,6,704,4,-1,0,3,1 324 | 41,1,1,0,no,219,0,1,2,18,6,1574,3,-1,0,3,1 325 | 41,1,0,1,no,3622,0,0,2,19,6,1135,4,-1,0,3,1 326 | 55,5,0,1,no,1580,0,1,2,19,6,1007,1,-1,0,3,1 327 | 53,9,1,1,no,1319,0,0,2,19,6,1318,3,-1,0,3,1 328 | 55,3,1,0,no,2152,0,0,2,19,6,922,2,-1,0,3,1 329 | 39,7,1,1,no,2843,1,0,2,19,6,585,1,-1,0,3,1 330 | 40,2,1,2,yes,0,0,0,2,19,6,952,4,-1,0,3,1 331 | 59,5,1,0,no,1115,0,0,2,19,6,834,1,-1,0,3,1 332 | 42,1,2,1,no,625,0,0,2,19,6,651,3,-1,0,3,1 333 | 41,1,1,0,no,246,0,0,2,19,6,683,3,-1,0,3,1 334 | 27,1,1,1,no,61,1,0,2,19,6,1012,1,-1,0,3,1 335 | 25,1,2,1,no,-191,1,0,2,19,6,958,7,-1,0,3,1 336 | 43,10,1,0,no,519,0,0,2,20,6,973,2,-1,0,3,1 337 | 44,7,1,1,no,522,0,1,2,20,6,911,1,-1,0,3,1 338 | 26,1,2,1,no,-46,1,0,2,20,6,382,7,-1,0,3,1 339 | 52,5,1,2,no,2269,0,0,2,20,6,1210,1,-1,0,3,1 340 | 54,7,1,3,no,386,0,1,2,20,6,838,1,-1,0,3,1 341 | 56,4,1,2,no,0,0,1,2,20,6,422,1,-1,0,3,1 342 | 57,4,1,2,no,9103,0,1,2,20,6,1098,2,-1,0,3,1 343 | 29,1,2,1,no,908,1,0,2,20,6,1663,1,-1,0,3,1 344 | 47,9,1,2,no,1694,0,1,2,20,6,560,3,-1,0,3,1 345 | 57,4,1,2,no,-92,1,1,2,20,6,1617,1,-1,0,3,1 346 | 40,1,1,1,no,188,1,1,2,20,6,570,2,-1,0,3,1 347 | 46,7,0,1,no,0,0,0,2,20,6,1023,3,-1,0,3,1 348 | 43,1,2,1,no,740,1,0,2,20,6,585,2,-1,0,3,1 349 | 45,4,1,2,no,655,0,0,2,20,6,693,3,-1,0,3,1 350 | 24,4,1,1,no,867,1,0,2,20,6,546,10,-1,0,3,1 351 | 40,1,1,1,no,1009,1,1,2,20,6,1036,5,-1,0,3,1 352 | 38,7,0,3,no,6360,0,0,2,20,6,1409,2,-1,0,3,1 353 | 36,10,1,2,no,4145,0,0,2,20,6,988,4,-1,0,3,1 354 | 27,1,2,1,yes,-27,0,0,2,23,6,498,2,-1,0,3,1 355 | 26,0,2,0,no,-122,1,0,2,23,6,670,3,-1,0,3,1 356 | 43,4,1,2,no,149,1,0,2,23,6,662,2,-1,0,3,1 357 | 25,1,2,0,no,52,1,0,2,24,6,528,2,-1,0,3,1 358 | 27,1,1,2,no,335,1,0,2,26,6,519,3,-1,0,3,1 359 | 30,9,1,2,no,526,1,1,2,26,6,677,3,-1,0,3,1 360 | 28,0,2,1,no,-7,0,1,2,26,6,3183,2,-1,0,3,1 361 | 27,1,1,0,no,-413,0,1,2,27,6,422,6,-1,0,3,1 362 | 32,0,0,1,no,-241,0,1,2,27,6,856,2,-1,0,3,1 363 | 41,1,2,1,no,75,1,0,2,1,5,479,9,-1,0,3,1 364 | 37,9,2,2,no,0,0,0,2,1,5,1135,2,-1,0,3,1 365 | 27,9,2,1,no,158,0,0,2,1,5,650,2,-1,0,3,1 366 | 29,7,2,1,no,84,0,1,2,3,5,587,3,-1,0,3,1 367 | 33,7,1,1,no,114,0,1,2,3,5,676,1,-1,0,3,1 368 | 36,9,1,0,no,0,0,0,2,4,5,584,1,-1,0,3,1 369 | 28,1,2,1,no,-127,1,0,0,4,5,1044,3,-1,0,3,1 370 | 55,1,1,1,no,49,1,0,0,7,5,494,4,-1,0,3,1 371 | 51,9,1,1,no,216,0,0,0,7,5,615,2,-1,0,3,1 372 | 39,9,2,1,no,506,1,0,0,7,5,718,2,-1,0,3,1 373 | 42,1,0,0,no,213,1,0,0,7,5,434,3,-1,0,3,1 374 | 33,7,2,1,no,170,0,0,0,7,5,782,1,-1,0,3,1 375 | 36,2,0,2,no,776,0,1,0,7,5,722,3,-1,0,3,1 376 | 45,9,1,1,no,879,0,0,0,7,5,621,2,-1,0,3,1 377 | 37,6,1,1,no,1633,0,0,0,7,5,629,2,-1,0,3,1 378 | 46,10,1,1,no,926,0,0,0,7,5,385,5,-1,0,3,1 379 | 32,6,2,2,no,2559,0,0,0,7,5,889,1,-1,0,3,1 380 | 28,9,2,1,no,909,0,1,0,7,5,532,1,-1,0,3,1 381 | 45,0,0,1,no,-176,0,1,0,7,5,574,1,-1,0,3,1 382 | 41,1,2,1,yes,1085,1,1,0,7,5,599,1,-1,0,3,1 383 | 59,1,0,0,no,1265,0,1,0,7,5,849,1,-1,0,3,1 384 | 47,9,2,1,no,-239,1,1,0,7,5,973,3,-1,0,3,1 385 | 31,1,2,1,no,726,0,0,0,7,5,719,2,-1,0,3,1 386 | 31,4,2,2,no,296,0,1,0,8,5,509,4,-1,0,3,1 387 | 33,2,1,1,no,-627,1,1,0,8,5,740,1,-1,0,3,1 388 | 52,2,1,2,yes,-468,0,0,0,8,5,534,1,-1,0,3,1 389 | 26,6,2,1,no,551,0,0,0,8,5,531,1,-1,0,3,1 390 | 25,9,2,1,no,0,0,1,0,8,5,634,1,-1,0,3,1 391 | 25,1,1,1,no,101,0,1,0,8,5,460,3,-1,0,3,1 392 | 28,1,2,1,no,517,0,1,0,8,5,681,1,-1,0,3,1 393 | 33,2,0,2,no,37,0,1,0,8,5,1082,1,-1,0,3,1 394 | 45,4,0,2,no,854,0,1,0,8,5,730,1,-1,0,3,1 395 | 36,7,1,1,no,30,0,1,0,8,5,716,5,-1,0,3,1 396 | 26,1,2,1,yes,81,0,1,0,8,5,803,2,-1,0,3,1 397 | 38,9,1,1,no,0,0,0,0,8,5,635,1,-1,0,3,1 398 | 56,1,2,1,no,663,0,1,0,8,5,819,3,-1,0,3,1 399 | 26,0,2,1,no,102,0,1,0,8,5,658,2,-1,0,3,1 400 | 26,9,1,1,no,370,0,1,0,8,5,1061,4,-1,0,3,1 401 | 25,1,2,1,no,189,0,1,0,8,5,687,2,-1,0,3,1 402 | 26,10,2,1,no,111,0,0,0,9,5,537,1,-1,0,3,1 403 | 40,1,1,0,no,665,1,0,0,9,5,781,1,-1,0,3,1 404 | 37,0,1,0,no,214,1,0,0,9,5,645,1,-1,0,3,1 405 | 43,1,2,0,no,0,0,0,0,9,5,1242,1,-1,0,3,1 406 | 41,7,0,2,no,1583,1,0,0,9,5,662,1,-1,0,3,1 407 | 49,1,1,1,no,0,0,1,0,9,5,625,1,-1,0,3,1 408 | 29,0,0,2,yes,0,0,1,0,9,5,426,3,-1,0,3,1 409 | 39,7,0,1,no,687,1,0,0,9,5,869,1,-1,0,3,1 410 | 38,4,1,2,yes,327,1,1,0,9,5,556,5,-1,0,3,1 411 | 26,9,1,1,no,-119,0,1,0,9,5,359,5,-1,0,3,1 412 | 32,4,1,2,no,794,1,1,0,9,5,1014,1,-1,0,3,1 413 | 33,1,1,0,no,-191,1,1,0,9,5,1290,4,-1,0,3,1 414 | 31,4,2,2,no,409,0,1,0,9,5,564,5,-1,0,3,1 415 | 49,1,1,0,no,3,0,1,0,9,5,1439,6,-1,0,3,1 416 | 27,10,2,0,no,9,0,1,0,9,5,919,2,-1,0,3,1 417 | 30,1,2,1,no,155,1,1,0,9,5,1426,3,-1,0,3,1 418 | 35,1,1,1,no,1330,0,0,0,10,5,901,1,-1,0,3,1 419 | 26,1,2,1,no,-29,1,0,0,10,5,998,4,-1,0,3,1 420 | 45,2,1,3,no,3133,1,1,0,10,5,804,1,-1,0,3,1 421 | 34,0,1,1,no,-33,1,1,0,10,5,961,1,-1,0,3,1 422 | 28,0,2,1,no,-170,0,1,0,10,5,720,2,-1,0,3,1 423 | 25,1,2,1,no,1243,0,1,0,10,5,1341,7,-1,0,3,1 424 | 30,1,2,1,no,1598,1,0,0,10,5,634,1,-1,0,3,1 425 | 30,7,1,1,no,5,0,1,0,10,5,829,5,-1,0,3,1 426 | 47,7,1,1,no,367,1,0,0,10,5,476,4,-1,0,3,1 427 | 57,0,0,1,no,658,0,0,0,10,5,724,1,-1,0,3,1 428 | 38,4,1,2,no,1722,1,0,0,10,5,500,1,-1,0,3,1 429 | 54,9,1,1,no,-315,0,1,0,10,5,2029,1,-1,0,3,1 430 | 25,1,2,0,no,2483,0,0,0,10,5,1499,3,-1,0,3,1 431 | 30,1,1,1,no,0,1,1,0,10,5,1399,1,-1,0,3,1 432 | 26,4,2,2,no,45,0,0,0,10,5,1187,3,-1,0,3,1 433 | 31,8,1,0,no,46,1,0,0,10,5,487,1,-1,0,3,1 434 | 29,1,2,1,no,482,0,1,0,10,5,1097,3,-1,0,3,1 435 | 49,1,1,1,no,3728,1,0,0,11,5,1060,2,-1,0,3,1 436 | 60,5,0,1,no,1099,0,0,0,11,5,764,2,-1,0,3,1 437 | 60,4,1,2,no,2222,1,0,0,11,5,1120,2,-1,0,3,1 438 | 29,4,2,2,no,983,1,0,0,11,5,963,2,-1,0,3,1 439 | 55,4,0,1,no,204,1,0,0,11,5,1973,2,-1,0,3,1 440 | 44,0,1,1,no,1074,1,0,0,11,5,911,2,-1,0,3,1 441 | 47,9,1,1,no,3466,1,0,0,11,5,853,2,-1,0,3,1 442 | 35,0,1,1,no,-203,1,0,0,11,5,1649,2,-1,0,3,1 443 | 35,2,1,1,no,2436,0,1,0,14,5,1397,3,-1,0,3,1 444 | 38,9,1,1,no,976,1,0,0,14,5,766,3,-1,0,3,1 445 | 34,2,1,2,no,146,1,0,0,14,5,788,2,-1,0,3,1 446 | 50,9,1,1,no,156,0,1,0,14,5,1130,3,-1,0,3,1 447 | 43,0,1,2,no,189,1,0,0,14,5,1062,4,-1,0,3,1 448 | 44,0,0,1,no,163,1,0,0,14,5,1669,2,-1,0,3,1 449 | 51,7,0,1,no,242,0,1,0,14,5,1336,4,-1,0,3,1 450 | 31,9,1,1,no,1173,1,0,0,14,5,444,3,-1,0,3,1 451 | 39,9,1,2,no,0,1,0,0,14,5,894,7,-1,0,3,1 452 | 59,4,1,2,no,296,0,1,0,15,5,805,3,-1,0,3,1 453 | 25,7,2,1,no,333,0,1,0,15,5,1056,2,-1,0,3,1 454 | 58,4,1,2,no,3864,1,0,0,15,5,815,4,-1,0,3,1 455 | 30,2,1,0,no,2686,1,0,0,15,5,808,3,-1,0,3,1 456 | 34,1,2,0,no,-109,1,0,0,15,5,860,4,-1,0,3,1 457 | 51,7,0,1,no,65,0,1,0,15,5,881,1,-1,0,3,1 458 | 60,5,1,1,no,597,0,0,0,15,5,981,1,-1,0,3,1 459 | 51,2,1,1,no,-799,0,1,0,15,5,1001,3,-1,0,3,1 460 | 37,4,0,2,no,1775,0,0,0,15,5,514,5,-1,0,3,1 461 | 30,1,2,0,no,0,0,0,0,15,5,716,3,-1,0,3,1 462 | 46,2,0,1,no,-563,0,1,0,15,5,769,4,-1,0,3,1 463 | 29,8,2,2,no,5,0,0,0,15,5,889,5,-1,0,3,1 464 | 27,2,1,2,no,139,0,0,0,15,5,448,5,-1,0,3,1 465 | 28,4,2,2,no,0,1,0,0,16,5,767,5,-1,0,3,1 466 | 34,1,2,1,no,855,1,0,0,16,5,578,1,-1,0,3,1 467 | 29,1,2,3,no,694,1,1,0,16,5,1806,4,-1,0,3,1 468 | 27,4,2,2,no,151,1,0,0,16,5,496,1,-1,0,3,1 469 | 26,9,2,1,no,354,0,0,0,16,5,586,1,-1,0,3,1 470 | 36,4,1,2,no,673,1,0,0,16,5,689,1,-1,0,3,1 471 | 44,1,1,0,no,1265,1,0,0,16,5,555,1,-1,0,3,1 472 | 39,1,1,2,no,597,0,0,0,16,5,686,6,-1,0,3,1 473 | 30,9,1,1,no,-393,1,1,0,16,5,435,1,-1,0,3,1 474 | 31,10,0,2,no,256,1,0,0,16,5,873,2,-1,0,3,1 475 | 56,1,1,0,no,0,1,0,0,16,5,1222,1,-1,0,3,1 476 | 29,1,1,1,no,2870,1,0,0,16,5,988,1,-1,0,3,1 477 | 43,1,1,1,no,184,0,1,0,16,5,1019,11,-1,0,3,1 478 | 32,1,1,1,no,1905,1,0,0,16,5,709,2,-1,0,3,1 479 | 30,0,2,1,no,-36,0,0,0,16,5,561,9,-1,0,3,1 480 | 40,1,1,1,no,-839,1,1,0,16,5,1018,2,-1,0,3,1 481 | 42,7,1,1,no,3465,1,1,0,16,5,1039,2,-1,0,3,1 482 | 27,8,1,1,no,119,1,0,0,16,5,781,2,-1,0,3,1 483 | 35,4,1,2,no,36,0,1,0,16,5,1656,5,-1,0,3,1 484 | 35,3,1,0,no,113,1,0,0,17,5,923,2,-1,0,3,1 485 | 32,1,2,1,no,118,1,0,0,17,5,1275,1,-1,0,3,1 486 | 39,9,1,1,no,251,1,0,0,17,5,342,2,-1,0,3,1 487 | 31,1,0,1,no,-277,1,0,1,17,5,1008,2,-1,0,3,1 488 | 26,7,2,1,no,-189,0,1,0,17,5,538,2,-1,0,3,1 489 | 42,4,1,2,no,4,1,0,0,17,5,795,1,-1,0,3,1 490 | 27,7,2,1,no,1820,1,0,0,17,5,1027,1,-1,0,3,1 491 | 33,6,0,1,no,1904,1,0,0,17,5,1584,2,-1,0,3,1 492 | 51,1,1,0,no,201,1,0,0,17,5,582,2,-1,0,3,1 493 | 33,0,1,1,no,0,1,0,1,17,5,1448,7,-1,0,3,1 494 | 30,9,2,2,no,178,0,0,0,17,5,656,3,-1,0,3,1 495 | 52,3,1,0,no,3798,1,0,0,17,5,1208,1,-1,0,3,1 496 | 42,1,0,0,no,301,1,0,0,17,5,1175,2,-1,0,3,1 497 | 31,4,0,2,no,294,1,0,0,17,5,536,5,-1,0,3,1 498 | 27,7,1,1,no,0,1,0,0,17,5,991,5,-1,0,3,1 499 | 44,1,2,1,no,292,0,1,0,17,5,1153,4,-1,0,3,1 500 | 40,2,0,1,no,2998,1,0,0,18,5,623,3,-1,0,3,1 501 | 46,3,0,0,no,2084,1,0,0,18,5,1081,2,-1,0,3,1 502 | 32,7,0,1,no,454,1,0,0,18,5,978,1,-1,0,3,1 503 | 56,9,0,1,no,179,0,0,0,18,5,536,1,-1,0,3,1 504 | 33,1,1,1,no,661,1,0,0,18,5,968,1,-1,0,3,1 505 | 33,1,1,1,no,89,1,0,0,18,5,278,2,-1,0,3,1 506 | 32,6,1,0,no,102,1,0,0,18,5,560,1,-1,0,3,1 507 | 41,1,0,0,no,138,1,0,0,18,5,640,3,-1,0,3,1 508 | 31,0,2,1,no,622,0,0,0,18,5,420,1,-1,0,3,1 509 | 30,0,0,1,no,10,1,0,0,18,5,658,2,-1,0,3,1 510 | 34,0,0,1,no,-251,0,0,0,18,5,641,1,-1,0,3,1 511 | 43,2,1,1,no,104,1,0,0,18,5,635,3,-1,0,3,1 512 | 28,1,1,1,no,173,0,0,0,18,5,588,7,-1,0,3,1 513 | 32,0,0,1,no,-46,0,1,0,18,5,1373,4,-1,0,3,1 514 | 31,1,0,1,no,0,1,1,0,18,5,913,4,-1,0,3,1 515 | 35,2,2,2,no,145,1,0,0,18,5,799,2,-1,0,3,1 516 | 31,4,1,2,no,325,1,0,0,18,5,1139,2,-1,0,3,1 517 | 25,7,2,1,no,2311,0,1,0,21,5,1105,2,-1,0,3,1 518 | 56,5,0,2,no,158,0,1,0,21,5,854,1,-1,0,3,1 519 | 47,9,1,1,no,1981,1,0,0,21,5,919,2,-1,0,3,1 520 | 39,4,1,2,no,1423,1,0,1,21,5,733,3,-1,0,3,1 521 | 51,4,1,2,no,2455,1,0,0,21,5,553,1,-1,0,3,1 522 | 54,9,0,1,no,859,0,1,0,21,5,710,1,-1,0,3,1 523 | 58,5,1,3,no,9004,0,0,0,21,5,891,4,-1,0,3,1 524 | 42,10,1,3,no,970,1,0,1,21,5,691,10,-1,0,3,1 525 | 56,7,1,0,no,486,0,1,1,21,5,1877,1,-1,0,3,1 526 | 55,3,1,1,no,127,1,0,1,21,5,1130,8,-1,0,3,1 527 | 26,1,2,0,no,409,1,0,0,21,5,577,2,-1,0,3,1 528 | 40,0,2,1,no,664,0,0,0,21,5,1342,2,-1,0,3,1 529 | 26,9,2,1,no,223,0,1,0,21,5,1002,3,-1,0,3,1 530 | 35,4,2,2,no,-324,0,1,0,21,5,985,11,-1,0,3,1 531 | 34,7,2,1,no,-30,0,0,0,21,5,1360,3,-1,0,3,1 532 | 41,4,2,2,no,1253,0,0,0,21,5,1134,5,-1,0,3,1 533 | 44,4,1,0,no,4758,1,0,0,22,5,712,6,-1,0,3,1 534 | 33,4,2,2,no,30,1,0,0,22,5,1077,1,-1,0,3,1 535 | 36,4,0,2,no,81,0,1,0,22,5,560,1,-1,0,3,1 536 | 38,9,1,1,no,0,1,0,0,22,5,1545,1,-1,0,3,1 537 | 35,9,1,2,no,328,1,0,0,22,5,654,2,-1,0,3,1 538 | 41,9,2,2,no,145,0,0,0,22,5,1833,2,-1,0,3,1 539 | 41,4,0,3,no,465,1,0,0,22,5,1508,1,-1,0,3,1 540 | 28,1,1,1,no,751,1,0,0,22,5,598,5,-1,0,3,1 541 | 26,7,1,1,no,-379,1,1,0,22,5,1237,7,-1,0,3,1 542 | 39,1,2,1,no,0,0,0,0,22,5,864,4,-1,0,3,1 543 | 25,0,1,3,no,349,1,0,0,22,5,1037,8,-1,0,3,1 544 | 37,0,1,1,no,1110,1,0,1,22,5,802,11,-1,0,3,1 545 | 25,4,1,1,no,2228,1,0,0,23,5,754,1,-1,0,3,1 546 | 46,4,0,2,no,2166,1,0,0,23,5,870,1,-1,0,3,1 547 | 25,1,2,1,no,-247,1,0,0,23,5,633,1,-1,0,3,1 548 | 28,9,2,2,no,-123,0,1,0,23,5,690,1,-1,0,3,1 549 | 33,7,0,1,no,44,0,1,0,23,5,529,1,-1,0,3,1 550 | 32,4,2,1,no,131,1,1,0,23,5,1152,2,-1,0,3,1 551 | 32,1,1,1,no,476,1,0,0,23,5,956,9,-1,0,3,1 552 | 47,9,2,1,no,1101,1,0,0,23,5,957,3,-1,0,3,1 553 | 50,2,1,2,no,-168,1,0,0,23,5,1327,2,-1,0,3,1 554 | 27,9,1,1,no,-392,1,0,0,23,5,228,2,-1,0,3,1 555 | 45,0,1,1,no,524,1,0,0,23,5,808,1,-1,0,3,1 556 | 51,0,2,1,no,895,0,0,0,23,5,638,2,-1,0,3,1 557 | 36,0,1,1,no,212,1,1,0,23,5,1201,6,-1,0,3,1 558 | 51,6,1,1,no,95,1,0,0,23,5,1309,4,-1,0,3,1 559 | 33,7,2,2,no,395,1,0,0,23,5,1359,6,-1,0,3,1 560 | 59,9,1,1,no,135,1,1,0,23,5,416,4,-1,0,3,1 561 | 38,2,1,2,no,0,0,0,0,23,5,532,2,-1,0,3,1 562 | 40,7,1,1,no,471,1,0,0,24,5,430,4,-1,0,3,1 563 | 51,1,1,0,no,603,1,0,0,24,5,853,2,-1,0,3,1 564 | 41,2,2,1,no,663,0,0,0,24,5,1204,1,-1,0,3,1 565 | 38,2,1,1,no,593,1,1,0,24,5,1484,24,-1,0,3,1 566 | 31,4,1,2,no,1224,1,1,0,24,5,1441,2,-1,0,3,1 567 | 28,0,2,1,no,2,1,0,0,24,5,454,1,-1,0,3,1 568 | 41,1,1,1,yes,720,0,1,0,24,5,651,1,-1,0,3,1 569 | 33,4,0,2,no,990,1,0,0,24,5,1491,2,-1,0,3,1 570 | 25,8,2,1,no,1747,1,0,0,24,5,903,1,-1,0,3,1 571 | 45,0,0,1,no,59,1,0,0,24,5,873,8,-1,0,3,1 572 | 31,1,1,0,no,33,1,0,0,24,5,901,2,-1,0,3,1 573 | 28,7,2,1,no,340,1,0,0,24,5,821,2,-1,0,3,1 574 | 49,9,0,1,no,305,0,0,0,24,5,367,2,-1,0,3,1 575 | 25,7,1,1,no,0,1,0,0,24,5,458,9,-1,0,3,1 576 | 28,10,2,2,no,497,1,0,0,24,5,1602,6,-1,0,3,1 577 | 54,5,1,1,no,1371,0,0,1,25,5,1492,2,-1,0,3,1 578 | 28,9,2,2,no,1628,1,0,0,25,5,1422,2,-1,0,3,1 579 | 27,9,2,2,no,372,1,1,0,25,5,728,2,-1,0,3,1 580 | 49,1,2,0,no,566,1,0,0,25,5,979,2,-1,0,3,1 581 | 53,1,1,0,no,343,1,0,0,25,5,616,5,-1,0,3,1 582 | 26,9,2,1,no,941,0,1,0,25,5,784,1,-1,0,3,1 583 | 37,7,1,1,no,775,0,1,0,25,5,1000,1,-1,0,3,1 584 | 34,1,2,1,no,1759,0,1,0,25,5,695,1,-1,0,3,1 585 | 51,5,1,0,no,1495,0,0,1,25,5,1946,4,-1,0,3,1 586 | 39,3,1,1,no,1444,0,0,1,25,5,362,8,-1,0,3,1 587 | 35,9,2,2,no,470,1,0,0,25,5,743,13,-1,0,3,1 588 | 51,10,1,0,no,1162,1,0,0,25,5,2015,1,-1,0,3,1 589 | 30,4,2,2,no,2,1,1,0,25,5,1031,8,-1,0,3,1 590 | 42,1,2,0,no,1628,1,0,0,25,5,590,8,-1,0,3,1 591 | 54,4,1,1,no,1134,1,0,0,25,5,1330,3,-1,0,3,1 592 | 51,1,1,0,no,0,1,1,0,25,5,946,3,-1,0,3,1 593 | 57,5,0,0,no,63,0,1,0,25,5,1448,17,-1,0,3,1 594 | 30,4,1,2,no,1221,1,0,1,25,5,279,4,-1,0,3,1 595 | 37,1,2,1,no,5024,0,1,0,25,5,661,4,-1,0,3,1 596 | 39,9,1,1,no,10685,1,0,0,25,5,1369,9,-1,0,3,1 597 | 35,9,1,2,no,1629,0,0,0,25,5,653,4,-1,0,3,1 598 | 48,4,1,1,no,268,1,0,0,28,5,458,7,-1,0,3,1 599 | 31,4,1,2,no,0,1,1,0,28,5,664,8,-1,0,3,1 600 | 35,6,1,1,no,5613,1,0,0,28,5,699,3,-1,0,3,1 601 | 48,3,1,1,no,513,0,1,0,28,5,939,3,-1,0,3,1 602 | 53,3,1,0,no,362,0,0,0,28,5,1169,2,-1,0,3,1 603 | 27,9,2,1,no,3933,0,0,0,28,5,837,3,-1,0,3,1 604 | 55,9,1,1,no,3899,1,0,1,28,5,596,4,-1,0,3,1 605 | 40,7,1,1,no,473,1,0,0,28,5,383,3,-1,0,3,1 606 | 40,6,2,2,no,1616,0,0,1,28,5,1009,7,-1,0,3,1 607 | 38,1,0,1,no,1817,1,0,0,28,5,1096,4,-1,0,3,1 608 | 54,5,2,1,no,129,0,1,0,28,5,867,2,-1,0,3,1 609 | 54,9,1,1,no,1938,0,1,0,28,5,551,3,-1,0,3,1 610 | 50,0,2,1,no,133,0,0,0,28,5,890,4,-1,0,3,1 611 | 48,4,0,2,no,2263,0,0,0,29,5,874,2,-1,0,3,1 612 | 41,7,1,3,no,1538,0,0,0,29,5,710,2,-1,0,3,1 613 | 51,1,0,0,no,5,0,0,0,29,5,600,2,-1,0,3,1 614 | 42,9,2,2,no,2625,0,0,0,29,5,2516,2,-1,0,3,1 615 | 58,5,0,1,no,1080,1,0,0,29,5,1058,2,-1,0,3,1 616 | 47,10,1,1,no,312,0,0,0,29,5,884,1,-1,0,3,1 617 | 58,0,1,1,no,145,1,0,1,29,5,616,13,-1,0,3,1 618 | 41,1,1,1,no,-190,0,1,0,29,5,808,3,-1,0,3,1 619 | 50,0,1,1,no,2678,0,0,1,30,5,1011,6,-1,0,3,1 620 | 46,3,1,1,no,271,1,0,0,30,5,1013,29,-1,0,3,1 621 | 44,1,1,1,no,0,0,0,0,30,5,684,2,-1,0,3,1 622 | 36,7,1,1,no,0,0,0,0,30,5,843,2,-1,0,3,1 623 | 58,4,0,1,no,3161,0,0,1,30,5,542,2,-1,0,3,1 624 | 44,10,2,0,yes,-4,0,1,0,30,5,788,2,-1,0,3,1 625 | 49,2,0,3,yes,-701,1,0,0,30,5,988,2,-1,0,3,1 626 | 55,4,1,2,no,-375,0,0,0,30,5,814,2,-1,0,3,1 627 | 52,10,1,0,no,959,0,0,1,30,5,694,2,-1,0,3,1 628 | 60,5,0,1,no,1091,0,0,0,30,5,441,5,-1,0,3,1 629 | 41,7,1,3,yes,-1,0,1,0,30,5,1171,4,-1,0,3,1 630 | 36,4,1,2,no,96,0,0,0,30,5,729,4,-1,0,3,1 631 | 41,1,0,0,no,0,0,0,0,30,5,509,2,-1,0,3,1 632 | 50,2,1,2,no,230,0,0,1,30,5,1089,4,-1,0,3,1 633 | 44,3,1,0,no,-156,0,1,0,30,5,1211,2,-1,0,3,1 634 | 54,9,1,1,no,114,0,0,0,30,5,458,3,-1,0,3,1 635 | 35,4,1,2,no,-468,1,0,0,30,5,540,8,-1,0,3,1 636 | 59,5,0,1,no,679,0,0,0,30,5,761,4,-1,0,3,1 637 | 42,2,1,3,no,206,1,0,0,30,5,695,2,-1,0,3,1 638 | 27,4,2,2,no,1,1,0,0,30,5,764,4,-1,0,3,1 639 | 59,3,0,0,no,2518,0,0,0,30,5,675,3,-1,0,3,1 640 | 43,0,1,1,no,378,0,0,0,30,5,782,8,-1,0,3,1 641 | 39,9,2,1,no,391,0,1,0,30,5,962,4,-1,0,3,1 642 | 33,10,1,1,no,-144,0,0,0,30,5,481,3,-1,0,3,1 643 | 36,6,2,2,no,1240,0,0,0,31,5,812,3,-1,0,3,1 644 | 35,1,0,0,no,1792,1,0,0,31,5,630,6,-1,0,3,1 645 | 50,1,1,3,no,2284,1,0,1,31,5,1088,17,-1,0,3,1 646 | 55,9,1,1,no,264,1,1,0,31,5,1150,5,-1,0,3,1 647 | 51,10,0,2,no,0,0,0,0,31,5,574,2,-1,0,3,1 648 | 48,2,1,2,yes,-392,0,0,0,31,5,725,4,-1,0,3,1 649 | 55,3,0,0,no,77,0,0,0,31,5,463,6,-1,0,3,1 650 | 30,7,1,1,no,48,1,0,0,31,5,662,13,-1,0,3,1 651 | 53,1,1,0,no,252,0,0,0,31,5,727,4,-1,0,3,1 652 | 40,1,1,1,no,1954,0,0,0,31,5,1107,2,-1,0,3,1 653 | 44,9,1,0,no,755,0,1,0,31,5,829,4,-1,0,3,1 654 | 49,4,0,2,no,755,0,0,0,31,5,1212,4,-1,0,3,1 655 | 40,0,1,1,no,341,0,0,0,31,5,1142,5,-1,0,3,1 656 | 58,10,1,0,no,1183,0,0,0,31,5,1721,5,-1,0,3,1 657 | 49,7,1,1,no,388,0,0,1,31,5,1032,4,-1,0,3,1 658 | 38,9,1,1,no,1919,0,0,0,4,1,846,1,-1,0,3,1 659 | 51,3,0,2,no,1613,0,0,0,4,1,870,1,-1,0,3,1 660 | 36,4,0,2,no,0,0,0,0,4,1,1165,1,-1,0,3,1 661 | 55,5,1,1,no,0,0,0,0,4,1,1110,1,-1,0,3,1 662 | 43,9,1,2,no,0,0,0,0,4,1,1488,2,-1,0,3,1 663 | 60,4,0,1,no,0,0,0,0,5,1,603,3,-1,0,3,1 664 | 57,5,1,1,no,466,0,0,0,5,1,641,2,-1,0,3,1 665 | 41,9,1,2,no,515,0,0,0,5,1,837,2,-1,0,3,1 666 | 50,9,1,2,no,2313,0,0,0,5,1,553,1,-1,0,3,1 667 | 53,1,1,1,no,-8,0,0,0,5,1,794,1,-1,0,3,1 668 | 59,5,1,0,no,1634,1,1,0,5,1,836,1,-1,0,3,1 669 | 59,3,1,0,no,100,0,0,0,5,1,1134,1,-1,0,3,1 670 | 52,11,1,3,no,5361,0,0,0,5,1,607,3,-1,0,3,1 671 | 58,4,0,2,no,1257,0,0,0,5,1,1536,1,-1,0,3,1 672 | 59,5,1,0,no,3400,1,0,0,5,1,515,3,-1,0,3,1 673 | 49,9,2,1,no,1377,0,0,0,5,1,935,1,-1,0,3,1 674 | 33,4,0,2,no,2885,0,0,0,5,1,1051,2,-1,0,3,1 675 | 55,1,1,0,no,512,0,0,0,5,1,1200,2,-1,0,3,1 676 | 42,9,1,2,no,921,0,0,0,5,1,531,3,-1,0,3,1 677 | 52,7,1,1,no,1000,0,0,0,5,1,766,2,-1,0,3,1 678 | 38,9,1,2,no,202,0,0,0,5,1,1311,2,-1,0,3,1 679 | 55,9,1,2,no,169,0,1,0,5,1,618,2,-1,0,3,1 680 | 49,9,1,2,no,3726,0,0,0,5,1,875,2,-1,0,3,1 681 | 56,9,0,1,no,146,1,0,0,5,1,419,3,-1,0,3,1 682 | 46,9,1,1,no,940,0,0,0,5,1,1227,2,-1,0,3,1 683 | 57,5,0,2,no,734,0,0,0,6,1,1357,4,-1,0,3,1 684 | 61,5,1,1,no,77,0,0,0,6,1,455,4,-1,0,3,1 685 | 45,0,1,1,no,925,0,0,0,6,1,406,2,-1,0,3,1 686 | 39,4,1,2,no,1836,0,0,0,6,1,902,3,-1,0,3,1 687 | 46,0,1,1,no,660,0,0,0,6,1,740,2,-1,0,3,1 688 | 43,4,2,2,no,1304,0,0,0,6,1,501,2,-1,0,3,1 689 | 34,7,1,1,no,2956,0,0,0,6,1,835,2,-1,0,3,1 690 | 35,4,0,2,no,146,0,0,0,6,1,720,2,-1,0,3,1 691 | 36,4,0,2,yes,36,0,0,0,6,1,836,4,-1,0,3,1 692 | 45,4,0,2,no,1,0,0,0,6,1,490,2,-1,0,3,1 693 | 47,9,1,1,no,2480,0,0,0,6,1,763,2,-1,0,3,1 694 | 42,9,1,2,no,14282,1,0,0,6,1,649,4,-1,0,3,1 695 | 40,4,2,2,no,0,0,0,0,6,1,641,6,-1,0,3,1 696 | 40,9,0,1,no,311,0,0,0,6,1,738,2,-1,0,3,1 697 | 32,9,1,1,no,560,0,0,0,6,1,1044,4,-1,0,3,1 698 | 33,9,2,1,no,3059,0,1,0,6,1,482,2,-1,0,3,1 699 | 32,6,1,2,no,1423,0,0,0,6,1,1249,4,-1,0,3,1 700 | 46,1,1,0,no,874,0,0,0,6,1,996,1,-1,0,3,1 701 | 50,0,1,1,no,706,1,0,0,6,1,1250,4,-1,0,3,1 702 | 51,1,1,0,no,7098,0,0,0,6,1,1471,7,-1,0,3,1 703 | 30,4,2,2,no,5389,0,0,0,6,1,1456,3,-1,0,3,1 704 | 39,9,2,1,no,1104,0,0,0,7,1,395,2,-1,0,3,1 705 | 47,0,1,1,no,3070,0,0,0,7,1,663,4,-1,0,3,1 706 | 40,9,0,1,no,0,0,0,0,7,1,674,2,-1,0,3,1 707 | 50,1,1,0,no,4108,1,0,0,7,1,526,2,-1,0,3,1 708 | 60,4,1,2,no,0,0,0,0,7,1,401,3,-1,0,3,1 709 | 32,4,2,2,no,4291,0,0,0,7,1,1321,9,-1,0,3,1 710 | 57,9,1,2,no,6822,0,0,0,7,1,797,2,-1,0,3,1 711 | 41,6,1,2,no,231,0,0,0,7,1,352,2,-1,0,3,1 712 | 50,9,1,1,no,3176,0,0,0,7,1,670,1,-1,0,3,1 713 | 46,9,0,1,no,-824,1,0,0,7,1,429,3,-1,0,3,1 714 | 31,4,1,2,no,676,0,0,0,7,1,1182,1,-1,0,3,1 715 | 36,9,2,1,no,651,0,0,0,7,1,876,1,-1,0,3,1 716 | 45,9,0,1,no,2657,0,0,0,7,1,895,3,-1,0,3,1 717 | 33,9,0,1,no,801,0,0,0,7,1,341,5,-1,0,3,1 718 | 47,7,1,1,no,405,0,1,0,8,1,994,2,-1,0,3,1 719 | 35,4,1,2,no,82,0,0,0,8,1,507,4,-1,0,3,1 720 | 38,4,1,2,no,179,0,0,2,8,1,317,4,-1,0,3,1 721 | 33,4,1,2,no,1047,0,0,0,8,1,512,4,-1,0,3,1 722 | 44,9,2,1,no,7138,0,0,0,8,1,809,2,-1,0,3,1 723 | 31,4,2,2,no,165,0,1,0,8,1,523,2,-1,0,3,1 724 | 31,4,2,2,no,37,0,0,0,8,1,669,2,-1,0,3,1 725 | 49,7,1,1,no,823,0,0,0,8,1,780,4,-1,0,3,1 726 | 52,7,1,0,no,1625,0,0,0,8,1,459,2,-1,0,3,1 727 | 31,9,1,2,no,120,0,0,0,8,1,1000,1,-1,0,3,1 728 | 35,4,1,2,no,2957,1,1,0,8,1,733,3,-1,0,3,1 729 | 31,4,2,2,no,2,0,0,0,8,1,848,1,-1,0,3,1 730 | 36,9,0,1,no,1,0,0,0,8,1,1242,3,-1,0,3,1 731 | 39,4,1,2,no,1795,0,0,0,8,1,922,7,-1,0,3,1 732 | 46,4,1,2,no,4,0,0,0,8,1,1206,4,-1,0,3,1 733 | 48,1,1,0,no,476,0,0,0,11,1,856,3,-1,0,3,1 734 | 36,4,1,2,no,9,0,0,0,11,1,386,4,-1,0,3,1 735 | 30,4,1,2,no,131,1,0,0,11,1,384,6,-1,0,3,1 736 | 33,4,2,2,no,1120,0,0,0,11,1,1070,2,-1,0,3,1 737 | 57,3,1,0,no,2785,0,0,0,11,1,610,4,-1,0,3,1 738 | 33,4,2,2,no,0,0,0,0,11,1,699,7,-1,0,3,1 739 | 35,4,1,2,no,43,0,0,0,11,1,917,4,-1,0,3,1 740 | 53,4,1,2,no,0,0,0,0,11,1,608,2,-1,0,3,1 741 | 41,4,1,1,no,456,1,0,0,11,1,734,4,-1,0,3,1 742 | 32,4,2,1,no,2018,0,0,0,11,1,1238,1,-1,0,3,1 743 | 41,4,1,2,no,0,0,0,0,11,1,1242,5,-1,0,3,1 744 | 35,9,2,1,no,354,0,0,0,11,1,444,3,-1,0,3,1 745 | 48,1,1,0,no,719,0,1,0,11,1,418,5,-1,0,3,1 746 | 34,4,1,2,no,828,0,1,0,11,1,1080,3,-1,0,3,1 747 | 47,1,1,1,no,2787,0,0,0,11,1,479,1,-1,0,3,1 748 | 46,10,1,1,no,2551,0,0,0,12,1,645,2,-1,0,3,1 749 | 60,5,1,1,no,2722,0,0,0,12,1,506,2,-1,0,3,1 750 | 39,4,1,1,no,1880,0,0,0,12,1,768,2,-1,0,3,1 751 | 57,1,1,0,no,2112,0,0,0,12,1,1134,2,-1,0,3,1 752 | 36,10,0,1,no,871,0,0,0,12,1,396,3,-1,0,3,1 753 | 30,4,1,1,no,850,0,0,0,12,1,507,2,-1,0,3,1 754 | 33,9,1,1,no,3243,0,0,0,12,1,439,4,-1,0,3,1 755 | 30,4,1,2,no,1567,1,0,0,12,1,1133,4,-1,0,3,1 756 | 36,9,2,1,yes,12,0,0,0,12,1,587,2,-1,0,3,1 757 | 49,1,1,0,no,128,0,0,0,12,1,696,2,-1,0,3,1 758 | 49,1,1,1,no,553,0,0,0,12,1,645,8,-1,0,3,1 759 | 40,10,0,1,no,1694,0,0,0,12,1,473,6,-1,0,3,1 760 | 49,1,1,0,no,408,0,0,0,12,1,559,4,-1,0,3,1 761 | 38,4,1,2,no,1223,0,0,0,12,1,1092,5,-1,0,3,1 762 | 31,4,2,2,no,1858,0,0,0,12,1,453,3,-1,0,3,1 763 | 34,4,1,2,no,520,1,0,0,12,1,1307,4,-1,0,3,1 764 | 39,4,1,2,no,271,1,0,0,12,1,1344,1,-1,0,3,1 765 | 40,4,1,1,no,4396,0,0,0,12,1,432,3,-1,0,3,1 766 | 47,9,1,1,no,568,1,0,0,12,1,1613,1,-1,0,3,1 767 | 37,9,2,1,no,3665,0,0,0,12,1,664,3,-1,0,3,1 768 | 44,9,2,1,no,244,1,0,0,12,1,1735,4,-1,0,3,1 769 | 55,4,1,2,no,568,0,0,0,12,1,940,10,-1,0,3,1 770 | 40,9,1,1,no,42,0,0,0,13,1,1842,2,-1,0,3,1 771 | 54,5,1,0,no,309,0,0,0,13,1,969,2,-1,0,3,1 772 | 35,9,1,1,no,-180,0,1,0,13,1,823,2,-1,0,3,1 773 | 32,9,2,2,no,34646,0,0,1,13,1,618,9,-1,0,3,1 774 | 36,4,0,2,no,59,0,0,0,13,1,526,2,-1,0,3,1 775 | 37,4,1,2,no,4436,0,0,0,13,1,846,5,-1,0,3,1 776 | 37,9,2,1,no,-242,1,0,0,13,1,1149,6,-1,0,3,1 777 | 50,6,1,1,no,10052,0,0,0,13,1,665,2,-1,0,3,1 778 | 58,5,1,2,no,0,0,0,0,13,1,641,4,-1,0,3,1 779 | 52,9,0,1,no,152,0,0,0,13,1,404,6,-1,0,3,1 780 | 31,4,1,2,no,156,0,0,0,13,1,657,7,-1,0,3,1 781 | 40,4,1,2,no,66,0,0,0,13,1,737,4,-1,0,3,1 782 | 34,4,2,2,no,9827,1,0,0,13,1,871,4,-1,0,3,1 783 | 53,7,1,1,no,6170,0,0,0,13,1,838,4,-1,0,3,1 784 | 43,4,1,2,no,2383,0,0,0,13,1,379,6,-1,0,3,1 785 | 30,9,2,2,no,635,0,0,0,13,1,851,8,-1,0,3,1 786 | 34,9,2,1,no,16,0,0,0,13,1,990,1,-1,0,3,1 787 | 48,0,0,1,no,362,0,0,0,13,1,529,1,-1,0,3,1 788 | 33,4,1,2,no,191,0,0,0,13,1,1148,2,-1,0,3,1 789 | 37,4,2,2,no,455,1,0,0,13,1,904,6,-1,0,3,1 790 | 57,9,0,1,no,3043,0,0,0,14,1,707,2,-1,0,3,1 791 | 55,5,1,0,no,3334,0,0,0,14,1,632,4,-1,0,3,1 792 | 34,9,1,2,no,8029,1,0,0,14,1,593,4,-1,0,3,1 793 | 44,2,1,2,no,624,0,0,1,14,1,552,2,-1,0,3,1 794 | 35,4,2,2,no,331,0,0,0,14,1,857,2,-1,0,3,1 795 | 34,3,1,0,no,2929,0,0,0,14,1,518,2,-1,0,3,1 796 | 49,7,1,1,no,1314,0,0,0,14,1,834,14,-1,0,3,1 797 | 30,9,1,1,no,289,1,0,0,14,1,1184,4,-1,0,3,1 798 | 32,4,2,2,no,20,0,0,0,14,1,1151,2,-1,0,3,1 799 | 56,1,1,0,no,8163,0,0,0,14,1,1231,4,-1,0,3,1 800 | 50,4,2,2,no,1200,0,0,0,14,1,904,3,-1,0,3,1 801 | 32,9,2,1,no,1732,1,0,0,14,1,1871,3,-1,0,3,1 --------------------------------------------------------------------------------