├── .gitignore ├── images ├── aidc.png ├── guid.png ├── roi.png ├── roi1.png ├── roi2.png ├── vsc.png ├── autoai.png ├── dataset.png ├── finding.png ├── panel.png ├── run_all.png ├── tier1.png ├── track.png ├── trend.png ├── add_model.png ├── evaluate.png ├── feedback.png ├── first_run.png ├── new_asset.png ├── roi_panel.png ├── set_tier.png ├── tracking.png ├── comparison.png ├── confidence.png ├── deployment.png ├── end_review.png ├── improvement.png ├── insert_cell.png ├── model_info.png ├── new_project.png ├── object_types.png ├── open_issue.png ├── roi_filter.png ├── roi_workflow.png ├── second_run.png ├── space_fail.png ├── start_review.png ├── subscription.png ├── add_deployment.png ├── edit_notebook.png ├── model_use_case.png ├── autoai_completed.png ├── final_dashboard.png ├── open_issue_stage.png ├── openscale_monitor.png └── start_roi_review.png ├── cloud_assets ├── endpoint.png ├── evaluate.png ├── metric_group.png ├── configure_monitors.png ├── service.yml ├── Dockerfile ├── deploy.yml ├── aidc.json └── app.py ├── cp4d_assets ├── openpages-env-mig-082823104325.jar └── Dockerfile ├── docs ├── Integrations.md ├── BYOM.md ├── Local.md ├── OpenScale.md ├── Cloud.md ├── OpenPages.md ├── WML.md └── AutoAI.md ├── README.md ├── LICENSE └── notebooks └── OpenScale_flow.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | *.zip 3 | *.DS_Store 4 | notebooks/AIDC.json 5 | -------------------------------------------------------------------------------- /images/aidc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/aidc.png -------------------------------------------------------------------------------- /images/guid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/guid.png -------------------------------------------------------------------------------- /images/roi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/roi.png -------------------------------------------------------------------------------- /images/roi1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/roi1.png -------------------------------------------------------------------------------- /images/roi2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/roi2.png -------------------------------------------------------------------------------- /images/vsc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/vsc.png -------------------------------------------------------------------------------- /images/autoai.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/autoai.png -------------------------------------------------------------------------------- /images/dataset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/dataset.png -------------------------------------------------------------------------------- /images/finding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/finding.png -------------------------------------------------------------------------------- /images/panel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/panel.png -------------------------------------------------------------------------------- /images/run_all.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/run_all.png -------------------------------------------------------------------------------- /images/tier1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/tier1.png -------------------------------------------------------------------------------- /images/track.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/track.png -------------------------------------------------------------------------------- /images/trend.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/trend.png -------------------------------------------------------------------------------- /images/add_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/add_model.png -------------------------------------------------------------------------------- /images/evaluate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/evaluate.png -------------------------------------------------------------------------------- /images/feedback.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/feedback.png -------------------------------------------------------------------------------- /images/first_run.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/first_run.png -------------------------------------------------------------------------------- /images/new_asset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/new_asset.png -------------------------------------------------------------------------------- /images/roi_panel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/roi_panel.png -------------------------------------------------------------------------------- /images/set_tier.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/set_tier.png -------------------------------------------------------------------------------- /images/tracking.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/tracking.png -------------------------------------------------------------------------------- /images/comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/comparison.png -------------------------------------------------------------------------------- /images/confidence.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/confidence.png -------------------------------------------------------------------------------- /images/deployment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/deployment.png -------------------------------------------------------------------------------- /images/end_review.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/end_review.png -------------------------------------------------------------------------------- /images/improvement.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/improvement.png -------------------------------------------------------------------------------- /images/insert_cell.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/insert_cell.png -------------------------------------------------------------------------------- /images/model_info.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/model_info.png -------------------------------------------------------------------------------- /images/new_project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/new_project.png -------------------------------------------------------------------------------- /images/object_types.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/object_types.png -------------------------------------------------------------------------------- /images/open_issue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/open_issue.png -------------------------------------------------------------------------------- /images/roi_filter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/roi_filter.png -------------------------------------------------------------------------------- /images/roi_workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/roi_workflow.png -------------------------------------------------------------------------------- /images/second_run.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/second_run.png -------------------------------------------------------------------------------- /images/space_fail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/space_fail.png -------------------------------------------------------------------------------- /images/start_review.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/start_review.png -------------------------------------------------------------------------------- /images/subscription.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/subscription.png -------------------------------------------------------------------------------- /cloud_assets/endpoint.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/cloud_assets/endpoint.png -------------------------------------------------------------------------------- /cloud_assets/evaluate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/cloud_assets/evaluate.png -------------------------------------------------------------------------------- /images/add_deployment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/add_deployment.png -------------------------------------------------------------------------------- /images/edit_notebook.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/edit_notebook.png -------------------------------------------------------------------------------- /images/model_use_case.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/model_use_case.png -------------------------------------------------------------------------------- /images/autoai_completed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/autoai_completed.png -------------------------------------------------------------------------------- /images/final_dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/final_dashboard.png -------------------------------------------------------------------------------- /images/open_issue_stage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/open_issue_stage.png -------------------------------------------------------------------------------- /images/openscale_monitor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/openscale_monitor.png -------------------------------------------------------------------------------- /images/start_roi_review.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/images/start_roi_review.png -------------------------------------------------------------------------------- /cloud_assets/metric_group.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/cloud_assets/metric_group.png -------------------------------------------------------------------------------- /cloud_assets/configure_monitors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/cloud_assets/configure_monitors.png -------------------------------------------------------------------------------- /cp4d_assets/openpages-env-mig-082823104325.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/aidc-samples/main/cp4d_assets/openpages-env-mig-082823104325.jar -------------------------------------------------------------------------------- /cloud_assets/service.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: test-monitor 5 | spec: 6 | selector: 7 | app.kubernetes.io/name: test-monitor 8 | ports: 9 | - protocol: TCP 10 | port: 444 11 | targetPort: 8080 12 | --- -------------------------------------------------------------------------------- /cloud_assets/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | RUN apt update && apt install nodejs -y 4 | 5 | RUN python -m pip install gunicorn flask numpy pandas requests 6 | 7 | EXPOSE 8080 8 | 9 | WORKDIR /workspace 10 | 11 | COPY app.py /workspace 12 | 13 | COPY aidc-2.0.zip ./ 14 | 15 | RUN python -m pip install aidc-2.0.zip 16 | 17 | CMD [ "python3", "-m" , "flask", "run", "--host=0.0.0.0", "--port=8080"] -------------------------------------------------------------------------------- /cp4d_assets/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG base_image_tag 2 | FROM ${base_image_tag} 3 | 4 | USER root:root 5 | RUN curl --silent --location https://rpm.nodesource.com/setup_18.x | bash - 6 | RUN yum install -y nodejs 7 | USER wmlfuser:condausers 8 | RUN umask 002 && \ 9 | pip install javascript 10 | 11 | USER wmlfuser:condausers 12 | RUN umask 002 && \ 13 | mkdir -p /home/wmlfuser/mypkgs && \ 14 | pip install pydot --target /home/wmlfuser/mypkgs 15 | ENV PYTHONPATH=$PYTHONPATH:/home/wmlfuser/mypkgs -------------------------------------------------------------------------------- /cloud_assets/deploy.yml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: test-monitor 5 | spec: 6 | selector: 7 | matchLabels: 8 | app.kubernetes.io/name: test-monitor 9 | replicas: 1 10 | template: 11 | metadata: 12 | labels: 13 | app.kubernetes.io/name: test-monitor 14 | spec: 15 | containers: 16 | - name: deployment-studio-container 17 | image: "icr.io/test/monitor-test" 18 | ports: 19 | - containerPort: 8080 20 | imagePullPolicy: IfNotPresent 21 | restartPolicy: Always -------------------------------------------------------------------------------- /docs/Integrations.md: -------------------------------------------------------------------------------- 1 | 2 | ## Integrations 3 | The below links and the flow chart demonstrate how AIDC can be used to determine 4 | the benefits of using AI by involving IBM's AI Governance framework. 5 | 6 | The described actions will be in the context of Cloud Pak for Data interface. 7 | 8 | [IBM AutoAI flow](AutoAI.md#autoai)
9 | Build an optimized model based on a dataset 10 | 11 | [Bring Your Own Model flow](BYOM.md#byom)
12 | Evaluate an existing Machine Learning model 13 | 14 | [Watson Machine Learning flow](WML.md#wml)
15 | Manage decisions made by a model 16 | 17 | [Watson OpenScale flow](OpenScale.md#openscale)
18 | Define monitoring for your model 19 | 20 | [IBM OpenPages flow](OpenPages.md#openpages)
21 | Govern your model and build custom workflows 22 | 23 | ![AIDC flows](../images/aidc.png) -------------------------------------------------------------------------------- /docs/BYOM.md: -------------------------------------------------------------------------------- 1 | 2 | ## Bring Your Own Model flow 3 | 4 | In this scenario, we will use the pre-existing model to produce 5 | required dataset based on our [dataset without human output](../data/credit_no_human.csv). 6 | 7 | Using the model, we obtain additional columns, namely
8 | `mlClass` (the decision made by the model) and
9 | `mlConfidence` (the probability that the transaction is a Risk) and combine it with the human results. 10 | 11 | Please see: [credit with human and ml](../data/credit_human_ml.csv) for an example. 12 | 13 | We will run the [BYOM_flow](../notebooks/BYOM_flow.ipynb) notebook. 14 | 15 | Similarly to the [AutoAI flow](#performance), we will start with default performance model
16 | then define different values in the performance model, calculate the ROI and finally save the rules as JSON. 17 | 18 | Follow the [Watson Machine Learning](WML.md#openscale) section to see how we can control the decisions made by the model. 19 | -------------------------------------------------------------------------------- /docs/Local.md: -------------------------------------------------------------------------------- 1 | 2 | ## Running locally 3 | In order to use the solution locally, let's start with setting up the environment. 4 | 5 | ``` 6 | 1) Make sure you have node running locally 7 | 8 | node -v 9 | 10 | RHEL: 11 | yum install -y nodejs 12 | 13 | MACOS: 14 | brew install node 15 | 16 | Others: 17 | https://nodejs.org/en/download/package-manager 18 | 19 | 2) Prepare an virtual environment 20 | 21 | python3 -m pip install virtualenv 22 | python3 -m virtualenv aidc 23 | 24 | source aidc/bin/activate 25 | 26 | python3 -m pip install pandas 27 | python3 -m pip install matplotlib 28 | python3 -m pip install aidc-2.0.zip 29 | 30 | 3) In case of large datasets, please import the customized javascript module (with larger timeouts) and extend Node default settings: 31 | 32 | export NODE_OPTIONS=--max-old-space-size=4092 33 | or withing a notebook: 34 | %env NODE_OPTIONS=--max-old-space-size=4092 35 | 36 | !pip install /project_data/data_asset/javascript-2023.2.0.zip 37 | !pip install /project_data/data_asset/aidc-2.0.zip; 38 | ``` 39 | 40 | You can now use the Python environment for example in Visual Studio Code: 41 | 42 | ![vsc](../images/vsc.png) 43 | 44 | Continue with the [Local.ipynb](../notebooks/Local.ipynb) notebook. 45 | 46 | We will be using the complete dataset with human & ml responses: [credit_human_ml.csv](../data/credit_human_ml.csv) -------------------------------------------------------------------------------- /cloud_assets/aidc.json: -------------------------------------------------------------------------------- 1 | {"entity":{"applies_to":{"input_data_type":["structured"]},"description":"","metrics":[{"applies_to":{"problem_type":["binary"]},"expected_direction":"unknown","id":"roi","name":"roi","thresholds":[{"type":"lower_limit"},{"type":"upper_limit"}]},{"applies_to":{"problem_type":["binary"]},"expected_direction":"unknown","id":"human","name":"human","thresholds":[{"type":"lower_limit"},{"type":"upper_limit"}]},{"applies_to":{"problem_type":["binary"]},"expected_direction":"unknown","id":"ml","name":"ml","thresholds":[{"type":"lower_limit"},{"type":"upper_limit"}]},{"applies_to":{"problem_type":["binary"]},"expected_direction":"unknown","id":"impact","name":"impact","thresholds":[{"type":"lower_limit"},{"type":"upper_limit"}]},{"applies_to":{"problem_type":["binary"]},"expected_direction":"unknown","id":"performance","name":"performance","thresholds":[{"type":"lower_limit"},{"type":"upper_limit"}]},{"applies_to":{"problem_type":["regression"]},"expected_direction":"unknown","id":"decisioncost","name":"decisioncost","thresholds":[{"type":"lower_limit"},{"type":"upper_limit"}]}],"monitor_runtime":{"type":"custom_metrics_provider"},"name":"AIDC","parameters_schema":{},"tags":[]},"metadata":{"created_at":"2023-10-23T09:40:13.014Z","created_by":"IBMid-270003NYJ4","crn":"crn:v1:bluemix:public:aiopenscale:us-south:a/306c1b2cad8745ab92330712d31bf595:b4995910-4dbb-4c4e-89e4-de33674e0f34:monitor_definition:aidc","id":"aidc","url":"/v2/monitor_definitions/aidc"}} -------------------------------------------------------------------------------- /docs/OpenScale.md: -------------------------------------------------------------------------------- 1 | 2 | ## Watson OpenScale flow 3 | 4 | After the [model](../README.md#model) and the [dispatch function](../docs/WML.md#wml) have been deployed, 5 | we can use the Watson OpenScale capabilities to monitor & alert if the AIDC rules change. 6 | 7 | We are able to calculate the human-to-ml distribution, the average cost of decision, 8 | average performance and impact costs, as well as the ROI metric. 9 | 10 | ROI is measured as the sum of improvements over each decision between the current process (done by human) and the augmented 11 | human+AI process. 12 | 13 | ![roi](../images/roi.png) 14 | 15 | Start with adding the deployed model to Watson OpenScale dashboard: 16 | 17 | ![add_model](../images/add_model.png) 18 | ![add_deployment](../images/add_deployment.png) 19 | 20 | next collect the Subscription ID from the model information: 21 | 22 | ![model_info](../images/model_info.png) 23 | ![subscription](../images/subscription.png) 24 | 25 | use this information in the [OpenScale notebook](../notebooks/OpenScale_flow.ipynb). 26 | 27 | Running th notebook for the first time, will result in 0s as the metrics' values. 28 | ![first_run](../images/first_run.png) 29 | 30 | To correct this, we need to upload the Feedback data to the model using the Evaluate function of Watson OpenScale. 31 | You can upload the [dataset with human and machine learning outputs](../data/credit_human_ml.csv). 32 | 33 | ![evaluate](../images/evaluate.png) 34 | 35 | ![feedback](../images/feedback.png) 36 | 37 | Running the 2 last steps of the notebook, should return: 38 | 39 | ![second_run](../images/second_run.png) 40 | 41 | Navigating back to OpenScale interface, we can observe the corresponding information: 42 | 43 | ![openscale_monitor](../images/openscale_monitor.png) 44 | 45 | We can now use the defined metrics in [OpenPages](OpenPages.md#openpages) 46 | -------------------------------------------------------------------------------- /docs/Cloud.md: -------------------------------------------------------------------------------- 1 | 1) Deploy a model on IBM Cloud 2 | 3 | You can use our [AutoAI steps](AutoAI.md) to build and deploy a model on [Watson Studio on IBM Cloud](https://cloud.ibm.com/catalog/services/watson-studio) 4 | 5 | 2) Deploy [Watson OpenScale on IBM Cloud](https://cloud.ibm.com/catalog/services/watson-openscale) 6 | 7 | 3) Update the WOS_CREDENTIALS section in [app.py](../cloud_assets/app.py) 8 | 9 | 4) Build the custom image using the provided [Dockerfile](../cloud_assets/Dockerfile) 10 | ``` 11 | podman build . -t test-monitor 12 | ``` 13 | 5) Push the image to a container registry, for example [IBM Cloud Container Registry](https://www.ibm.com/products/container-registry): 14 | ``` 15 | podman tag test-monitor icr.io/test/test-monitor 16 | podman push icr.io/test/test-monitor 17 | ``` 18 | 19 | 6) Run the container on a public container platform, for example: 20 | 21 | [IBM Cloud OpenShift](https://cloud.ibm.com/kubernetes/landing?platformType=openshift) 22 | or 23 | [Cloud Engine](https://cloud.ibm.com/codeengine/) 24 | 25 | 26 | To deploy on OpenShift, you can use the [Deployment](../cloud_assets/deploy.yml) and [Service](../cloud_assets/service.yml) examples. 27 | 28 | Please make sure that you expose the service so its publicly accessible. 29 | 30 | 7) Add your model to OpenScale Dashboard 31 | 32 | 8) Import custom Metric group inside of OpenScale using the [aidc.json](../cloud_assets/aidc.json) file 33 | 34 | ![metric_group](../cloud_assets/metric_group.png) 35 | 36 | 8) Add a Metric endpoint pointing at the URL defined in step 6. 37 | 38 | ![endpoint](../cloud_assets/endpoint.png) 39 | 40 | 9) Configure the monitors of your model,and add the AIDC group. 41 | 42 | ![configure_monitors](../cloud_assets/configure_monitors.png) 43 | 44 | 10) Upload Feedback data and Evaluate the model. 45 | 46 | ![evaluate](../cloud_assets/evaluate.png) -------------------------------------------------------------------------------- /docs/OpenPages.md: -------------------------------------------------------------------------------- 1 | 2 | ## IBM OpenPages flow 3 | 4 | To configure the tracking of the deployed model using the custom monitor, we need to start by 5 | configuring [OpenPages for Model Risk Governance](https://www.ibm.com/docs/en/cloud-paks/cp-data/4.7.x?topic=openpages-integrating) 6 | and [connecting Watson OpenScale](https://www.ibm.com/docs/en/cloud-paks/cp-data/4.7.x?topic=governance-end-end-model-tutorial#mrm-risk-config-dsx-work-step3). 7 | 8 | Next, let's introduce a new Model Use Case: 9 | ![model_use_case](../images/model_use_case.png) 10 | 11 | and start tracking the deployed model using the new Model Use Case: 12 | ![track](../images/track.png) 13 | 14 | Using the custom filters functionality of OpenPages, we can define a filter for the `roi` metric: 15 | 16 | ![object_types](../images/object_types.png) 17 | 18 | ![roi_filter](../images/roi_filter.png) 19 | 20 | Going back to the OpenPages dashboard, we can define a new panel 21 | ![panel](../images/panel.png) 22 | 23 | and add it to the dashboard: 24 | 25 | ![roi_panel](../images/roi_panel.png) 26 | 27 | We can now monitor the trends of the Metrics and corresponding Metric Values: 28 | 29 | ![trend](../images/trend.png) 30 | 31 | ![tracking](../images/tracking.png) 32 | 33 | Next let's create a custom Workflow to manage our models' ROI state: 34 | 35 | [Simple guide on creating Workflows](https://www.youtube.com/watch?v=ePnjAbRD0is) 36 | 37 | Our flow will have only 2 stages: Update & Review: 38 | 39 | ![roi_workflow](../images/roi_workflow.png) 40 | 41 | To simplify the process, the actions can only be taken when Breach Status is `Red` 42 | 43 | ![start_review](../images/start_review.png) 44 | 45 | Once we are in the ROI update stage, the next Action will open an Issue, 46 | and set the Tier of the corresponding model to Tier 1. 47 | 48 | ![open_issue](../images/open_issue.png) 49 | 50 | Tier needs to be configured using the Model Deployment ancestor: 51 | 52 | ![set_tier](../images/set_tier.png) 53 | 54 | After Publishing the Workflow, we can now run it on a Metric Value: 55 | 56 | ![start_roi_review](../images/start_roi_review.png) 57 | 58 | and observe the object transition between the stages: 59 | 60 | ![open_issue_stage](../images/open_issue_stage.png) 61 | 62 | ![end_review](../images/end_review.png) 63 | 64 | After completion, the Model will change the Final Tier assignment: 65 | 66 | ![tier1](../images/tier1.png) 67 | 68 | and Issues will be opened as expected: 69 | 70 | ![final_dashboard](../images/final_dashboard.png) 71 | 72 | Sample Workflow object can be found here: 73 | [workflow](../cp4d_assets/openpages-env-mig-082823104325.jar) -------------------------------------------------------------------------------- /docs/WML.md: -------------------------------------------------------------------------------- 1 | 2 | ## Watson Machine Learning flow 3 | 4 | In this process, we start by defining a [custom image](https://www.ibm.com/docs/en/cloud-paks/cp-data/4.7.x?topic=environments-building-custom-images) 5 | using the provided [Dockerfile](../cp4d_assets/Dockerfile). 6 | 7 | We also need the JSON representation of AIDC model from [AutoAI](../docs/AutoAI.md#autoai) or processing your [own model](../docs/BYOM.md#byom). 8 | 9 | Following the [WML_flow.ipynb notebook](../notebooks/WML_flow.ipynb) we will load the model, 10 | make sure it applies the decision table we have selected, and then deploy it as a function. 11 | 12 | This will allow all the future requests to the model be handled by our `dispatch` function. 13 | 14 | To deploy the function we follow the mechanism of deployable Python functions: 15 | 16 | https://www.ibm.com/docs/en/cloud-paks/cp-data/4.7.x?topic=functions-writing-deployable-python 17 | 18 | with private libraries: 19 | 20 | https://www.ibm.com/docs/en/cloud-paks/cp-data/4.7.x?topic=runtimes-customizing-third-party-private-python-libraries 21 | 22 | As inputs, it takes the request data (attributes and the values) as well as the URL of the model. 23 | 24 | It produces the output suggesting whether the machine learning model decision should be followed, or the human operator should be involved. 25 | For example: 26 | 27 | ``` 28 | test_values_for_human = [["CheckingStatus", "LoanDuration", "CreditHistory", "LoanPurpose", "LoanAmount", "ExistingSavings", "EmploymentDuration", "InstallmentPercent", "Sex", "OthersOnLoan", "CurrentResidenceDuration", "OwnsProperty", "Age", "InstallmentPlans", "Housing", "ExistingCreditsCount", "Job", "Dependents", "Telephone", "ForeignWorker"], 29 | ["no_checking", 20, "prior_payments_delayed", "repairs", 3094, "500_to_1000", "greater_7", 39, "male", "none", 3, "savings_insurance", 29, "none", "own", 1, "skilled", 1, "yes", "yes"], 30 | model_serving_url] 31 | 32 | job_payload = { 33 | client.deployments.ScoringMetaNames.INPUT_DATA: [{ 34 | "fields": [ "message" ], 35 | "values": [[test_values_for_human]] 36 | }] 37 | } 38 | 39 | function_result = client.deployments.score(deployment_uid, job_payload) 40 | print( function_result ) 41 | ``` 42 | would produce: 43 | ``` 44 | {'predictions': [{'fields': ['AIDC Decision'], 'values': [[{'toolToUse': 'human', 'type': 'h'}]]}]} 45 | ``` 46 | 47 | If we modify the data accordingly, the result may be: 48 | ``` 49 | {'predictions': [{'fields': ['AIDC Decision'], 'values': [[{'toolToUse': 'ml', 'type': 'ml', 'confidence': 0.8580362993842072, 'outcome': 'No Risk'}]]}]} 50 | ``` 51 | 52 | Having deployed the function, see how you can use [Watson OpenScale](OpenScale.md) to monitor the model. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # IBM AI Decision Coordination samples 2 | AI Decision Coordination sample assets and notebooks. 3 | 4 | 5 | [Introduction](#intro)
6 | [Key differentiators](#key)
7 | [Examples](#examples)
8 | [Collecting a dataset](#dataset)
9 | [Training a model](#model)
10 | [Using the solution](#using)
11 | [Contact](#contact)
12 | 13 | 14 | ## Introduction 15 | 16 | AI Decision Coordination software analyses your data and calculates the success of tasks 17 | that are completed by automated AI, human resources, or augmentation that combines the two. 18 | 19 | Our solution helps in obtaining the most optimal human-to-AI workload distribution and 20 | calculating the return on investment given specific business guidelines. 21 | 22 | 23 | ## Key differentiators 24 | 25 | ### Assessing the Effectiveness of Decision-Making Operators 26 | 27 | #### AI Decision Coordinator empowers business professionals to gauge the efficency of decision operators by allowing them to define and compute business metrics for various decision-making methods, whether they involve AI models, human input, or a hybrid approach, all in the context of specific datasets. 28 | 29 | ### AI Decision Coordinator is designed with business users in mind. 30 | 31 | #### Rather than merely stating a operator's technical accuracy, it provides insights into the financial implications of correct and incorrect decisions, offering a consolidated view of business performance. 32 | #### The comparison takes into account not only decision performance but also the associated costs. Users can assess how much they can save by automating their decision processes using machine learning compared to their current manual procedures. 33 | 34 | ### Rule-Based Selection 35 | 36 | #### AI Decision Coordinator allows the formulation of rules for selecting the most suitable decision operator for any future scenario, with the ultimate goal of maximizing overall business performance. This might entail assigning complex decisions to humans and reserving straightforward ones for machine learning models to optimize aggregate performance. 37 | 38 | ### Comparative Analysis 39 | 40 | #### It enables users to compare the overall business performance of different decision operators, whether human-based, automated, or a combination of both, in order to choose the best allocation that complements the targeted process and satisfies the present business needs. 41 | 42 | 43 | ### Examples 44 | 45 | Based on the provided dataset, assess the areas where AI and humans demonstrate their optimal performance, considering the default gain/costs model. 46 | 47 | ![roi1](images/roi1.png) 48 | 49 | Reevaluate the same dataset, this time accounting for the scenario-specific gain/cost model, to determine the areas where AI and humans perform best. 50 | 51 | ![roi2](images/roi2.png) 52 | 53 | By utilizing the recommended distribution, we can compute the projected enhancements that AI would deliver: 54 | 55 | ![improvement](images/improvement.png) 56 | 57 | 58 | ## Collecting a dataset. 59 | 60 | To implement the solution, we begin by gathering the data. 61 | The dataset should encompass attributes relevant to the decision-making process, including the target attribute (groundTruth) generated by experts, as well as the decisions made by human resources (hClass) in the current workflow. 62 | 63 | ![dataset](images/dataset.png) 64 | 65 | Sample dataset is available in the data folder: [credit_with_human.csv](data/credit_with_human.csv) 66 | 67 | On top of the above, we need to also collect the response of the Machine Learning model. 68 | 69 | 70 | ## Training a model 71 | 72 | With the provided dataset, we can train a model to predict the target attribute, which in our case is the risk (Risk/No Risk) associated with granting a loan. To gather additional information, we need to capture two properties: mlClass (the model's prediction) and mlConfidence (the probability associated with the prediction). 73 | 74 | If you already have a model, please proceed to the [Bring Your Own Model flow](docs/BYOM.md#byom). 75 | 76 | If not, let's explore how [IBM's AutoAI can assist us in this task](docs/AutoAI.md#autoai). 77 | 78 | 79 | ## Using the solution 80 | 81 | [Running locally](docs/Local.md#local)
82 | Experiment with AIDC functionallity locally
83 | 84 | [Integrations with Cloud Pak for Data/IBM AI Governance](docs/Integrations.md#integrations)
85 | Integrate with several of IBM products to create end-to-end solution to govern your models. 86 | 87 | [Running on IBM Cloud](docs/Cloud.md)
88 | Run the solution on IBM Cloud. 89 | 90 | 91 | ## Contact 92 | 93 | Please contact us at: 94 | AIDC.Contact@ibm.com -------------------------------------------------------------------------------- /docs/AutoAI.md: -------------------------------------------------------------------------------- 1 | 2 | ## AutoAI flow 3 | 4 | We start with creating an IBM Watson Studio project and uploading the necessary datasets: 5 | 6 | Complete file with human results: 7 | [credit_with_human.csv](../data/credit_with_human.csv) 8 | 9 | Training file with no human results: 10 | [credit_no_human.csv](../data/credit_no_human.csv) 11 | 12 | [AIDC library](https://aidecisioncoordination.com/) 13 | 14 | ![new project](../images/new_project.png) 15 | 16 | By clicking the "New asset" button, we will now create an AutoAI experiment using the dataset without human output. 17 | ![new project](../images/new_asset.png) 18 | 19 | We are going to predict the groundTruth, and use Risk as the positive class. 20 | 21 | ![autoai](../images/autoai.png) 22 | 23 | Once the experiment completes, "Save code" of the experiment and navigate to the new notebook. 24 | 25 | ![autoai completed](../images/autoai_completed.png) 26 | 27 | Edit the notebook to start the environment. 28 | ![edit notebook](../images/edit_notebook.png) 29 | 30 | Run the entire notebook: 31 | ![run all](../images/run_all.png) 32 | 33 | You will see that Deployment creation fails due to a missing Space id: 34 | ![space fail](../images/space_fail.png) 35 | 36 | Let's create a new Deployment space and copy it's GUID 37 | 38 | ![deployment](../images/deployment.png) 39 | ![guid](../images/guid.png) 40 | 41 | Coming back to the notebook, we can ignore the failed cell, and add a new cell at the end of the notebook 42 | as we will now introduce AIDC logic to continue the model creation. 43 | 44 | ![new cell](../images/insert_cell.png) 45 | 46 | Please now copy the new cells from the [AutoAI flow](../notebooks/AutoAI_flow.ipynb) notebook, 47 | starting at "Finding the most optimal model with AIDC" 48 | 49 | ![finding](../images/finding.png) 50 | 51 | Starting with library imports, we will analyze the AutoAI provided models using our unique algorithms. 52 | 53 | Initially we will determine how to best distribute the workload between human and AI, 54 | based on the `confidence` (defined as a number between 0 and 1 that represents the likelihood that the output of a Machine Learning model is correct). 55 | 56 | We apply a default cost matrix (also called `performance model`), like so: 57 | ``` 58 | #We gain 1 point for correctly determining a Risk scenario 59 | TruePositiveCost = "1" 60 | 61 | #We lose 1 point for incorrectly marking a scenario as Risk 62 | FalsePositiveCost = "-1" 63 | 64 | #We lose 1 point for incorrectly marking a scenario as No Risk 65 | FalseNegativeCost = "-1" 66 | 67 | #We gain 1 point for correctly determining a No Risk scenario 68 | TrueNegativeCost = "1" 69 | 70 | #For now lets assume the decision taking does not have a cost 71 | ModelDecisionCost = "0" 72 | HumanDecisionCost = "0" 73 | ``` 74 | 75 | You will notice that the distribution (called `dispatch rule`) 76 | will differ across the models, for example: 77 | 78 | ![confidence](../images/confidence.png) 79 | 80 | As expected when the Machine Learning model is not sure about the decision, we can utilize the human resources. 81 | 82 | What's interesting is how different models will compliment humans. 83 | Our tool allows you to calculate the most optimial distribution and provide the statistical information to justify it. 84 | 85 | We sort the models by the best performance (average number of points per decision). 86 | Suprisingly, it may not be the most accurate models which perform best - this is due the fact that human can fill the uncertainty of the models. 87 | 88 | In the next steps we will compare the models, and apply more complex `performance model`. 89 | We can use the attributes of the model, to calculate the impact of each outcome, for example: 90 | ``` 91 | # If Risk, we dont gain anything 92 | TruePositiveCost = "0" 93 | 94 | # We missed a No Risk application 95 | FalsePositiveCost = "-LoanAmount*0.3" 96 | 97 | # The loan was actually Risky 98 | FalseNegativeCost = "-LoanAmount*0.5" 99 | 100 | # Correctly identified as No Risk 101 | TrueNegativeCost = "LoanAmount*0.3" 102 | 103 | #Costs of each decision 104 | ModelDecisionCost = "-0.02" 105 | HumanDecisionCost = "-0.05" 106 | ``` 107 | 108 | Once we run the cells, we will realize that the most optimal solution 109 | based on the defined `performance matrix` depends on human in almost quarter of the decisions. 110 | 111 | `if confidence <35.42%: use ml else if confidence <59.462%: use human else use ml` 112 | 113 | For some of the more accurate models, the distribution is close to 50/50: 114 | 115 | `if confidence <28.39%: use ml else if confidence <81.765%: use human else use ml` 116 | 117 | ![comparison](../images/comparison.png) 118 | 119 | 120 | In the `Calculate ROI` we can see the improvement introduced by implementing the AIDC solution. 121 | ROI is measured as the sum of improvements over each decision between the current process (done by human) and the augmented 122 | human+AI process. 123 |

124 | The total improvement based on the dataset is calculated: 125 | 126 | ![improvement](../images/improvement.png) 127 | 128 | We will later see in the [OpenScale](OpenScale.md#openscale) section how we can monitor the return on investment introduced by AIDC. 129 | 130 | Finally we move to deploying the model and saving the JSON representation of our `dispatch rules`. 131 | 132 | This way we can use it in [Watson Machine Learning](WML.md). 133 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /cloud_assets/app.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import uuid 3 | import os 4 | import requests 5 | from requests.auth import HTTPBasicAuth 6 | import pandas as pd 7 | from flask import Flask, request, abort,current_app 8 | import aidc 9 | import logging 10 | 11 | #logging.basicConfig(filename='/tmp/flask.log', 12 | #level=logging.DEBUG, format=f'%(asctime)s %(levelname)s %('f'name)s %(threadName)s : %(message)s') 13 | 14 | app = Flask(__name__) 15 | 16 | WOS_CREDENTIALS = { 17 | "url": "https://eu-de.aiopenscale.cloud.ibm.com", 18 | "username": "not_used", 19 | "apikey": "your_key", 20 | } 21 | 22 | parms = { 23 | "url": WOS_CREDENTIALS["url"], 24 | "username": WOS_CREDENTIALS["username"], 25 | "apikey": WOS_CREDENTIALS["apikey"] 26 | } 27 | 28 | def custom_metrics_provider(parms = parms): 29 | ### Edit the values for your custom performance model. ### 30 | TruePositiveCost = "0" 31 | FalsePositiveCost = "-LoanAmount*0.03" 32 | FalseNegativeCost = "-LoanAmount*0.5" 33 | TrueNegativeCost = "LoanAmount*0.3" 34 | ModelDecisionCost = "-0.02" 35 | HumanDecisionCost = "-0.05" 36 | ### 37 | 38 | headers = {} 39 | headers["Content-Type"] = "application/json" 40 | headers["Accept"] = "application/json" 41 | 42 | def get_cloud_access_token(): 43 | apikey = parms['apikey'] 44 | url = "https://iam.cloud.ibm.com/identity/token" 45 | headers = { "Content-Type" : "application/x-www-form-urlencoded" } 46 | data = "apikey=" + apikey + "&grant_type=urn:ibm:params:oauth:grant-type:apikey" 47 | response = requests.post( url, headers=headers, data=data) 48 | iam_token = response.json()["access_token"] 49 | return iam_token 50 | 51 | # Get the access token 52 | def get_cp4d_access_token(): 53 | url = '{}/icp4d-api/v1/authorize'.format(parms['url']) 54 | payload = { 55 | 'username': parms['username'], 56 | 'api_key': parms['apikey'] 57 | } 58 | response = requests.post(url, headers=headers, json=payload, verify=False) 59 | json_data = response.json() 60 | access_token = json_data['token'] 61 | return access_token 62 | 63 | #Update the run status to Finished in the Monitor Run 64 | def update_monitor_run_status(base_url, access_token, custom_monitor_instance_id, run_id, status, error_msg = None): 65 | monitor_run_url = base_url + '/v2/monitor_instances/' + custom_monitor_instance_id + '/runs/'+run_id 66 | completed_timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ") 67 | patch_payload = [] 68 | base_path = "/status" 69 | patch_payload.append(get_patch_request_field(base_path, "state", status)) 70 | patch_payload.append(get_patch_request_field(base_path, "completed_at", completed_timestamp)) 71 | if error_msg != None: 72 | error_json = get_error_json(error_msg) 73 | patch_payload.append(get_patch_request_field(base_path, "failure", error_json)) 74 | headers["Authorization"] = "Bearer {}".format(access_token) 75 | response = requests.patch(monitor_run_url, headers=headers, json = patch_payload, verify=False) 76 | monitor_run_response = response.json() 77 | return response.status_code, monitor_run_response 78 | 79 | def get_error_json(error_message): 80 | trace = str(uuid.uuid4()) 81 | error_json = { 82 | 'trace': trace, 83 | 'errors': [{ 84 | 'code': "custom_metrics_error_code", 85 | 'message': str(error_message) 86 | }] 87 | } 88 | return error_json 89 | 90 | def get_patch_request_field(base_path, field_name, field_value, op_name="replace"): 91 | field_json = { 92 | "op": op_name, 93 | "path": "{0}/{1}".format(base_path, field_name), 94 | "value": field_value 95 | } 96 | return field_json 97 | 98 | def sum_performance(payload_data,taskmodel): 99 | total_value=0 100 | probability_position=payload_data['records'][0]['fields'].index("probability") 101 | 102 | for i in range(len(payload_data['records'][0]['values'])): 103 | probability=payload_data["records"][0]["values"][i][probability_position][0] 104 | difference=aidc.sumPerProbability(taskmodel,probability) 105 | total_value+=difference 106 | 107 | return total_value 108 | 109 | def collect_feedback_dataset(access_token, data_mart_id, feedback_dataset_id): 110 | offset = 0 111 | limit = 1000 112 | reading_data = True 113 | json_data = None 114 | annotations = {"annotations": []} 115 | fields = {"fields": []} 116 | values = {"values": []} 117 | result = None 118 | while reading_data: 119 | if feedback_dataset_id is not None: 120 | headers["Authorization"] = "Bearer {}".format(access_token) 121 | DATASETS_STORE_RECORDS_URL = parms["url"] + "/openscale/{0}/v2/data_sets/{1}/records?offset={2}&limit={3}&format=list".format(data_mart_id, feedback_dataset_id, offset, 1000) 122 | response = requests.get(DATASETS_STORE_RECORDS_URL, headers=headers, verify=False) 123 | json_data = response.json() 124 | offset += 1000 125 | if len(json_data["records"]) != 0: 126 | annotations["annotations"] = json_data["records"][0]["annotations"] 127 | fields["fields"] = json_data["records"][0]["fields"] 128 | for val in json_data["records"][0]["values"]: 129 | values["values"].append(val) 130 | if len(json_data["records"]) == 0: 131 | reading_data = False 132 | result = {"records":[{"annotations": annotations["annotations"], "fields": fields["fields"], 133 | "values": values["values"]}]} 134 | return result 135 | 136 | def collect_payload_dataset(access_token, data_mart_id, payload_dataset_id): 137 | json_data = None 138 | if payload_dataset_id is not None: 139 | headers["Authorization"] = "Bearer {}".format(access_token) 140 | DATASETS_STORE_RECORDS_URL = parms["url"] + "/openscale/{0}/v2/data_sets/{1}/records?limit={2}&format=list".format(data_mart_id, payload_dataset_id, 100) 141 | response = requests.get(DATASETS_STORE_RECORDS_URL, headers=headers, verify=False) 142 | json_data = response.json() 143 | return json_data 144 | 145 | def get_metrics(access_token, data_mart_id, subscription_id, 146 | feedback_dataset_id,payload_dataset_id): 147 | json_data = collect_feedback_dataset(access_token, data_mart_id, feedback_dataset_id) 148 | payload_data = collect_payload_dataset(access_token, data_mart_id, payload_dataset_id) 149 | 150 | decisioncost_value=0 151 | performance_value=0 152 | impact_value=0 153 | ml_value=0 154 | human_value=0 155 | roi=0 156 | if json_data is not None and len(json_data["records"][0]['values'])>0: 157 | fields = json_data['records'][0]['fields'] 158 | values = json_data['records'][0]['values'] 159 | feedback_data = pd.DataFrame(values, columns = fields) 160 | table = aidc.load_pandas_data(feedback_data) 161 | taskmodel_data={ 162 | 'id': "0", 163 | 'name': "taskModel", 164 | 'description': "aidc" 165 | } 166 | taskmodel=aidc.create_task_model(table,taskmodel_data) 167 | aidc.set_custom_indicators(taskmodel, TruePositiveCost, FalsePositiveCost, 168 | FalseNegativeCost, TrueNegativeCost, 169 | ModelDecisionCost, HumanDecisionCost) 170 | metrics = aidc.get_indicators(taskmodel) 171 | performance_value = float(metrics["Performance"]) 172 | impact_value = float(metrics["Impact"]) 173 | decisioncost_value = float(metrics["Decision Cost"]) 174 | ml_value = float(metrics["ml volume"]) 175 | human_value = float(metrics["human volume"]) 176 | 177 | if payload_data is not None and len(payload_data["records"])>0 and performance_value>0: 178 | roi=sum_performance(payload_data,taskmodel) 179 | 180 | metrics = {"decisioncost": decisioncost_value, 181 | "performance": performance_value, 182 | "impact": impact_value, 183 | "ml": ml_value, 184 | "human": human_value, 185 | "roi": roi} 186 | return metrics 187 | 188 | # Publishes the Custom Metrics to OpenScale 189 | def publish_metrics(base_url, access_token, data_mart_id, subscription_id, 190 | custom_monitor_id, custom_monitor_instance_id, custom_monitoring_run_id, 191 | feedback_dataset_id, timestamp,payload_dataset_id): 192 | # Generate an monitoring run id, where the publishing happens against this run id 193 | custom_metrics = get_metrics(access_token, data_mart_id, 194 | subscription_id, feedback_dataset_id,payload_dataset_id) 195 | measurements_payload = [ 196 | { 197 | "timestamp": timestamp, 198 | "run_id": custom_monitoring_run_id, 199 | "metrics": [custom_metrics] 200 | } 201 | ] 202 | headers["Authorization"] = "Bearer {}".format(access_token) 203 | measurements_url = base_url + '/v2/monitor_instances/' \ 204 | + custom_monitor_instance_id + '/measurements' 205 | response = requests.post(measurements_url, headers=headers, 206 | json = measurements_payload, verify=False) 207 | published_measurement = response.json() 208 | return response.status_code, published_measurement 209 | 210 | def publish( input_data ): 211 | timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ") 212 | payload = input_data.get("input_data")[0].get("values") 213 | data_mart_id = payload['data_mart_id'] 214 | subscription_id = payload['subscription_id'] 215 | custom_monitor_id = payload['custom_monitor_id'] 216 | custom_monitor_instance_id = payload['custom_monitor_instance_id'] 217 | custom_monitor_instance_params = payload['custom_monitor_instance_params'] 218 | custom_monitor_run_id = payload['custom_monitor_run_id'] 219 | payload_dataset_id = payload.get('payload_dataset_id') 220 | feedback_dataset_id = payload.get('feedback_dataset_id') 221 | base_url = parms['url'] + '/openscale' + '/' + data_mart_id 222 | access_token = get_cloud_access_token() 223 | published_measurements = [] 224 | error_msgs = [] 225 | run_status = "finished" 226 | try: 227 | last_run_time = custom_monitor_instance_params.get("last_run_time") 228 | max_records = custom_monitor_instance_params.get("max_records") 229 | min_records = custom_monitor_instance_params.get("min_records") 230 | error_msg = None 231 | status_code, published_measurement = publish_metrics(base_url, access_token, 232 | data_mart_id, 233 | subscription_id, 234 | custom_monitor_id, 235 | custom_monitor_instance_id, 236 | custom_monitor_run_id, 237 | feedback_dataset_id, timestamp, 238 | payload_dataset_id) 239 | if int(status_code) in [200, 201, 202]: 240 | published_measurements.append(published_measurement) 241 | else: 242 | run_status = "error" 243 | error_msg = published_measurement 244 | error_msgs.append(error_msg) 245 | status_code, response = update_monitor_run_status(base_url, access_token, 246 | custom_monitor_instance_id, 247 | custom_monitor_run_id, 248 | run_status, error_msg) 249 | if int(status_code) not in [200, 201, 202]: 250 | error_msgs.append(response) 251 | except Exception as ex: 252 | error_msgs.append(str(ex)) 253 | if len(error_msgs) == 0: 254 | response_payload = { 255 | "predictions" : [{ 256 | "values" : published_measurements 257 | }] 258 | } 259 | else: 260 | response_payload = { 261 | "predictions":[], 262 | "errors": error_msgs 263 | } 264 | return response_payload 265 | return publish 266 | 267 | @app.route('/',methods=['GET', 'POST']) 268 | def wml_online(openscale): 269 | aidc_function=custom_metrics_provider() 270 | custom_metric_response=aidc_function(request.json) 271 | return custom_metric_response 272 | 273 | if __name__ == '__main__': 274 | app.run(debug=True, port=8082, host='127.0.0.1') 275 | -------------------------------------------------------------------------------- /notebooks/OpenScale_flow.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "collapsed": true, 7 | "id": "9d097b85-66e5-4e0f-89e2-dc7678613509" 8 | }, 9 | "source": [ 10 | "# Sample Notebook - AIDC with OpenScale\n" 11 | ] 12 | }, 13 | { 14 | "cell_type": "markdown", 15 | "metadata": { 16 | "id": "8ea173f9ddba46adb85442c2773fce38" 17 | }, 18 | "source": [ 19 | "### Import AIDC library and requirements" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": 2, 25 | "metadata": { 26 | "id": "32e7042adfea4850968652436972d9ac" 27 | }, 28 | "outputs": [ 29 | { 30 | "name": "stdout", 31 | "output_type": "stream", 32 | "text": [ 33 | "Processing /project_data/data_asset/aidc-2.0.zip\n", 34 | " Preparing metadata (setup.py) ... \u001b[?25ldone\n", 35 | "\u001b[?25hCollecting javascript\n", 36 | " Using cached javascript-1%211.0.3-py3-none-any.whl (33 kB)\n", 37 | "Building wheels for collected packages: aidc\n", 38 | " Building wheel for aidc (setup.py) ... \u001b[?25ldone\n", 39 | "\u001b[?25h Created wheel for aidc: filename=aidc-2.0-py3-none-any.whl size=138109 sha256=d37c095c3affe5086ce7c85819dc67fa730b466c6e0665eb6134c3b0e14f0217\n", 40 | " Stored in directory: /tmp/1000710000/.cache/pip/wheels/36/35/e2/ffafef2711229494b6f5514e8d265d330c9b11bfa3285f4fce\n", 41 | "Successfully built aidc\n", 42 | "Installing collected packages: javascript, aidc\n", 43 | " Attempting uninstall: javascript\n", 44 | " Found existing installation: javascript 1!1.0.3\n", 45 | " Uninstalling javascript-1!1.0.3:\n", 46 | " Successfully uninstalled javascript-1!1.0.3\n", 47 | " Attempting uninstall: aidc\n", 48 | " Found existing installation: aidc 2.0\n", 49 | " Uninstalling aidc-2.0:\n", 50 | " Successfully uninstalled aidc-2.0\n", 51 | "Successfully installed aidc-2.0 javascript-1!1.0.3\n", 52 | "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/envs/Python-RT23.1-Premium/lib/python3.10/site-packages (from packaging->ibm-watson-machine-learning) (3.0.9)\n", 53 | "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/envs/Python-RT23.1-Premium/lib/python3.10/site-packages (from requests<3.0,>=2.0->ibm-watson-openscale) (2.0.4)\n" 54 | ] 55 | } 56 | ], 57 | "source": [ 58 | "## Import AIDC libraries\n", 59 | "!pip install /project_data/data_asset/aidc-2.0.zip;\n", 60 | "!pip install --upgrade ibm-watson-machine-learning | tail -n 1\n", 61 | "!pip install --upgrade ibm-watson-openscale --no-cache | tail -n 1\n", 62 | "import aidc" 63 | ] 64 | }, 65 | { 66 | "cell_type": "markdown", 67 | "metadata": { 68 | "id": "2e25e2f10a0348a397c5ec3ef8951d7b" 69 | }, 70 | "source": [ 71 | "### Configure WML client to deploy the function" 72 | ] 73 | }, 74 | { 75 | "cell_type": "code", 76 | "execution_count": 3, 77 | "metadata": { 78 | "id": "d85ffba037e44a388778f1d199efd1c4" 79 | }, 80 | "outputs": [], 81 | "source": [ 82 | "#Configure the WML client\n", 83 | "import os\n", 84 | "from ibm_watson_machine_learning import APIClient\n", 85 | "\n", 86 | "WOS_CREDENTIALS = {\n", 87 | " \"url\": os.environ.get(\"RUNTIME_ENV_APSX_URL\"),\n", 88 | " \"username\": \"admin\",\n", 89 | " \"apikey\": \"wTCDCBGZkMvCv9wfMqNHX38HIz65zuKxDZsZFJiM\",\n", 90 | "}\n", 91 | "\n", 92 | "#Subscription ID of the model from OpenScale\n", 93 | "subscription_id = \"40c1fef9-db6e-4dfb-9eb8-5b4b275b0cb8\"\n", 94 | "\n", 95 | "#Default guid\n", 96 | "WOS_GUID=\"00000000-0000-0000-0000-000000000000\"\n", 97 | "\n", 98 | "WML_CREDENTIALS = WOS_CREDENTIALS.copy()\n", 99 | "WML_CREDENTIALS['instance_id']='openshift'\n", 100 | "WML_CREDENTIALS['version']='4.7'\n", 101 | "\n", 102 | "PYTHON_FUNCTION_NAME = 'AIDC Custom Metric Function'\n", 103 | "DEPLOYMENT_NAME = 'AIDC Custom Metric Deployment'\n", 104 | "CUSTOM_METRICS_PROVIDER_NAME = \"AIDC Metrics Provider\"\n", 105 | "CUSTOM_MONITOR_NAME = 'Aidc Demo monitor'" 106 | ] 107 | }, 108 | { 109 | "cell_type": "markdown", 110 | "metadata": { 111 | "id": "07792798d2064051816113f333217419" 112 | }, 113 | "source": [ 114 | "### Connect to the space" 115 | ] 116 | }, 117 | { 118 | "cell_type": "code", 119 | "execution_count": 4, 120 | "metadata": { 121 | "id": "6d1b8047a70d436b8bda912a7c195882" 122 | }, 123 | "outputs": [ 124 | { 125 | "data": { 126 | "text/plain": [ 127 | "'SUCCESS'" 128 | ] 129 | }, 130 | "execution_count": 4, 131 | "metadata": {}, 132 | "output_type": "execute_result" 133 | } 134 | ], 135 | "source": [ 136 | "#Configure the space\n", 137 | "wml_client = APIClient(WML_CREDENTIALS)\n", 138 | "\n", 139 | "metadata = { \n", 140 | " wml_client.spaces.ConfigurationMetaNames.NAME: 'AIDC_space', \n", 141 | " wml_client.spaces.ConfigurationMetaNames.DESCRIPTION: 'AIDC space', \n", 142 | " }\n", 143 | "SPACE_NAME=metadata[wml_client.spaces.ConfigurationMetaNames.NAME]\n", 144 | "\n", 145 | "def guid_from_space_name(client, space_name):\n", 146 | " space = client.spaces.get_details()\n", 147 | " try:\n", 148 | " return_item=next(item for item in space['resources'] if item['entity'][\"name\"] == space_name)['metadata']['id']\n", 149 | " except:\n", 150 | " return_item=\"0\"\n", 151 | " return(return_item)\n", 152 | "\n", 153 | "space_uid = guid_from_space_name(wml_client, SPACE_NAME)\n", 154 | "if(space_uid==\"0\"):\n", 155 | " space_details = wml_client.spaces.store(meta_props=metadata,background_mode=False)\n", 156 | " space_uid = guid_from_space_name(wml_client, SPACE_NAME)\n", 157 | "wml_client.set.default_space(space_uid)" 158 | ] 159 | }, 160 | { 161 | "cell_type": "markdown", 162 | "metadata": { 163 | "id": "19711286649f4dd482d64cebb44594fe" 164 | }, 165 | "source": [ 166 | "### Define the monitor logic" 167 | ] 168 | }, 169 | { 170 | "cell_type": "code", 171 | "execution_count": 5, 172 | "metadata": { 173 | "id": "614a0cb08ac842b1aedff861325482e3" 174 | }, 175 | "outputs": [], 176 | "source": [ 177 | "#Custom metric functions\n", 178 | "parms = {\n", 179 | " \"url\": WOS_CREDENTIALS[\"url\"],\n", 180 | " \"username\": WOS_CREDENTIALS[\"username\"],\n", 181 | " \"apikey\": WOS_CREDENTIALS[\"apikey\"]\n", 182 | " }\n", 183 | "def custom_metrics_provider(parms = parms):\n", 184 | " \n", 185 | " ######################## Edit the values for your custom performance model. ############################\n", 186 | " TruePositiveCost = \"0\"\n", 187 | " FalsePositiveCost = \"-LoanAmount*0.03\"\n", 188 | " FalseNegativeCost = \"-LoanAmount*0.5\"\n", 189 | " TrueNegativeCost = \"LoanAmount*0.3\"\n", 190 | " ModelDecisionCost = \"-0.02\"\n", 191 | " HumanDecisionCost = \"-0.05\"\n", 192 | " #################################################################################################\n", 193 | " \n", 194 | " import json\n", 195 | " import requests\n", 196 | " import base64\n", 197 | " from requests.auth import HTTPBasicAuth\n", 198 | " import time\n", 199 | " import uuid\n", 200 | " import datetime\n", 201 | " import aidc\n", 202 | " import pandas as pd\n", 203 | " \n", 204 | " headers = {}\n", 205 | " headers[\"Content-Type\"] = \"application/json\"\n", 206 | " headers[\"Accept\"] = \"application/json\"\n", 207 | " \n", 208 | " \n", 209 | " # Get the access token\n", 210 | " def get_access_token():\n", 211 | " url = '{}/icp4d-api/v1/authorize'.format(parms['url'])\n", 212 | " payload = {\n", 213 | " 'username': parms['username'],\n", 214 | " 'api_key': parms['apikey']\n", 215 | " }\n", 216 | " response = requests.post(url, headers=headers, json=payload, verify=False)\n", 217 | " json_data = response.json()\n", 218 | " access_token = json_data['token']\n", 219 | " return access_token \n", 220 | " \n", 221 | " #Update the run status to Finished in the Monitor Run\n", 222 | " def update_monitor_run_status(base_url, access_token, custom_monitor_instance_id, run_id, status, error_msg = None):\n", 223 | " monitor_run_url = base_url + '/v2/monitor_instances/' + custom_monitor_instance_id + '/runs/'+run_id\n", 224 | " completed_timestamp = datetime.datetime.utcnow().strftime(\"%Y-%m-%dT%H:%M:%S.%fZ\")\n", 225 | " patch_payload = []\n", 226 | " base_path = \"/status\" \n", 227 | " patch_payload.append(get_patch_request_field(base_path, \"state\", status))\n", 228 | " patch_payload.append(get_patch_request_field(base_path, \"completed_at\", completed_timestamp))\n", 229 | " if error_msg != None:\n", 230 | " error_json = get_error_json(error_msg)\n", 231 | " patch_payload.append(get_patch_request_field(base_path, \"failure\", error_json)) \n", 232 | " headers[\"Authorization\"] = \"Bearer {}\".format(access_token)\n", 233 | " response = requests.patch(monitor_run_url, headers=headers, json = patch_payload, verify=False)\n", 234 | " monitor_run_response = response.json()\n", 235 | " return response.status_code, monitor_run_response\n", 236 | " \n", 237 | " \n", 238 | " def get_error_json(error_message):\n", 239 | " trace = str(uuid.uuid4())\n", 240 | " error_json = {\n", 241 | " 'trace': trace,\n", 242 | " 'errors': [{\n", 243 | " 'code': \"custom_metrics_error_code\",\n", 244 | " 'message': str(error_message)\n", 245 | " }]\n", 246 | " }\n", 247 | " return error_json\n", 248 | " \n", 249 | " def get_patch_request_field(base_path, field_name, field_value, op_name=\"replace\"):\n", 250 | " field_json = {\n", 251 | " \"op\": op_name,\n", 252 | " \"path\": \"{0}/{1}\".format(base_path, field_name),\n", 253 | " \"value\": field_value\n", 254 | " }\n", 255 | " return field_json\n", 256 | " \n", 257 | " def sumPerformance(payload_data,taskmodel):\n", 258 | " total_value=0\n", 259 | " probability_position=payload_data['records'][0]['fields'].index(\"probability\") \n", 260 | "\n", 261 | " for i in range(len(payload_data['records'][0]['values'])):\n", 262 | " probability=payload_data[\"records\"][0][\"values\"][i][probability_position][0]\n", 263 | " difference=aidc.sumPerProbability(taskmodel,probability)\n", 264 | " total_value+=difference \n", 265 | " \n", 266 | " return total_value\n", 267 | " \n", 268 | " def collect_feedback_dataset(access_token, data_mart_id, feedback_dataset_id): \n", 269 | " offset = 0\n", 270 | " limit = 1000\n", 271 | " reading_data = True\n", 272 | " json_data = None\n", 273 | " annotations = {\"annotations\": []}\n", 274 | " fields = {\"fields\": []}\n", 275 | " values = {\"values\": []} \n", 276 | " result = None \n", 277 | " while reading_data:\n", 278 | " if feedback_dataset_id is not None:\n", 279 | " headers[\"Authorization\"] = \"Bearer {}\".format(access_token)\n", 280 | " DATASETS_STORE_RECORDS_URL = parms[\"url\"] + \"/openscale/{0}/v2/data_sets/{1}/records?offset={2}&limit={3}&format=list\".format(data_mart_id, feedback_dataset_id, offset, 1000)\n", 281 | " response = requests.get(DATASETS_STORE_RECORDS_URL, headers=headers, verify=False)\n", 282 | " json_data = response.json() \n", 283 | " offset += 1000 \n", 284 | " if len(json_data[\"records\"]) != 0: \n", 285 | " annotations[\"annotations\"] = json_data[\"records\"][0][\"annotations\"] \n", 286 | " fields[\"fields\"] = json_data[\"records\"][0][\"fields\"] \n", 287 | " for val in json_data[\"records\"][0][\"values\"]: \n", 288 | " values[\"values\"].append(val) \n", 289 | " if len(json_data[\"records\"]) == 0: \n", 290 | " reading_data = False \n", 291 | " result = {\"records\":[{\"annotations\": annotations[\"annotations\"], \"fields\": fields[\"fields\"], \"values\": values[\"values\"]}]} \n", 292 | " return result\n", 293 | " \n", 294 | " def collect_payload_dataset(access_token, data_mart_id, payload_dataset_id):\n", 295 | " json_data = None\n", 296 | " if payload_dataset_id is not None:\n", 297 | " headers[\"Authorization\"] = \"Bearer {}\".format(access_token)\n", 298 | " DATASETS_STORE_RECORDS_URL = parms[\"url\"] + \"/openscale/{0}/v2/data_sets/{1}/records?limit={2}&format=list\".format(data_mart_id, payload_dataset_id, 100)\n", 299 | " response = requests.get(DATASETS_STORE_RECORDS_URL, headers=headers, verify=False)\n", 300 | " json_data = response.json()\n", 301 | " return json_data\n", 302 | " \n", 303 | " def get_metrics(access_token, data_mart_id, subscription_id, feedback_dataset_id,payload_dataset_id): \n", 304 | " json_data = collect_feedback_dataset(access_token, data_mart_id, feedback_dataset_id) \n", 305 | " payload_data = collect_payload_dataset(access_token, data_mart_id, payload_dataset_id) \n", 306 | " \n", 307 | " number_of_requests=0\n", 308 | " decisioncost_value=0\n", 309 | " performance_value=0\n", 310 | " impact_value=0\n", 311 | " ml_value=0\n", 312 | " human_value=0\n", 313 | " roi=0\n", 314 | " if json_data is not None and len(json_data[\"records\"][0]['values'])>0:\n", 315 | " fields = json_data['records'][0]['fields']\n", 316 | " values = json_data['records'][0]['values']\n", 317 | " feedback_data = pd.DataFrame(values, columns = fields)\n", 318 | " table = aidc.load_pandas_data(feedback_data) \n", 319 | " taskmodel_data={\n", 320 | " 'id': \"0\",\n", 321 | " 'name': \"taskModel\",\n", 322 | " 'description': \"aidc\"\n", 323 | " }\n", 324 | " taskmodel=aidc.create_task_model(table,taskmodel_data) \n", 325 | " aidc.set_custom_indicators(taskmodel, TruePositiveCost, FalsePositiveCost, FalseNegativeCost, TrueNegativeCost, ModelDecisionCost, HumanDecisionCost) \n", 326 | " metrics = aidc.get_indicators(taskmodel) \n", 327 | " performance_value = float(metrics[\"Performance\"]) \n", 328 | " impact_value = float(metrics[\"Impact\"]) \n", 329 | " decisioncost_value = float(metrics[\"Decision Cost\"]) \n", 330 | " ml_value = float(metrics[\"ml volume\"]) \n", 331 | " human_value = float(metrics[\"human volume\"]) \n", 332 | " \n", 333 | " if payload_data is not None and len(payload_data[\"records\"])>0 and performance_value>0:\n", 334 | " roi=sumPerformance(payload_data,taskmodel)\n", 335 | " \n", 336 | " metrics = {\"decisioncost\": decisioncost_value,\n", 337 | " \"performance\": performance_value, \n", 338 | " \"impact\": impact_value, \n", 339 | " \"ml\": ml_value, \n", 340 | " \"human\": human_value,\n", 341 | " \"roi\": roi}\n", 342 | " return metrics\n", 343 | " \n", 344 | " # Publishes the Custom Metrics to OpenScale\n", 345 | " def publish_metrics(base_url, access_token, data_mart_id, subscription_id, custom_monitor_id, custom_monitor_instance_id, custom_monitoring_run_id, feedback_dataset_id, timestamp,payload_dataset_id):\n", 346 | " # Generate an monitoring run id, where the publishing happens against this run id\n", 347 | " custom_metrics = get_metrics(access_token, data_mart_id, subscription_id, feedback_dataset_id,payload_dataset_id)\n", 348 | " measurements_payload = [\n", 349 | " {\n", 350 | " \"timestamp\": timestamp,\n", 351 | " \"run_id\": custom_monitoring_run_id,\n", 352 | " \"metrics\": [custom_metrics]\n", 353 | " }\n", 354 | " ]\n", 355 | " headers[\"Authorization\"] = \"Bearer {}\".format(access_token)\n", 356 | " measurements_url = base_url + '/v2/monitor_instances/' + custom_monitor_instance_id + '/measurements'\n", 357 | " response = requests.post(measurements_url, headers=headers, json = measurements_payload, verify=False)\n", 358 | " published_measurement = response.json()\n", 359 | " return response.status_code, published_measurement\n", 360 | " \n", 361 | " \n", 362 | " def publish( input_data ): \n", 363 | " timestamp = datetime.datetime.utcnow().strftime(\"%Y-%m-%dT%H:%M:%S.%fZ\") \n", 364 | " payload = input_data.get(\"input_data\")[0].get(\"values\")\n", 365 | " data_mart_id = payload['data_mart_id']\n", 366 | " subscription_id = payload['subscription_id']\n", 367 | " custom_monitor_id = payload['custom_monitor_id']\n", 368 | " custom_monitor_instance_id = payload['custom_monitor_instance_id']\n", 369 | " custom_monitor_instance_params = payload['custom_monitor_instance_params']\n", 370 | " custom_monitor_run_id = payload['custom_monitor_run_id']\n", 371 | " payload_dataset_id = payload.get('payload_dataset_id')\n", 372 | " feedback_dataset_id = payload.get('feedback_dataset_id')\n", 373 | " base_url = parms['url'] + '/openscale' + '/' + data_mart_id\n", 374 | " access_token = get_access_token() \n", 375 | " published_measurements = []\n", 376 | " error_msgs = []\n", 377 | " run_status = \"finished\" \n", 378 | " try:\n", 379 | " last_run_time = custom_monitor_instance_params.get(\"last_run_time\")\n", 380 | " max_records = custom_monitor_instance_params.get(\"max_records\")\n", 381 | " min_records = custom_monitor_instance_params.get(\"min_records\")\n", 382 | " error_msg = None \n", 383 | " status_code, published_measurement = publish_metrics(base_url, access_token, data_mart_id, subscription_id, custom_monitor_id, custom_monitor_instance_id, custom_monitor_run_id, feedback_dataset_id, timestamp, payload_dataset_id)\n", 384 | " if int(status_code) in [200, 201, 202]:\n", 385 | " published_measurements.append(published_measurement)\n", 386 | " else:\n", 387 | " run_status = \"error\"\n", 388 | " error_msg = published_measurement\n", 389 | " error_msgs.append(error_msg) \n", 390 | " status_code, response = update_monitor_run_status(base_url, access_token, custom_monitor_instance_id, custom_monitor_run_id, run_status, error_msg) \n", 391 | " if not int(status_code) in [200, 201, 202]:\n", 392 | " error_msgs.append(response) \n", 393 | " except Exception as ex:\n", 394 | " error_msgs.append(str(ex))\n", 395 | " if len(error_msgs) == 0:\n", 396 | " response_payload = {\n", 397 | " \"predictions\" : [{ \n", 398 | " \"values\" : published_measurements\n", 399 | " }]\n", 400 | " }\n", 401 | " else:\n", 402 | " response_payload = {\n", 403 | " \"predictions\":[],\n", 404 | " \"errors\": error_msgs\n", 405 | " } \n", 406 | " return response_payload\n", 407 | " \n", 408 | " return publish" 409 | ] 410 | }, 411 | { 412 | "cell_type": "markdown", 413 | "metadata": { 414 | "id": "910f5a31068a49658fa67824d85d187c" 415 | }, 416 | "source": [ 417 | "### Prepare software specifications" 418 | ] 419 | }, 420 | { 421 | "cell_type": "code", 422 | "execution_count": 6, 423 | "metadata": { 424 | "id": "f589ac2de4824b8b9572aa184daea669" 425 | }, 426 | "outputs": [ 427 | { 428 | "name": "stdout", 429 | "output_type": "stream", 430 | "text": [ 431 | "Deleting deployment id 3c40e6cf-d29f-4f99-826c-00cf4aa96387\n", 432 | "Deleting model id 79fcc6b8-3634-4c19-ab30-bf2cb6500ea3\n", 433 | "aidc-spec\n", 434 | "Creating package extensions\n", 435 | "SUCCESS\n", 436 | "\n", 437 | "\n", 438 | "#######################################################################################\n", 439 | "\n", 440 | "Synchronous deployment creation for uid: '45308d01-198a-42b5-8a8a-1f30a3072c41' started\n", 441 | "\n", 442 | "#######################################################################################\n", 443 | "\n", 444 | "\n", 445 | "initializing\n", 446 | "Note: online_url is deprecated and will be removed in a future release. Use serving_urls instead.\n", 447 | "......................................\n", 448 | "ready\n", 449 | "\n", 450 | "\n", 451 | "------------------------------------------------------------------------------------------------\n", 452 | "Successfully finished deployment creation, deployment_uid='89d2ea08-e531-4e68-b84c-1df3778b42f1'\n", 453 | "------------------------------------------------------------------------------------------------\n", 454 | "\n", 455 | "\n" 456 | ] 457 | } 458 | ], 459 | "source": [ 460 | "#Prepare and deploy the custom function\n", 461 | "\n", 462 | "#Clean previous deployments\n", 463 | "deployments_list = wml_client.deployments.get_details()\n", 464 | "for deployment in deployments_list[\"resources\"]:\n", 465 | " model_id = deployment[\"entity\"][\"asset\"][\"id\"]\n", 466 | " deployment_id = deployment[\"metadata\"][\"id\"]\n", 467 | " if deployment[\"metadata\"][\"name\"] == DEPLOYMENT_NAME:\n", 468 | " print(\"Deleting deployment id\", deployment_id)\n", 469 | " wml_client.deployments.delete(deployment_id)\n", 470 | " print(\"Deleting model id\", model_id)\n", 471 | " wml_client.repository.delete(model_id)\n", 472 | " \n", 473 | "#Find the AIDC custom software definition\n", 474 | "file_path_AIDC = '/project_data/data_asset/aidc-2.0.zip'\n", 475 | "specs_list = wml_client.software_specifications.get_details()\n", 476 | "for spec in specs_list[\"resources\"]:\n", 477 | " name=(spec['metadata']['name'])\n", 478 | " if(\"aidc-spec\" in name):\n", 479 | " print(name)\n", 480 | " base_software_spec_id = wml_client.software_specifications.get_id_by_name(name)\n", 481 | "\n", 482 | "\n", 483 | "# Package extension metadata\n", 484 | "meta_prop_pkg_ext = {\n", 485 | " wml_client.package_extensions.ConfigurationMetaNames.NAME: \"AIDC_custom_package\",\n", 486 | " wml_client.package_extensions.ConfigurationMetaNames.DESCRIPTION: \"AIDC custom package\",\n", 487 | " wml_client.package_extensions.ConfigurationMetaNames.TYPE: \"pip_zip\"\n", 488 | "}\n", 489 | "package_ext_details = wml_client.package_extensions.store(meta_props=meta_prop_pkg_ext, file_path = file_path_AIDC)\n", 490 | "package_ext_uid = wml_client.package_extensions.get_uid(package_ext_details)\n", 491 | "\n", 492 | "# Prepare software specificaion\n", 493 | "meta_prop_sw_spec = {\n", 494 | " wml_client.software_specifications.ConfigurationMetaNames.NAME: \"AIDC_software_spec\",\n", 495 | " wml_client.software_specifications.ConfigurationMetaNames.DESCRIPTION: \"AIDC software specification\",\n", 496 | " wml_client.software_specifications.ConfigurationMetaNames.PACKAGE_EXTENSIONS : [{\n", 497 | " \"guid\": package_ext_uid\n", 498 | " }],\n", 499 | " wml_client.software_specifications.ConfigurationMetaNames.BASE_SOFTWARE_SPECIFICATION: {\n", 500 | " \"guid\": base_software_spec_id\n", 501 | " }\n", 502 | "}\n", 503 | "software_spec_details = wml_client.software_specifications.store(meta_props=meta_prop_sw_spec)\n", 504 | "software_spec_uid = wml_client.software_specifications.get_uid(software_spec_details)\n", 505 | "\n", 506 | "#Function meta details\n", 507 | "meta_props = {\n", 508 | " wml_client.repository.FunctionMetaNames.NAME: PYTHON_FUNCTION_NAME,\n", 509 | " wml_client.repository.FunctionMetaNames.SOFTWARE_SPEC_ID: software_spec_uid\n", 510 | "}\n", 511 | "\n", 512 | "#Prepare the function\n", 513 | "function_artifact = wml_client.repository.store_function(meta_props=meta_props, function=custom_metrics_provider)\n", 514 | "function_uid = wml_client.repository.get_function_id(function_artifact)\n", 515 | "\n", 516 | "#Prepare hardware specs\n", 517 | "hardware_spec_id = wml_client.hardware_specifications.get_id_by_name('S')\n", 518 | "\n", 519 | "#Prepare the deployment metadata\n", 520 | "deploy_meta = {\n", 521 | " wml_client.deployments.ConfigurationMetaNames.NAME: DEPLOYMENT_NAME,\n", 522 | " wml_client.deployments.ConfigurationMetaNames.ONLINE: {},\n", 523 | " wml_client.deployments.ConfigurationMetaNames.HARDWARE_SPEC: { \"id\": hardware_spec_id}\n", 524 | "}\n", 525 | "\n", 526 | "#Deploy the function\n", 527 | "deployment_details = wml_client.deployments.create(function_uid, meta_props=deploy_meta)\n", 528 | "\n", 529 | "created_at = deployment_details['metadata']['created_at']\n", 530 | "find_string_pos = created_at.find(\"T\")\n", 531 | "if find_string_pos != -1:\n", 532 | " current_date = created_at[0:find_string_pos]\n", 533 | "scoring_url = wml_client.deployments.get_scoring_href(deployment_details)\n", 534 | "scoring_url = scoring_url + \"?version=\" + current_date" 535 | ] 536 | }, 537 | { 538 | "cell_type": "markdown", 539 | "metadata": { 540 | "id": "eba647375bf2455ab7dc53e173ae7b1d" 541 | }, 542 | "source": [ 543 | "### Configure Watson OpenScale client" 544 | ] 545 | }, 546 | { 547 | "cell_type": "code", 548 | "execution_count": 7, 549 | "metadata": { 550 | "id": "ccdc3db67d0e4f11965302b08b91d9a5" 551 | }, 552 | "outputs": [], 553 | "source": [ 554 | "#Configure WOS client\n", 555 | "from ibm_watson_openscale import APIClient\n", 556 | "from ibm_watson_openscale.base_classes.watson_open_scale_v2 import MonitorMeasurementRequest\n", 557 | "from ibm_watson_openscale.base_classes.watson_open_scale_v2 import MonitorMetricRequest\n", 558 | "from ibm_watson_openscale.base_classes.watson_open_scale_v2 import MetricThreshold\n", 559 | "from ibm_watson_openscale.supporting_classes.enums import MetricThresholdTypes\n", 560 | "from ibm_watson_openscale.base_classes.watson_open_scale_v2 import MonitorTagRequest\n", 561 | "from ibm_watson_openscale.base_classes.watson_open_scale_v2 import Target\n", 562 | "from ibm_watson_openscale.supporting_classes.enums import TargetTypes\n", 563 | "from ibm_watson_openscale.base_classes.watson_open_scale_v2 import IntegratedSystems, ApplicabilitySelection\n", 564 | "from ibm_watson_openscale.base_classes.watson_open_scale_v2 import MonitorInstanceSchedule, ScheduleStartTime, MonitorRuntime\n", 565 | "\n", 566 | "from datetime import datetime, timezone, timedelta\n", 567 | "import uuid\n", 568 | "\n", 569 | "from ibm_cloud_sdk_core.authenticators import CloudPakForDataAuthenticator\n", 570 | "\n", 571 | "from ibm_watson_openscale import *\n", 572 | "from ibm_watson_openscale.supporting_classes.enums import *\n", 573 | "from ibm_watson_openscale.supporting_classes import *\n", 574 | "\n", 575 | "\n", 576 | "authenticator = CloudPakForDataAuthenticator(\n", 577 | " url=WOS_CREDENTIALS['url'],\n", 578 | " username=WOS_CREDENTIALS['username'],\n", 579 | " apikey=WOS_CREDENTIALS['apikey'],\n", 580 | " disable_ssl_verification=True\n", 581 | " )\n", 582 | "wos_client = APIClient(service_url=WOS_CREDENTIALS['url'],authenticator=authenticator, service_instance_id=WOS_GUID)" 583 | ] 584 | }, 585 | { 586 | "cell_type": "markdown", 587 | "metadata": { 588 | "id": "2c4b9432fc7949deb5ec12e1451ac492" 589 | }, 590 | "source": [ 591 | "### Configure authentication" 592 | ] 593 | }, 594 | { 595 | "cell_type": "code", 596 | "execution_count": 8, 597 | "metadata": { 598 | "id": "5b124b58332b4016822c1b1fc3446d8a" 599 | }, 600 | "outputs": [], 601 | "source": [ 602 | "auth_type = \"bearer\" #Supported values are basic and bearer\n", 603 | "\n", 604 | "if auth_type == \"basic\":\n", 605 | " CUSTOM_METRICS_PROVIDER_CREDENTIALS = {\n", 606 | " \"auth_type\":\"basic\",\n", 607 | " \"username\": \"*****\",# update the username here \n", 608 | " \"password\": \"*****\"# Update the password here\n", 609 | " }\n", 610 | " \n", 611 | "if auth_type == \"bearer\":\n", 612 | " CUSTOM_METRICS_PROVIDER_CREDENTIALS = {\n", 613 | " \"auth_type\":\"bearer\",\n", 614 | " \"token_info\": {\n", 615 | " \"url\": \"{}/icp4d-api/v1/authorize\".format(WOS_CREDENTIALS['url']),\n", 616 | " \"headers\": {\n", 617 | " \"Content-Type\": \"application/json\",\n", 618 | " \"Accept\": \"application/json\"\n", 619 | " },\n", 620 | " \"payload\": {\n", 621 | " \"username\": WOS_CREDENTIALS['username'],\n", 622 | " \"api_key\": WOS_CREDENTIALS['apikey'],\n", 623 | " },\n", 624 | " \"method\": \"post\"\n", 625 | " }\n", 626 | " }" 627 | ] 628 | }, 629 | { 630 | "cell_type": "markdown", 631 | "metadata": { 632 | "id": "1de1ac8b344f4fc0891c4f28349370d4" 633 | }, 634 | "source": [ 635 | "### Define the integration system" 636 | ] 637 | }, 638 | { 639 | "cell_type": "code", 640 | "execution_count": 9, 641 | "metadata": { 642 | "id": "4721c08698864a82806b5dcae1cd89f8" 643 | }, 644 | "outputs": [ 645 | { 646 | "name": "stdout", 647 | "output_type": "stream", 648 | "text": [ 649 | "Deleting integrated system AIDC Metrics Provider\n" 650 | ] 651 | } 652 | ], 653 | "source": [ 654 | "# Delete existing custom metrics provider integrated systems if present\n", 655 | "integrated_systems = IntegratedSystems(wos_client).list().result.integrated_systems\n", 656 | "for system in integrated_systems:\n", 657 | " if system.entity.type == 'custom_metrics_provider' and system.entity.name == CUSTOM_METRICS_PROVIDER_NAME:\n", 658 | " print(\"Deleting integrated system {}\".format(system.entity.name))\n", 659 | " IntegratedSystems(wos_client).delete(integrated_system_id=system.metadata.id)\n", 660 | "\n", 661 | "#Create new integrated system\n", 662 | "custom_metrics_integrated_system = IntegratedSystems(wos_client).add(\n", 663 | " name=CUSTOM_METRICS_PROVIDER_NAME,\n", 664 | " description=CUSTOM_METRICS_PROVIDER_NAME,\n", 665 | " type=\"custom_metrics_provider\",\n", 666 | " credentials= CUSTOM_METRICS_PROVIDER_CREDENTIALS,\n", 667 | " connection={\n", 668 | " \"display_name\": CUSTOM_METRICS_PROVIDER_NAME,\n", 669 | " \"endpoint\": scoring_url\n", 670 | " }\n", 671 | ").result\n", 672 | "\n", 673 | "integrated_system_id = custom_metrics_integrated_system.metadata.id \n", 674 | "\n", 675 | "def get_custom_monitor_definition():\n", 676 | " monitor_definitions = wos_client.monitor_definitions.list().result.monitor_definitions\n", 677 | " for definition in monitor_definitions:\n", 678 | " if CUSTOM_MONITOR_NAME == definition.entity.name:\n", 679 | " return definition\n", 680 | " return None " 681 | ] 682 | }, 683 | { 684 | "cell_type": "markdown", 685 | "metadata": { 686 | "id": "95206a6ad54243c1b1e24e4bb1485ba7" 687 | }, 688 | "source": [ 689 | "### Configure the metrics" 690 | ] 691 | }, 692 | { 693 | "cell_type": "code", 694 | "execution_count": 10, 695 | "metadata": { 696 | "id": "5048f968f4674d40845ea33821cfcaf9" 697 | }, 698 | "outputs": [], 699 | "source": [ 700 | "# Update the input data type of your model. \n", 701 | "input_data_type = [\"structured\"]\n", 702 | "algorithm_types = [\"binary\"]\n", 703 | "problemTypeSelection = ApplicabilitySelection(problem_type=algorithm_types)\n", 704 | "inputDataTypeSelection = ApplicabilitySelection(input_data_type=input_data_type)\n", 705 | "monitor_description = 'Set of AIDC Metrics'\n", 706 | "\n", 707 | "#Update the tag values if you want to fetch the metrics by tags\n", 708 | "TAGS= ['provider']\n", 709 | "TAG_DESCRIPTION =['AIDC']\n", 710 | "CUSTOM_MONITOR_METRICS_NAMES = [\"decisioncost\",'performance', 'impact', 'ml', 'human','roi']\n", 711 | "CUSTOM_DESCRIPTION = ['decision cost for the model', 'Average performance of the model', 'Average impact for the model', 'ml percentage', 'human percentage','Return on investment']" 712 | ] 713 | }, 714 | { 715 | "cell_type": "markdown", 716 | "metadata": { 717 | "id": "8a035186e4a048dd835df7f2b863f116" 718 | }, 719 | "source": [ 720 | "### Define thresholds" 721 | ] 722 | }, 723 | { 724 | "cell_type": "code", 725 | "execution_count": 11, 726 | "metadata": { 727 | "id": "9a3a05aa37bf4b05930337360cd2d44e" 728 | }, 729 | "outputs": [], 730 | "source": [ 731 | "#Update the Threshold types and default values of the metrics\n", 732 | "def custom_metric_definitions():\n", 733 | " \n", 734 | " metrics = [\n", 735 | " MonitorMetricRequest(name=CUSTOM_MONITOR_METRICS_NAMES[0], applies_to=problemTypeSelection,\n", 736 | " description=CUSTOM_DESCRIPTION[0],\n", 737 | " thresholds=[MetricThreshold(type=MetricThresholdTypes.LOWER_LIMIT, default=-1),\n", 738 | " MetricThreshold(type=MetricThresholdTypes.UPPER_LIMIT, default=1)]),\n", 739 | " MonitorMetricRequest(name=CUSTOM_MONITOR_METRICS_NAMES[1], applies_to=problemTypeSelection,\n", 740 | " description=CUSTOM_DESCRIPTION[1],\n", 741 | " thresholds=[MetricThreshold(type=MetricThresholdTypes.LOWER_LIMIT, default=10),\n", 742 | " MetricThreshold(type=MetricThresholdTypes.UPPER_LIMIT, default=1000)]),\n", 743 | " MonitorMetricRequest(name=CUSTOM_MONITOR_METRICS_NAMES[2], applies_to=problemTypeSelection,\n", 744 | " description=CUSTOM_DESCRIPTION[2], \n", 745 | " thresholds=[MetricThreshold(type=MetricThresholdTypes.LOWER_LIMIT, default=10),\n", 746 | " MetricThreshold(type=MetricThresholdTypes.UPPER_LIMIT, default=1000)]),\n", 747 | " MonitorMetricRequest(name=CUSTOM_MONITOR_METRICS_NAMES[3], applies_to=problemTypeSelection,\n", 748 | " description=CUSTOM_DESCRIPTION[3], \n", 749 | " thresholds=[MetricThreshold(type=MetricThresholdTypes.LOWER_LIMIT, default=40),\n", 750 | " MetricThreshold(type=MetricThresholdTypes.UPPER_LIMIT, default=80)]),\n", 751 | " MonitorMetricRequest(name=CUSTOM_MONITOR_METRICS_NAMES[4], applies_to=problemTypeSelection,\n", 752 | " description=CUSTOM_DESCRIPTION[4],\n", 753 | " thresholds=[MetricThreshold(type=MetricThresholdTypes.LOWER_LIMIT, default=20),\n", 754 | " MetricThreshold(type=MetricThresholdTypes.UPPER_LIMIT, default=50)]),\n", 755 | " MonitorMetricRequest(name=CUSTOM_MONITOR_METRICS_NAMES[5], applies_to=problemTypeSelection,\n", 756 | " description=CUSTOM_DESCRIPTION[5],\n", 757 | " thresholds=[MetricThreshold(type=MetricThresholdTypes.LOWER_LIMIT, default=100),\n", 758 | " MetricThreshold(type=MetricThresholdTypes.UPPER_LIMIT, default=1000.0)])\n", 759 | " ]\n", 760 | " #Comment the below tags code if there are no tags to be created\n", 761 | " tags = [MonitorTagRequest(name=TAGS[0], description=TAG_DESCRIPTION[0])]\n", 762 | "\n", 763 | " return metrics, tags" 764 | ] 765 | }, 766 | { 767 | "cell_type": "markdown", 768 | "metadata": { 769 | "id": "317463ff1ec143208a3c9ba4a36e9eb0" 770 | }, 771 | "source": [ 772 | "### Configure schedule" 773 | ] 774 | }, 775 | { 776 | "cell_type": "code", 777 | "execution_count": 12, 778 | "metadata": { 779 | "id": "203ce57916584ed58de684c8ead50ac9" 780 | }, 781 | "outputs": [], 782 | "source": [ 783 | "#Enable the schedule for your custom monitor. Update the value to false if you want to disable the schedule\n", 784 | "ENABLE_SCHEDULE = True\n", 785 | "\n", 786 | "#Update the repeat interval and repeat type: Default is 1 hour\n", 787 | "repeat_interval = 1\n", 788 | "repeat_type = \"hour\"\n", 789 | "\n", 790 | "#Update the delay unit and interval to trigger the first schedule run after the custom monitor instance is created. Default is 30 minutes\n", 791 | "delay_unit = \"minute\"\n", 792 | "delay_time= 30\n", 793 | "\n", 794 | "start_time= ScheduleStartTime(type = \"relative\", delay_unit= delay_unit, delay = delay_time)\n", 795 | "schedule = MonitorInstanceSchedule(repeat_interval=repeat_interval,repeat_unit=repeat_type, start_time=start_time) \n", 796 | "\n", 797 | "#Update the wait time here.\n", 798 | "custom_metrics_wait_time = 50 #time in seconds \n", 799 | "\n", 800 | "#Maximum number of records to consider during custom monitor run evaluation. Update max_records value as per the requirement\n", 801 | "max_records = None\n", 802 | "#Minimum number of records to consider during custom monitor run evaluation. Update min_records value as per the requirement\n", 803 | "min_records = None" 804 | ] 805 | }, 806 | { 807 | "cell_type": "markdown", 808 | "metadata": { 809 | "id": "8053cc712ebb40f49d0c80c1960964c1" 810 | }, 811 | "source": [ 812 | "### Define the monitor " 813 | ] 814 | }, 815 | { 816 | "cell_type": "code", 817 | "execution_count": 13, 818 | "metadata": { 819 | "id": "e4eea89b80e64b498214d7fdf0a8aea2" 820 | }, 821 | "outputs": [ 822 | { 823 | "name": "stdout", 824 | "output_type": "stream", 825 | "text": [ 826 | "Deletting the existing monitor definition..\n", 827 | "\n", 828 | "\n", 829 | "==================================================================\n", 830 | "\n", 831 | " Waiting for end of deleting monitor definition aidc_demo_monitor \n", 832 | "\n", 833 | "==================================================================\n", 834 | "\n", 835 | "\n", 836 | "\n", 837 | "finished\n", 838 | "\n", 839 | "---------------------------------------------------\n", 840 | " Successfully finished deleting monitor definition \n", 841 | "---------------------------------------------------\n", 842 | "\n", 843 | "\n", 844 | "Creating the existing monitor definition..\n", 845 | "\n", 846 | "\n", 847 | "================================================================\n", 848 | "\n", 849 | " Waiting for end of adding monitor definition aidc_demo_monitor \n", 850 | "\n", 851 | "================================================================\n", 852 | "\n", 853 | "\n", 854 | "\n", 855 | "finished\n", 856 | "\n", 857 | "-------------------------------------------------\n", 858 | " Successfully finished adding monitor definition \n", 859 | "-------------------------------------------------\n", 860 | "\n", 861 | "\n" 862 | ] 863 | } 864 | ], 865 | "source": [ 866 | "def create_custom_monitor_definition():\n", 867 | " # check if the custom monitor definition already exists or not\n", 868 | " existing_definition = get_custom_monitor_definition()\n", 869 | "\n", 870 | " # if it does not exists, then create a new one.\n", 871 | " if existing_definition is not None:\n", 872 | " print('Deletting the existing monitor definition..')\n", 873 | " wos_client.monitor_definitions.delete(monitor_definition_id = existing_definition.metadata.id, background_mode = False, force = True)\n", 874 | " \n", 875 | " # make sure it is indeed deleted..\n", 876 | " existing_definition = get_custom_monitor_definition()\n", 877 | " if existing_definition is None:\n", 878 | " print('Creating the existing monitor definition..')\n", 879 | " metrics, tags = custom_metric_definitions()\n", 880 | " if ENABLE_SCHEDULE:\n", 881 | " monitor_runtime = MonitorRuntime(type=\"custom_metrics_provider\")\n", 882 | " custom_monitor_details = wos_client.monitor_definitions.add(name=CUSTOM_MONITOR_NAME, description=monitor_description, metrics=metrics, tags=tags, schedule = schedule, \n", 883 | " applies_to=inputDataTypeSelection, monitor_runtime = monitor_runtime, background_mode=False).result\n", 884 | " else:\n", 885 | " custom_monitor_details = wos_client.monitor_definitions.add(name=CUSTOM_MONITOR_NAME, description=monitor_description, metrics=metrics, tags=tags, applies_to=inputDataTypeSelection, background_mode=False).result\n", 886 | " else:\n", 887 | " # otherwise, send the existing definition\n", 888 | " print('Monitor deletion did not happen.')\n", 889 | " return custom_monitor_details\n", 890 | "\n", 891 | "custom_monitor_details = create_custom_monitor_definition()\n", 892 | "custom_monitor_id = custom_monitor_details.metadata.id" 893 | ] 894 | }, 895 | { 896 | "cell_type": "markdown", 897 | "metadata": { 898 | "id": "38f68f991e0543629c8e7b644d744701" 899 | }, 900 | "source": [ 901 | "### Create the monitor" 902 | ] 903 | }, 904 | { 905 | "cell_type": "code", 906 | "execution_count": 14, 907 | "metadata": { 908 | "id": "422588e6169e42d181617a03e3c35a29" 909 | }, 910 | "outputs": [ 911 | { 912 | "name": "stdout", 913 | "output_type": "stream", 914 | "text": [ 915 | "\n", 916 | "\n", 917 | "===================================================================================\n", 918 | "\n", 919 | " Waiting for end of monitor instance creation cd09f5db-409b-4244-bcae-25a5123c55b5 \n", 920 | "\n", 921 | "===================================================================================\n", 922 | "\n", 923 | "\n", 924 | "\n", 925 | "active\n", 926 | "\n", 927 | "---------------------------------------\n", 928 | " Monitor instance successfully created \n", 929 | "---------------------------------------\n", 930 | "\n", 931 | "\n" 932 | ] 933 | } 934 | ], 935 | "source": [ 936 | "def get_custom_monitor_instance(custom_monitor_id):\n", 937 | " monitor_instances = wos_client.monitor_instances.list(data_mart_id = WOS_GUID, monitor_definition_id = custom_monitor_id, target_target_id = subscription_id).result.monitor_instances\n", 938 | " if len(monitor_instances) == 1:\n", 939 | " return monitor_instances[0]\n", 940 | " return None\n", 941 | "\n", 942 | "def update_custom_monitor_instance(custom_monitor_instance_id):\n", 943 | " payload = [\n", 944 | " {\n", 945 | " \"op\": \"replace\",\n", 946 | " \"path\": \"/parameters\",\n", 947 | " \"value\": {\n", 948 | " \"custom_metrics_provider_id\": integrated_system_id,\n", 949 | " \"custom_metrics_wait_time\": custom_metrics_wait_time,\n", 950 | " \"enable_custom_metric_runs\": True\n", 951 | " }\n", 952 | " }\n", 953 | " ]\n", 954 | " if max_records is not None:\n", 955 | " payload[0][\"value\"][\"max_records\"] = max_records \n", 956 | " if min_records is not None:\n", 957 | " payload[0][\"value\"][\"min_records\"] = min_records \n", 958 | " \n", 959 | " response = wos_client.monitor_instances.update(custom_monitor_instance_id, payload, update_metadata_only = True)\n", 960 | " result = response.result\n", 961 | " return result\n", 962 | "\n", 963 | "def create_custom_monitor_instance(custom_monitor_id):\n", 964 | " # Check if an custom monitor instance already exists\n", 965 | " existing_monitor_instance = get_custom_monitor_instance(custom_monitor_id)\n", 966 | "\n", 967 | " # If it does not exist, then create one\n", 968 | " if existing_monitor_instance is None:\n", 969 | " target = Target(\n", 970 | " target_type=TargetTypes.SUBSCRIPTION,\n", 971 | " target_id=subscription_id\n", 972 | " )\n", 973 | " parameters = {\n", 974 | " \"custom_metrics_provider_id\": integrated_system_id,\n", 975 | " \"custom_metrics_wait_time\": custom_metrics_wait_time,\n", 976 | " \"enable_custom_metric_runs\": True\n", 977 | " }\n", 978 | " # create the custom monitor instance id here.\n", 979 | " custom_monitor_instance_details = wos_client.monitor_instances.create(\n", 980 | " data_mart_id=WOS_GUID,\n", 981 | " background_mode=False,\n", 982 | " monitor_definition_id=custom_monitor_id,\n", 983 | " target=target,\n", 984 | " parameters=parameters\n", 985 | " ).result\n", 986 | " else:\n", 987 | " # otherwise, update the existing one with latest integrated system details.\n", 988 | " instance_id = existing_monitor_instance.metadata.id\n", 989 | " custom_monitor_instance_details = update_custom_monitor_instance(instance_id)\n", 990 | " return custom_monitor_instance_details\n", 991 | "\n", 992 | "monitor_instance_details = create_custom_monitor_instance(custom_monitor_id)\n", 993 | "custom_monitor_instance_id = monitor_instance_details.metadata.id\n", 994 | "\n", 995 | "def update_monitor_definition_id_in_integrated_systems(custom_monitor_id):\n", 996 | " payload = [\n", 997 | " {\n", 998 | " \"op\": \"add\",\n", 999 | " \"path\": \"/parameters\",\n", 1000 | " \"value\": {\n", 1001 | " \"monitor_definition_ids\": [ custom_monitor_id ]\n", 1002 | " }\n", 1003 | " }\n", 1004 | " ]\n", 1005 | " response = wos_client.integrated_systems.update(integrated_system_id, payload)\n", 1006 | " result = response.result\n", 1007 | " return result\n", 1008 | "\n", 1009 | "response = update_monitor_definition_id_in_integrated_systems(custom_monitor_id)" 1010 | ] 1011 | }, 1012 | { 1013 | "cell_type": "markdown", 1014 | "metadata": { 1015 | "id": "661eb24c1ee24279bc991f3eedc9b34e" 1016 | }, 1017 | "source": [ 1018 | "### Execute the monitor" 1019 | ] 1020 | }, 1021 | { 1022 | "cell_type": "code", 1023 | "execution_count": 18, 1024 | "metadata": { 1025 | "id": "f607dc1456464ad49d92cdf2598ed94c" 1026 | }, 1027 | "outputs": [ 1028 | { 1029 | "name": "stdout", 1030 | "output_type": "stream", 1031 | "text": [ 1032 | "\n", 1033 | "\n", 1034 | "========================================================================\n", 1035 | "\n", 1036 | " Waiting for end of monitoring run 975dfd43-2f6d-4d12-8440-f253fbcd716b \n", 1037 | "\n", 1038 | "========================================================================\n", 1039 | "\n", 1040 | "\n", 1041 | "\n", 1042 | "running......\n", 1043 | "finished\n", 1044 | "\n", 1045 | "---------------------------\n", 1046 | " Successfully finished run \n", 1047 | "---------------------------\n", 1048 | "\n", 1049 | "\n" 1050 | ] 1051 | } 1052 | ], 1053 | "source": [ 1054 | "#Execute the custom metrics provider deployment\n", 1055 | "monitor_instance_run_info = wos_client.monitor_instances.run(\n", 1056 | " background_mode=False,\n", 1057 | " monitor_instance_id=custom_monitor_instance_id\n", 1058 | " ).result\n", 1059 | "\n", 1060 | "monitor_instance_run_info\n", 1061 | "custom_monitor_run_id = monitor_instance_run_info.metadata.id" 1062 | ] 1063 | }, 1064 | { 1065 | "cell_type": "markdown", 1066 | "metadata": { 1067 | "id": "78869e90a0864f8b86fec50e4bcbcf84" 1068 | }, 1069 | "source": [ 1070 | "### Display the results" 1071 | ] 1072 | }, 1073 | { 1074 | "cell_type": "code", 1075 | "execution_count": 19, 1076 | "metadata": { 1077 | "id": "4151071a379049d28f7dd4f43ba94f14" 1078 | }, 1079 | "outputs": [ 1080 | { 1081 | "data": { 1082 | "text/html": [ 1083 | "\n", 1084 | " \n", 1085 | "

cd09f5db-409b-4244-bcae-25a5123c55b5 Monitor Runs Metrics from: 2023-08-18 15:40:52.666734 till: 2023-08-25 15:40:52.666751

\n", 1086 | " \n", 1087 | " \n", 1088 | " \n", 1089 | "
tsidmeasurement_idvaluelower_limitupper_limittagsmonitor_definition_idmonitor_instance_idrun_idtarget_typetarget_id
2023-08-25 15:40:13.212362+00:00human4b719e76-a726-4a90-b896-3250643b5c6317.1320.050.0[]aidc_demo_monitorcd09f5db-409b-4244-bcae-25a5123c55b5975dfd43-2f6d-4d12-8440-f253fbcd716bsubscription40c1fef9-db6e-4dfb-9eb8-5b4b275b0cb8
2023-08-25 15:40:13.212362+00:00roi4b719e76-a726-4a90-b896-3250643b5c63433.19521674406684100.01000.0[]aidc_demo_monitorcd09f5db-409b-4244-bcae-25a5123c55b5975dfd43-2f6d-4d12-8440-f253fbcd716bsubscription40c1fef9-db6e-4dfb-9eb8-5b4b275b0cb8
2023-08-25 15:40:13.212362+00:00impact4b719e76-a726-4a90-b896-3250643b5c63383.010.01000.0[]aidc_demo_monitorcd09f5db-409b-4244-bcae-25a5123c55b5975dfd43-2f6d-4d12-8440-f253fbcd716bsubscription40c1fef9-db6e-4dfb-9eb8-5b4b275b0cb8
2023-08-25 15:40:13.212362+00:00ml4b719e76-a726-4a90-b896-3250643b5c6382.8740.080.0[]aidc_demo_monitorcd09f5db-409b-4244-bcae-25a5123c55b5975dfd43-2f6d-4d12-8440-f253fbcd716bsubscription40c1fef9-db6e-4dfb-9eb8-5b4b275b0cb8
2023-08-25 15:40:13.212362+00:00decisioncost4b719e76-a726-4a90-b896-3250643b5c63-0.04486-1.01.0[]aidc_demo_monitorcd09f5db-409b-4244-bcae-25a5123c55b5975dfd43-2f6d-4d12-8440-f253fbcd716bsubscription40c1fef9-db6e-4dfb-9eb8-5b4b275b0cb8
2023-08-25 15:40:13.212362+00:00performance4b719e76-a726-4a90-b896-3250643b5c63382.910.01000.0[]aidc_demo_monitorcd09f5db-409b-4244-bcae-25a5123c55b5975dfd43-2f6d-4d12-8440-f253fbcd716bsubscription40c1fef9-db6e-4dfb-9eb8-5b4b275b0cb8
2023-08-25 15:31:38.653505+00:00human9a9071a0-c13d-4f44-97c7-a2ccd8463a530.020.050.0[]aidc_demo_monitorcd09f5db-409b-4244-bcae-25a5123c55b56199a8ed-47c4-4383-b831-9b6364ee5f5bsubscription40c1fef9-db6e-4dfb-9eb8-5b4b275b0cb8
2023-08-25 15:31:38.653505+00:00roi9a9071a0-c13d-4f44-97c7-a2ccd8463a530.0100.01000.0[]aidc_demo_monitorcd09f5db-409b-4244-bcae-25a5123c55b56199a8ed-47c4-4383-b831-9b6364ee5f5bsubscription40c1fef9-db6e-4dfb-9eb8-5b4b275b0cb8
2023-08-25 15:31:38.653505+00:00impact9a9071a0-c13d-4f44-97c7-a2ccd8463a530.010.01000.0[]aidc_demo_monitorcd09f5db-409b-4244-bcae-25a5123c55b56199a8ed-47c4-4383-b831-9b6364ee5f5bsubscription40c1fef9-db6e-4dfb-9eb8-5b4b275b0cb8
2023-08-25 15:31:38.653505+00:00ml9a9071a0-c13d-4f44-97c7-a2ccd8463a530.040.080.0[]aidc_demo_monitorcd09f5db-409b-4244-bcae-25a5123c55b56199a8ed-47c4-4383-b831-9b6364ee5f5bsubscription40c1fef9-db6e-4dfb-9eb8-5b4b275b0cb8
\n", 1090 | " \n", 1091 | " " 1092 | ], 1093 | "text/plain": [ 1094 | "" 1095 | ] 1096 | }, 1097 | "metadata": {}, 1098 | "output_type": "display_data" 1099 | }, 1100 | { 1101 | "name": "stdout", 1102 | "output_type": "stream", 1103 | "text": [ 1104 | "Note: First 10 records were displayed.\n" 1105 | ] 1106 | } 1107 | ], 1108 | "source": [ 1109 | "#Display the results\n", 1110 | "wos_client.monitor_instances.show_metrics(monitor_instance_id=custom_monitor_instance_id)" 1111 | ] 1112 | } 1113 | ], 1114 | "metadata": { 1115 | "kernelspec": { 1116 | "display_name": "Python 3.10", 1117 | "language": "python", 1118 | "name": "python3" 1119 | }, 1120 | "language_info": { 1121 | "codemirror_mode": { 1122 | "name": "ipython", 1123 | "version": 3 1124 | }, 1125 | "file_extension": ".py", 1126 | "mimetype": "text/x-python", 1127 | "name": "python", 1128 | "nbconvert_exporter": "python", 1129 | "pygments_lexer": "ipython3", 1130 | "version": "3.10.10" 1131 | } 1132 | }, 1133 | "nbformat": 4, 1134 | "nbformat_minor": 1 1135 | } 1136 | --------------------------------------------------------------------------------