├── .devcontainer ├── Dockerfile ├── devcontainer.json └── requirements.txt ├── .github ├── dependabot.yml ├── docs │ ├── getting-started.md │ ├── images │ │ ├── overview-01.png │ │ ├── overview-02.png │ │ ├── setup-01.png │ │ ├── setup-02.png │ │ ├── setup-03.png │ │ ├── setup-04.png │ │ ├── setup-05.png │ │ ├── setup-06.png │ │ ├── setup-07.png │ │ ├── setup-08.png │ │ ├── setup-09.png │ │ ├── setup-10.png │ │ ├── setup-11.png │ │ ├── setup-12.png │ │ ├── setup-13.png │ │ ├── setup-14.png │ │ ├── setup-15.png │ │ └── setup-16.png │ └── step-by-step-setup.md ├── templates │ ├── deploy-databricks-bundle │ │ └── action.yml │ ├── deploy-to-container-app │ │ └── action.yml │ └── deploy-to-kubernetes │ │ └── action.yml └── workflows │ ├── deploy-container-app.yml │ ├── deploy-infrastructure.yml │ └── deploy-kubernetes.yml ├── .gitignore ├── LICENSE ├── README.md ├── app ├── Dockerfile ├── main.py ├── model.py ├── requirements.txt └── sample-request.json ├── databricks ├── data │ ├── curated.csv │ └── inference.csv ├── databricks.yml ├── resources │ └── train_register_model.yml └── src │ ├── 00-create-external-table.ipynb │ ├── 01-train-model.ipynb │ └── 02-register-model.ipynb ├── infrastructure ├── main.bicep └── modules │ ├── application-insights.bicep │ ├── container-app-environment.bicep │ ├── container-registry.bicep │ ├── databricks.bicep │ ├── kubernetes-service.bicep │ ├── log-analytics-workspace.bicep │ ├── storage-account.bicep │ └── user-assigned-identity.bicep └── kubernetes └── manifest.yml /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/devcontainers/python:1-3.10-bullseye 2 | 3 | # # Install Databricks CLI 4 | RUN curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sudo sh 5 | 6 | # Copy Python dependencies 7 | COPY ./requirements.txt ./requirements.txt 8 | 9 | # Install Python dependencies 10 | RUN pip install --no-cache-dir --upgrade pip && \ 11 | pip install --no-cache-dir -r ./requirements.txt 12 | 13 | # Install Azure ClI 14 | RUN curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash && az bicep install 15 | 16 | # Connect as root non-root user. More info: https://aka.ms/dev-containers-non-root. 17 | USER vscode 18 | 19 | # Add zsh-autosuggestions 20 | RUN git clone https://github.com/zsh-users/zsh-autosuggestions ~/.zsh/zsh-autosuggestions 21 | RUN printf 'source ~/.zsh/zsh-autosuggestions/zsh-autosuggestions.zsh\n' >> ~/.zshrc 22 | 23 | # Add zsh-syntax-highlighting 24 | RUN git clone https://github.com/zsh-users/zsh-syntax-highlighting.git ~/.zsh/zsh-syntax-highlighting 25 | RUN printf 'source ~/.zsh/zsh-syntax-highlighting/zsh-syntax-highlighting.zsh\n' >> ~/.zshrc 26 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json. For config options, see the 2 | // README at: https://github.com/devcontainers/templates/tree/main/src/python 3 | { 4 | "name": "azure-databricks-containers-mlops", 5 | // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile 6 | "build": { 7 | // Path is relative to the devcontainer.json file. 8 | "dockerfile": "Dockerfile" 9 | }, 10 | // Features to add to the dev container. More info: https://containers.dev/features. 11 | "features": { 12 | "ghcr.io/devcontainers/features/docker-in-docker:2.10.2": {}, 13 | "ghcr.io/devcontainers/features/github-cli:1": {}, 14 | "ghcr.io/devcontainers-contrib/features/zsh-plugins:0": {} 15 | }, 16 | // Use 'forwardPorts' to make a list of ports inside the container available locally. 17 | "forwardPorts": [ 18 | 5000 19 | ], 20 | // Use 'postCreateCommand' to run commands after the container is created. 21 | // "postCreateCommand": "", 22 | // Configure tool-specific properties. 23 | "customizations": { 24 | "vscode": { 25 | "extensions": [ 26 | "ms-azuretools.vscode-docker", 27 | "charliermarsh.ruff", 28 | "ms-vscode.vscode-node-azure-pack", 29 | "ms-vscode.azurecli", 30 | "ms-azuretools.vscode-bicep", 31 | "ms-vscode-remote.remote-containers", 32 | "databricks.databricks", 33 | "github.vscode-github-actions", 34 | "redhat.vscode-yaml" 35 | ] 36 | }, 37 | "settings": { 38 | "editor.autoClosingBrackets": "always", 39 | "editor.codeActionsOnSave": { 40 | "source.organizeImports": "explicit" 41 | }, 42 | "editor.formatOnPaste": true, 43 | "editor.formatOnSave": true, 44 | "editor.inlineSuggest.enabled": true, 45 | "files.autoSave": "afterDelay", 46 | "git.autofetch": true, 47 | "github.copilot.enable": { 48 | "*": true 49 | }, 50 | "notebook.formatOnSave.enabled": true, 51 | "notebook.codeActionsOnSave": { 52 | "notebook.source.fixAll": "explicit", 53 | "notebook.source.organizeImports": "explicit" 54 | }, 55 | "terminal.integrated.defaultProfile.linux": "zsh", 56 | "terminal.integrated.profiles.linux": { 57 | "zsh": { 58 | "path": "/usr/bin/zsh" 59 | } 60 | }, 61 | "[python]": { 62 | "editor.formatOnSave": true, 63 | "editor.codeActionsOnSave": { 64 | "source.fixAll": "explicit", 65 | "source.organizeImports": "explicit" 66 | }, 67 | "editor.defaultFormatter": "charliermarsh.ruff" 68 | } 69 | } 70 | }, 71 | // You can use the mounts property to persist the user profile (to keep things like shell history). 72 | "mounts": [ 73 | "source=profile,target=/root,type=volume", 74 | "target=/root/.vscode-server,type=volume" 75 | ] 76 | // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. 77 | // "remoteUser": "root" 78 | } -------------------------------------------------------------------------------- /.devcontainer/requirements.txt: -------------------------------------------------------------------------------- 1 | # API 2 | fastapi==0.110.2 3 | uvicorn==0.29.0 4 | 5 | # Model 6 | alibi-detect==0.12.0 7 | configparser==5.2.0 8 | mlflow==2.10.0 9 | psutil==5.9.0 10 | cloudpickle==2.0.0 11 | joblib==1.2.0 12 | numpy==1.23.5 13 | pandas==1.5.3 14 | scikit-learn==1.1.1 15 | typing-extensions==4.8.0 -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for more information: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | # https://containers.dev/guide/dependabot 6 | 7 | version: 2 8 | updates: 9 | 10 | - package-ecosystem: "devcontainers" 11 | directory: ".devcontainer/" 12 | schedule: 13 | interval: weekly 14 | 15 | - package-ecosystem: "docker" 16 | directory: "/" 17 | schedule: 18 | interval: weekly 19 | 20 | - package-ecosystem: "github-actions" 21 | directory: "/" 22 | schedule: 23 | interval: "weekly" 24 | 25 | - package-ecosystem: "pip" 26 | directory: "/" 27 | schedule: 28 | interval: "weekly" 29 | -------------------------------------------------------------------------------- /.github/docs/getting-started.md: -------------------------------------------------------------------------------- 1 | # Getting Started 2 | 3 | The purpose of this section is to provide an overview of each example scenario. 4 | 5 | ## Potential Use Cases 6 | 7 | This approach is best suited for: 8 | 9 | - Low-latency and interactive workloads 10 | - Exposing machine learning models as a REST API to integrate with external applications 11 | - Preference to deploy models as containers on infrastructure other than Azure Databricks. 12 | - Need to use machine learning models in remote locations or edge devices with limited connectivity to Azure Databricks. 13 | 14 | > [!NOTE] 15 | > 16 | > - Databricks Model Serving is a Databricks-native appraoch to deploying models. This appraoch is less flexible than deploying models as containers, but it is easier to use and manage. 17 | > - This approach is not recommended for batch inference scenarios. 18 | 19 | ## Solution Design 20 | 21 | The below diagram shows a high-level design for implementing online scoring workloads suitable for classical machine learning scenarios using Azure Machine Learning. 22 | 23 | ![Solution Design](./images/overview-01.png) 24 | 25 | The solution consists of the following services: 26 | 27 | - **[Databricks](https://learn.microsoft.com/azure/databricks/introduction/)**: Used for training and registering the model. 28 | - **[Container Registry](https://learn.microsoft.com/azure/container-registry/container-registry-intro)**: Used for storing the Docker image. 29 | - **[Container App Environment](https://learn.microsoft.com/azure/container-apps/environment)**: Used for hosting Container Apps. 30 | - **[Container App](https://learn.microsoft.com/azure/container-apps/containers)**: Used for exposing the container as a REST API. 31 | - **[Kubernetes Service](https://learn.microsoft.com/azure/aks/what-is-aks)**: Used for exposing the container as a REST API. 32 | - **[Log Analytics Workspace](https://learn.microsoft.com/azure/azure-monitor/logs/log-analytics-overview)**: Used for monitoring the container app. 33 | 34 | The typical workflow for this solution is as follows: 35 | 36 | 1. Train and register a model using Databricks using Databricks Asset Bundles. Model metrics will be logged in the MLflow tracking server. Model artifacts will be stored and versioned in the Databricks workspace model registry. 37 | 2. Download the model artifacts from the Databricks workspace model registry and package them into a Docker image. The Docker image will contain the model, the scoring script, and any dependencies required for the model to be consumed. The Docker image will be pushed to the Azure Container Registry. 38 | 3. Deploy the Docker image as a Container App in the Azure Container App Environment or a service on an Azure Kubernetes cluster. The service will expose the model as a REST API. 39 | 4. Review the logs and metrics in the Log Analytics workspace. This can be used to monitor the performance of the model and the application. 40 | 41 | > [!CAUTION] 42 | > This solution design is intended for proof-of-concept scenarios and is not recommended for enterprise production scenarios. It is advised to review and adjust the design based on your specific requirements if you plan to use this in a production environment. This could include: 43 | > 44 | > - Securing the solution through network controls. 45 | > - Upflift observability by enabling monitoring and logging for different services. 46 | > - Defining an operational support and lifecycle management plan for the solution. 47 | > - Implementing alerting and notification mechanisms to notify support teams of any issues (e.g. performance, budget, etc.). 48 | > 49 | > The Azure Well-Architected Framework provides guidance on best practices for designing, building, and maintaining cloud solutions. For more information, see the [Azure Well-Architected Framework](https://learn.microsoft.com/azure/well-architected/what-is-well-architected-framework). 50 | 51 | ## Automated Deployment Workflow 52 | 53 | The below diagram shows the overall CI/CD process as built with GitHub Actions. This approach consists of three environments consisting of an identical set of resources. 54 | 55 | ![Automated Deployment](./images/overview-02.png) 56 | 57 | The environments include: 58 | 59 | - **Development:** used by developers to build and test their solutions. 60 | - **Staging:** used to test deployments before going to production in a production-like environment. Any integration tests are run in this environment. 61 | - **Production:** used for the final production environment. 62 | 63 | The end-to-end workflow operation consists of: 64 | 65 | 1. Triggering the `Model Build` workflow based on an event (e.g. merging code to the main branch via a pull request). This will trigger a job in the Databricks workspace that will result in a new model being registered. 66 | 2. Triggering the `Containerize Model` workflow on completion of the `Model Build` workflow. This will result in a new Docker image being built and pushed to the Azure Container Registry. 67 | 3. Triggering the `Deploy to Staging` workflow on completion of the `Containerize Model` workflow. This will result in the new Docker image being deployed as a Container App or as a service on an Azure Kubernetes cluster in the staging environment. 68 | 4. Triggering the `Smoke Test` workflow on completion of the `Deploy to Staging` workflow. This will run a test to ensure the deployment is successful. 69 | 5. The `Deploy to Production` workflow will only be triggered after the `Deploy to Staging` workflow has completed successfully, the current branch is set to `main` (reflecting the production state of the codebase), and the user has approved the deployment. 70 | 71 | > [!NOTE] 72 | > The current workflows are designed to be triggered manually for ease of demonstration. However, they can be modified to be triggered automatically based on specific events. 73 | > 74 | > After developers have tested their code in the development environment, they can merge their code to the main branch via a pull request. This will trigger steps (1) to (4) automatically, but will not deploy to the production environment. This will validate that the code is working as expected in the staging environment, and allow the team to make adjustments if needed. 75 | > 76 | > The GitHub Action workflow `on` trigger will contain the following block to ensure it is triggered only when pull requests into the main branch are opened: 77 | > 78 | >```yml 79 | >on: 80 | > pull_request: 81 | > types: 82 | > - opened 83 | > - synchronize 84 | > branches: 85 | > - main 86 | > ``` 87 | > 88 | > Once the team is ready to deploy to production, the pull request can be approved and merged into the main branch. This will trigger steps (1) to (5) as described above to deploy the code to the production environment. 89 | > 90 | > The GitHub Action workflow `on` trigger will contain the following block to ensure it is triggered only when new code is pushed into the main branch: 91 | > 92 | >```yml 93 | >on: 94 | > push: 95 | > branches: 96 | > - main 97 | > ``` 98 | 99 | ## Related Resources 100 | 101 | - [MLOps workflows on Databricks](https://docs.databricks.com/en/machine-learning/mlops/mlops-workflow.html) 102 | - [What are Databricks Asset Bundles?](https://docs.databricks.com/en/dev-tools/bundles/index.html) 103 | - [What is FAST API?](https://fastapi.tiangolo.com) 104 | -------------------------------------------------------------------------------- /.github/docs/images/overview-01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/overview-01.png -------------------------------------------------------------------------------- /.github/docs/images/overview-02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/overview-02.png -------------------------------------------------------------------------------- /.github/docs/images/setup-01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/setup-01.png -------------------------------------------------------------------------------- /.github/docs/images/setup-02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/setup-02.png -------------------------------------------------------------------------------- /.github/docs/images/setup-03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/setup-03.png -------------------------------------------------------------------------------- /.github/docs/images/setup-04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/setup-04.png -------------------------------------------------------------------------------- /.github/docs/images/setup-05.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/setup-05.png -------------------------------------------------------------------------------- /.github/docs/images/setup-06.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/setup-06.png -------------------------------------------------------------------------------- /.github/docs/images/setup-07.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/setup-07.png -------------------------------------------------------------------------------- /.github/docs/images/setup-08.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/setup-08.png -------------------------------------------------------------------------------- /.github/docs/images/setup-09.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/setup-09.png -------------------------------------------------------------------------------- /.github/docs/images/setup-10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/setup-10.png -------------------------------------------------------------------------------- /.github/docs/images/setup-11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/setup-11.png -------------------------------------------------------------------------------- /.github/docs/images/setup-12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/setup-12.png -------------------------------------------------------------------------------- /.github/docs/images/setup-13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/setup-13.png -------------------------------------------------------------------------------- /.github/docs/images/setup-14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/setup-14.png -------------------------------------------------------------------------------- /.github/docs/images/setup-15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/setup-15.png -------------------------------------------------------------------------------- /.github/docs/images/setup-16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nfmoore/azure-databricks-containers-mlops-example-scenarios/9a54d7cb0d403262e434cc992d013464a4644dd7/.github/docs/images/setup-16.png -------------------------------------------------------------------------------- /.github/docs/step-by-step-setup.md: -------------------------------------------------------------------------------- 1 | # Step-by-Step Setup 2 | 3 | The purpose of this section is to describe the steps required to setup each example scenario. 4 | 5 | > [!TIP] 6 | > 7 | > - The following options are recommended to complete the setup: 8 | > 1. Using the [Azure Cloud Shell](https://learn.microsoft.com/azure/cloud-shell/overview) within the Azure Portal. 9 | > 2. Using a [GitHub Codespace](https://docs.github.com/en/codespaces/prebuilding-your-codespaces/about-github-codespaces-prebuilds) 10 | > 3. Using your local VSCode environment with the environment specified in the [development container](https://docs.github.com/en/codespaces/setting-up-your-project-for-codespaces/adding-a-dev-container-configuration/introduction-to-dev-containers). This will be the most efficient way to complete the setup. 11 | > - The example scenarios are designed to be executed in sequence. 12 | 13 | > [!WARNING] 14 | > 15 | > - As with all Azure Deployments, this will incur associated costs. Remember to teardown all related resources after use to avoid unnecessary costs. 16 | > - If you use the Azure Cloud Shell you may need to update the version of the GitHub CLI. 17 | > - If you use a GitHub Codespace you may encounter HTTP 403 errors when executing GitHub CLI commands. If you encounter these issues try executing `export GITHUB_TOKEN=` to overwrite the GITHUB_TOKEN environment variable then execute `gh auth login`. Codespaces also uses GITHUB_TOKEN, but the token used is less permissive. 18 | 19 | > [!NOTE] 20 | > 21 | > - Common setup section should take 25 minutes to complete. 22 | > - Example scenarios should take 35 minutes to complete. 23 | 24 | ## Prerequisites 25 | 26 | Before implementing this example scenario the following is needed: 27 | 28 | - Azure subscription with Owner permissions. 29 | - GitHub account. 30 | 31 | ## 1. Common Setup 32 | 33 | ## 1.1. Create a GitHub repository 34 | 35 | 1. Log in to your GitHub account and navigate to the [azure-databricks-containers-mlops-example-scenarios](https://github.com/nfmoore/azure-databricks-containers-mlops-example-scenarios) repository and click `Use this template` to create a new repository from this template. 36 | 37 | Rename the template and leave it public. Ensure you click `Include all branches` to copy all branches. 38 | 39 | > [!NOTE] 40 | > 41 | > - You can learn more about creating a repository from a template [here](https://docs.github.com/en/repositories/creating-and-managing-repositories/creating-a-template-repository). 42 | 43 | ## 1.2. Configure a federated identity credential on a service principal 44 | 45 | 1. Create a Microsoft Entra application by executing the following command: 46 | 47 | ```bash 48 | az ad app create --display-name 49 | ``` 50 | 51 | Take note of the `appId` value (the Application ID or Client ID) returned by the command as it will be used in the next step. 52 | 53 | 2. Create a Microsoft Entra service principal by executing the following command: 54 | 55 | ```bash 56 | az ad sp create --id 57 | ``` 58 | 59 | Take note of the `id` value (the Object ID or Principal ID) returned by the command as it will be used in the next step. 60 | 61 | 3. Assign the service principal as a `Owner` of an Azure subscription by executing the following command: 62 | 63 | ```bash 64 | az role assignment create \ 65 | --role "Owner" \ 66 | --assignee-object-id \ 67 | --assignee-principal-type ServicePrincipal \ 68 | --scope /subscriptions/ 69 | ``` 70 | 71 | 4. Create federated credentials for your Microsoft Entra application by executing the following command: 72 | 73 | ```bash 74 | # set environment variables for the federated credentials 75 | export APPLICATION_ID= # replace with your application id 76 | export REPOSITORY_NAME=example-scenarios-databricks-containers-mlops # replace with your repository name 77 | export OWNER=# replace with your GitHub username 78 | 79 | # create federated credential for workflows targeting the Production environment 80 | az ad app federated-credential create \ 81 | --id $APPLICATION_ID \ 82 | --parameters '{ 83 | "name": "ProductionEnvironmentCredential", 84 | "issuer": "https://token.actions.githubusercontent.com", 85 | "subject": "repo:'${OWNER}'/'${REPOSITORY_NAME}':environment:Production", 86 | "description": "Federated credential for workflows in the Production environment", 87 | "audiences": [ "api://AzureADTokenExchange" ] 88 | }' 89 | 90 | # create federated credential for jobs tied to the Staging environment 91 | az ad app federated-credential create \ 92 | --id $APPLICATION_ID \ 93 | --parameters '{ 94 | "name": "StagingEnvironmentCredential", 95 | "issuer": "https://token.actions.githubusercontent.com", 96 | "subject": "repo:'${OWNER}'/'${REPOSITORY_NAME}':environment:Staging", 97 | "description": "Federated credential for jobs tied to the Staging environment", 98 | "audiences": [ "api://AzureADTokenExchange" ] 99 | }' 100 | 101 | # create federated credential for jobs tied to the main branch 102 | az ad app federated-credential create \ 103 | --id $APPLICATION_ID \ 104 | --parameters '{ 105 | "name": "MainBranchCredential", 106 | "issuer": "https://token.actions.githubusercontent.com", 107 | "subject": "repo:'${OWNER}'/'${REPOSITORY_NAME}':ref:refs/heads/main", 108 | "description": "Federated credential for jobs tied to the main branch", 109 | "audiences": [ "api://AzureADTokenExchange" ] 110 | }' 111 | ``` 112 | 113 | After executing these steps you will have a federated identity credential on a service principal that can be used to authenticate with Azure services. This can be viewed in Microsoft Entra by navigating to the `App registrations` blade and selecting the application created in step 1. 114 | 115 | ![Federated Credential in Microsoft Entra](./images/setup-01.png) 116 | 117 | > [!NOTE] 118 | > 119 | > - Ensure note of the Client ID, Tenant ID and Subscription ID as they will be used in the next steps. 120 | > - More information about setting up an Azure Login with OpenID Connect and use it in a GitHub Actions workflow is available [here](https://learn.microsoft.com/azure/developer/github/connect-from-azure?tabs=azure-cli). 121 | 122 | ## 1.3. Configure GitHub repository secrets, variables, and environments 123 | 124 | **Method 1: GitHub CLI**: 125 | 126 | 1. Create the following GitHub Actions repository secrets by executing the following command: 127 | 128 | ```bash 129 | # optional - used to authenticate with your GitHub account 130 | gh auth login 131 | 132 | # set environment variables for GitHub repository secrets 133 | export AZURE_CLIENT_ID= 134 | export AZURE_TENANT_ID= 135 | export AZURE_SUBSCRIPTION_ID= 136 | 137 | # set GitHub repository secrets 138 | gh secret set AZURE_CLIENT_ID --body "$AZURE_CLIENT_ID" 139 | gh secret set AZURE_TENANT_ID --body "$AZURE_TENANT_ID" 140 | gh secret set AZURE_SUBSCRIPTION_ID --body "$AZURE_SUBSCRIPTION_ID" 141 | ``` 142 | 143 | 2. Create the following GitHub Actions repository variables by executing the following command: 144 | 145 | ```bash 146 | # optional - used to authenticate with your GitHub account 147 | gh auth login 148 | 149 | # set environment variables for GitHub repository variables 150 | export DEPLOYMENT_LOCATION= # region to deploy resources e.g. australiaeast 151 | export BASE_NAME=example-scenarios-databricks-containers-mlops # set for convenience 152 | export DEPLOYMENT_RESOURCE_GROUP_NAME=rg-$BASE_NAME-01 153 | export DEPLOYMENT_DATARBICKS_MANAGED_RESOURCE_GROUP_NAME=rgm-databricks-$BASE_NAME-01 154 | export DEPLOYMENT_KUBERNETES_MANAGED_RESOURCE_GROUP_NAME=rgm-kubernetes-$BASE_NAME-01 155 | export DEPLOY_CONTAINER_APPS=true # requred to deploy Azure Container Apps for the Container Apps scenario 156 | export DEPLOY_KUBERNETES=true # requred to deploy Azure Kubernetes Service for the Kubernetes Service scenario 157 | 158 | # set GitHub repository variables 159 | gh variable set DEPLOYMENT_LOCATION --body "$DEPLOYMENT_LOCATION" 160 | gh variable set DEPLOYMENT_RESOURCE_GROUP_NAME --body "$DEPLOYMENT_RESOURCE_GROUP_NAME" 161 | gh variable set DEPLOYMENT_DATARBICKS_MANAGED_RESOURCE_GROUP_NAME --body "$DEPLOYMENT_DATARBICKS_MANAGED_RESOURCE_GROUP_NAME" 162 | gh variable set DEPLOYMENT_KUBERNETES_MANAGED_RESOURCE_GROUP_NAME --body "$DEPLOYMENT_KUBERNETES_MANAGED_RESOURCE_GROUP_NAME" 163 | gh variable set DEPLOY_KUBERNETES --body true 164 | gh variable set DEPLOY_CONTAINER_APPS --body true 165 | ``` 166 | 167 | 3. Create the following GitHub Actions environments by executing the following command: 168 | 169 | ```bash 170 | # optional - used to authenticate with your GitHub account 171 | gh auth login 172 | 173 | # set environment variables for GitHub repository environments 174 | OWNER=$(gh api user | jq -r '.login') 175 | OWNER_ID=$(gh api user | jq -r '.id') 176 | REPOSITORY_NAME=$(gh repo view --json name | jq '.name' -r) 177 | 178 | # create the staging environment 179 | gh api \ 180 | --method PUT \ 181 | -H "Accept: application/vnd.github+json" \ 182 | -H "X-GitHub-Api-Version: 2022-11-28" \ 183 | /repos/$OWNER/$REPOSITORY_NAME/environments/Staging 184 | 185 | # create the production environment with a reviewer and a wait timer 186 | gh api \ 187 | --method PUT \ 188 | -H "Accept: application/vnd.github+json" \ 189 | -H "X-GitHub-Api-Version: 2022-11-28" \ 190 | /repos/$OWNER/$REPOSITORY_NAME/environments/Production \ 191 | -F "prevent_self_review=false" -f "reviewers[][type]=User" -F "reviewers[][id]=$OWNER_ID" 192 | ``` 193 | 194 | **Method 2: GitHub Actions UI**: 195 | 196 | 1. Create the following GitHub Actions repository secrets by: 197 | 1. Navigate to the GitHub repository. 198 | 2. Click on the `Settings` tab. 199 | 3. Click on the `Secrets and variables` link and selecting `Actions`. 200 | 4. Clicking on the `New repository secret` button. 201 | 5. Add the following secrets: 202 | - `AZURE_CLIENT_ID` with the value of the `Client ID` from the federated identity credential. 203 | - `AZURE_TENANT_ID` with the value of the `Tenant ID` from the federated identity credential. 204 | - `AZURE_SUBSCRIPTION_ID` with the value of the `Subscription ID` from the federated identity credential. 205 | 206 | 2. Create the following GitHub Actions repository variables by: 207 | 1. Navigate to the GitHub repository. 208 | 2. Click on the `Settings` tab. 209 | 3. Click on the `Secrets and variables` link and selecting `Actions`. 210 | 4. Clicking on `Variables` in the tab. 211 | 5. Clicking on the `New repository variable` button. 212 | 6. Add the following variables: 213 | - `DEPLOYMENT_LOCATION` with the value of the Azure region to deploy resources. 214 | - `DEPLOYMENT_RESOURCE_GROUP_NAME` with the value of the resource group name for the deployment. 215 | - `DEPLOYMENT_DATARBICKS_MANAGED_RESOURCE_GROUP_NAME` with the value of the resource group name for the Databricks managed resources. 216 | - `DEPLOYMENT_KUBERNETES_MANAGED_RESOURCE_GROUP_NAME` with the value of the resource group name for the Kubernetes managed resources. 217 | - `DEPLOY_CONTAINER_APPS` with the value of `true` to deploy Azure Container Apps for the Container Apps scenario. 218 | - `DEPLOY_KUBERNETES` with the value of `true` to deploy Azure Kubernetes Service for the Kubernetes Service scenario. 219 | 220 | 3. Create the following GitHub Actions environments by: 221 | 1. Navigate to the GitHub repository. 222 | 2. Click on the `Settings` tab. 223 | 3. Click on the `Environments` link. 224 | 4. Click on the `New environment` button. 225 | 5. Add the following environments: 226 | - `Staging` with no reviewers. 227 | - `Production` with a reviewer and a wait timer. 228 | 229 | After executing these steps you will have configured the GitHub repository with the required secrets, variables, and environments. 230 | 231 | ![GitHub Repository Secrets](./images/setup-02.png) 232 | 233 | ![GitHub Repository Variables](./images/setup-03.png) 234 | 235 | ![GitHub Repository Environments](./images/setup-04.png) 236 | 237 | > [!NOTE] 238 | > 239 | > - More information about using secrets in GitHub Actions is available [here](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions). 240 | > - More information about using variables in GitHub Actions is available [here](https://docs.github.com/en/actions/learn-github-actions/variables). 241 | > - More information about using environments for deployments in GitHub Actions is available [here](https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment). 242 | 243 | ## 1.4. Deploy Azure Resources 244 | 245 | Execute the `Deploy Azure Resources` workflow to deploy all Azure resources required for the example scenarios. 246 | 247 | To workflow can be executed via the following methods: 248 | 249 | **Method 1: GitHub CLI**: 250 | 251 | 1. Trigger the workflow via the GitHub CLI by executing the following command: 252 | 253 | ```bash 254 | # optional - used to authenticate with your GitHub account 255 | gh auth login 256 | 257 | # trigger the workflow 258 | gh workflow run "Deploy Azure Resources" 259 | ``` 260 | 261 | **Method 2: GitHub Actions UI**: 262 | 263 | 1. Manually trigger the workflow via the GitHub Actions UI by following these steps: 264 | 265 | 1. Navigate to the GitHub repository. 266 | 2. Click on the `Actions` tab. 267 | 3. Click on the `Deploy Azure Resources` workflow. 268 | 4. Click on the `Run workflow` button. 269 | 5. Click on the `Run workflow` button again to confirm the action. 270 | 271 | After executing these steps you will have deployed all Azure resources required for the example scenarios. 272 | 273 | ![GitHub Actions Workflow](./images/setup-05.png) 274 | 275 | ![Azure Resources](./images/setup-06.png) 276 | 277 | > [!NOTE] 278 | > 279 | > - The `Deploy Azure Resources` workflow is configured with a `workflow_dispatch` trigger (a manual process) for illistration purposes only. 280 | > - The service principal is added as an workspace administrator to the Databricks workspace. This same service principal will be used to authenticate with Azure Databricks to create different artefacts such as clusters, jobs, and notebooks. This is present in all GitHub Actions workflows in this repository. 281 | > - More information about CI/CD with GitHub Actions is available [here](https://docs.github.com/en/actions/learn-github-actions/understanding-github-actions). 282 | 283 | ## 2. Example Sceanrios 284 | 285 | ## 2.1. Azure Container Apps 286 | 287 | Execute the `Deploy to Container Apps` workflow to train a model, create a container image, deploy the image to an Azure Container App, and smoke test the deployed model. 288 | 289 | To workflow can be executed via the following methods: 290 | 291 | **Method 1: GitHub CLI**: 292 | 293 | 1. Trigger the workflow via the GitHub CLI by executing the following command: 294 | 295 | ```bash 296 | # optional - used to authenticate with your GitHub account 297 | gh auth login 298 | 299 | # trigger the workflow 300 | gh workflow run "Deploy to Container Apps" 301 | ``` 302 | 303 | **Method 2: GitHub Actions UI**: 304 | 305 | 1. Manually trigger the workflow via the GitHub Actions UI by following these steps: 306 | 307 | 1. Navigate to the GitHub repository. 308 | 2. Click on the `Actions` tab. 309 | 3. Click on the `Deploy to Container Apps` workflow. 310 | 4. Click on the `Run workflow` button. 311 | 5. Click on the `Run workflow` button again to confirm the action. 312 | 313 | After executing these steps the `Deploy to Container Apps` workflow will train a model, create a container image, deploy the image to an Azure Container App, and smoke test the deployed model. 314 | 315 | After the `Train Model` job completes you will have registered a model in the Databricks workspace. 316 | 317 | ![Databricks Workspace Job](./images/setup-07.png) 318 | 319 | ![Databricks Workspace Model](./images/setup-08.png) 320 | 321 | After the `Build Container` job and `Staging Deployment` job completes you will have deployed a container image to Azure Container Apps in the `Staging` environment. 322 | 323 | You will be prompted to review the deployment before deploying the container to the `Production` environment. Click the `Review deployments` button to give approval and commence the Production job. 324 | 325 | ![GitHub Actions Workflow](./images/setup-09.png) 326 | 327 | After the `Production Deployment` job completes you will have deployed a container image to Azure Container Apps in the `Production` environment. 328 | 329 | You can view the Container App in the Azure portal by clicking on the `Container App` resource. 330 | 331 | ![Container App in Azure Portal](./images/setup-10.png) 332 | 333 | By clicking `Application Url` you can view the Swagger UI for [FastAPI](https://fastapi.tiangolo.com) service. This page displays the API endpoints you can consume as part of the service. 334 | 335 | ![App Swagger UI](./images/setup-11.png) 336 | 337 | You can view logs for the Container App by clicking on the `Logs` tab. This will display the logs for the Container App. 338 | 339 | For example, you can view model related telemetry by execuring the following Kusto query: 340 | 341 | ```kql 342 | ContainerAppConsoleLogs_CL 343 | | where Log_s has 'credit-default-api' 344 | | extend Log_s = trim("INFO:root:", Log_s) 345 | | project TimeGenerated, Data=parse_json(tostring(Log_s)) 346 | | evaluate bag_unpack(Data) 347 | ``` 348 | 349 | ![Container App Logs](./images/setup-12.png) 350 | 351 | > [!IMPORTANT] 352 | > 353 | > - The `Deploy Azure Resources` workflow is a prerequisite for the `Deploy to Container Apps` workflow. 354 | 355 | > [!NOTE] 356 | > 357 | > - The `Deploy to Container Apps` workflow is configured with a `workflow_dispatch` trigger (a manual process) for illistration purposes only. 358 | > - More information about CI/CD with GitHub Actions is available [here](https://docs.github.com/en/actions/learn-github-actions/understanding-github-actions). 359 | 360 | ## 2.2. Azure Kubernetes Service 361 | 362 | Execute the `Deploy to Kubernetes Service` workflow to train a model, create a container image, deploy the image to Azure Kubernetes Service, and smoke test the deployed model. 363 | 364 | To workflow can be executed via the following methods: 365 | 366 | **Method 1: GitHub CLI**: 367 | 368 | 1. Trigger the workflow via the GitHub CLI by executing the following command: 369 | 370 | ```bash 371 | gh auth login # (optional - used to authenticate with your GitHub account) 372 | gh workflow run "Deploy to Kubernetes Service" 373 | ``` 374 | 375 | **Method 2: GitHub Actions UI**: 376 | 377 | 1. Manually trigger the workflow via the GitHub Actions UI by following these steps: 378 | 379 | 1. Navigate to the GitHub repository. 380 | 2. Click on the `Actions` tab. 381 | 3. Click on the `Deploy to Kubernetes Service` workflow. 382 | 4. Click on the `Run workflow` button. 383 | 5. Click on the `Run workflow` button again to confirm the action. 384 | 385 | After executing these steps the `Deploy to Kubernetes Service` workflow will train a model, create a container image, deploy the image to an Azure Kubernetes Service as a deployment, and smoke test the deployed model. 386 | 387 | After the `Train Model` job completes you will have registered a model in the Databricks workspace. 388 | 389 | ![Databricks Workspace Job](./images/setup-07.png) 390 | 391 | ![Databricks Workspace Model](./images/setup-08.png) 392 | 393 | After the `Build Container` job and `Staging Deployment` job completes you will have deployed a container image to Azure Kubernetes Service in the `Staging` environment. 394 | 395 | You will be prompted to review the deployment before deploying the container to the `Production` environment. Click the `Review deployments` button to give approval and commence the Production job. 396 | 397 | ![GitHub Actions Workflow](./images/setup-13.png) 398 | 399 | After the `Production Deployment` job completes you will have deployed a container image to Azure Kubernetes Service in the `Production` environment. 400 | 401 | You can view the Azure Kubernetes Service deployment in the Azure portal by clicking on the `Kubernetes service` resource. 402 | 403 | ![Kubernetes Service in Azure Portal](./images/setup-14.png) 404 | 405 | By selecting the `Workloads` tab, and clicking on the `External IP` for the `credit-default-api` service you can view the Swagger UI for [FastAPI](https://fastapi.tiangolo.com) service. This page displays the API endpoints you can consume as part of the service. 406 | 407 | ![Service Swagger UI](./images/setup-15.png) 408 | 409 | You can view logs for the service by clicking on the `Logs` tab. This will display the logs for the service. 410 | 411 | For example, you can view model related telemetry by execuring the following Kusto query: 412 | 413 | ```kql 414 | ContainerLog 415 | | where LogEntry has 'credit-default-api' 416 | | extend LogEntry = trim("INFO:root:", LogEntry) 417 | | project TimeGenerated, Data=parse_json(tostring(LogEntry)) 418 | | evaluate bag_unpack(Data) 419 | ``` 420 | 421 | ![Kubernetes Service Logs](./images/setup-16.png) 422 | 423 | > [!IMPORTANT] 424 | > 425 | > - The `Deploy Azure Resources` workflow is a prerequisite for the `Deploy to Kubernetes Service` workflow. 426 | 427 | > [!NOTE] 428 | > 429 | > - The `Deploy to Kubernetes Service` workflow is configured with a `workflow_dispatch` trigger (a manual process) for illistration purposes only. 430 | > - More information about CI/CD with GitHub Actions is available [here](https://docs.github.com/en/actions/learn-github-actions/understanding-github-actions). 431 | -------------------------------------------------------------------------------- /.github/templates/deploy-databricks-bundle/action.yml: -------------------------------------------------------------------------------- 1 | name: Deploy Databricks Bundle 2 | 3 | description: This workflow deploys a Databricks bundle to an Azure Databricks workspace. 4 | 5 | inputs: 6 | resource_group: 7 | description: The resource group where the container app is deployed. 8 | required: true 9 | 10 | bundle_config_file: 11 | description: The path to the Databricks bundle configuration file. 12 | required: false 13 | default: databricks/databricks.yml 14 | 15 | bundle_directory: 16 | description: The directory where the Databricks bundle is stored. 17 | required: false 18 | default: databricks 19 | 20 | runs: 21 | using: "composite" 22 | 23 | steps: 24 | 25 | # Add Databricks Azure CLI extension 26 | - name: Add Databricks extension 27 | uses: azure/cli@v2 28 | with: 29 | azcliversion: ${{ env.AZ_CLI_VERSION }} 30 | inlineScript: | 31 | # Add databricks extension 32 | az extension add --name databricks 33 | 34 | # Set Databricks host and token environment variables and MLFlow tracking URI 35 | - name: Set Databricks environment variables 36 | uses: azure/cli@v2 37 | with: 38 | azcliversion: ${{ env.AZ_CLI_VERSION }} 39 | inlineScript: | 40 | DATABRICKS_WORKSPACE_NAME=$(az resource list --resource-group ${{ inputs.resource_group }} \ 41 | | jq '.[] | select(.type == "Microsoft.Databricks/workspaces") | .name' -r) 42 | 43 | echo "DATABRICKS_HOST=https://$(az databricks workspace show --name $DATABRICKS_WORKSPACE_NAME \ 44 | --resource-group ${{ inputs.resource_group }} | jq '.workspaceUrl' -r)" >> $GITHUB_ENV 45 | 46 | echo "DATABRICKS_TOKEN=$(az account get-access-token \ 47 | --resource 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d | jq .accessToken -r)" >> $GITHUB_ENV 48 | 49 | echo "MLFLOW_TRACKING_URI=databricks" >> $GITHUB_ENV 50 | 51 | # Add host to databricks.yml 52 | - name: Add databricks host 53 | shell: bash 54 | run: | 55 | yq -i '.workspace.host = "'"${DATABRICKS_HOST}"'"' ${{ inputs.bundle_config_file }} 56 | cat ${{ inputs.bundle_config_file }} 57 | 58 | # Install the Databricks CLI 59 | - name: Install Databricks CLI 60 | shell: bash 61 | run: | 62 | curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh 63 | 64 | # Configure the Databricks CLI 65 | - name: Databricks CLI config 66 | shell: bash 67 | run: | 68 | cat > ~/.databrickscfg << EOF 69 | [DEFAULT] 70 | host = $DATABRICKS_HOST 71 | token = $DATABRICKS_TOKEN 72 | EOF 73 | 74 | # Deploy Databricks Bundle 75 | - name: Deploy bundle 76 | working-directory: ${{ inputs.bundle_directory }} 77 | shell: bash 78 | run: | 79 | databricks bundle validate 80 | databricks bundle deploy 81 | -------------------------------------------------------------------------------- /.github/templates/deploy-to-container-app/action.yml: -------------------------------------------------------------------------------- 1 | name: Deploy to Container App 2 | 3 | description: This workflow deploys a container to an Azure Container App. 4 | 5 | inputs: 6 | container_app_name: 7 | description: The name of the container app resource. 8 | required: true 9 | 10 | resource_group: 11 | description: The resource group where the container app is deployed. 12 | required: true 13 | 14 | container_app_environment_name: 15 | description: The name of the container app environment resource. 16 | required: true 17 | 18 | user_assigned_identity_name: 19 | description: The name of the user-assigned identity to use for the container app. 20 | required: true 21 | 22 | container_registry_hostname: 23 | description: The hostname of the container registry server. 24 | required: true 25 | 26 | container_image: 27 | description: The container image to deploy. 28 | required: true 29 | 30 | target_port: 31 | description: The port to expose on the container app. 32 | required: false 33 | default: "5000" 34 | 35 | revision_suffix: 36 | description: The suffix to append to the container app revision. 37 | required: false 38 | default: ${{ github.run_id }} 39 | 40 | artifacts_directory: 41 | description: The directory where the deployment artifacts are stored. 42 | required: false 43 | default: "artifacts" 44 | 45 | runs: 46 | using: "composite" 47 | 48 | steps: 49 | # Add Azure Container Apps CLI extension 50 | - name: Add Container Apps extension 51 | uses: azure/cli@v2 52 | with: 53 | azcliversion: ${{ env.AZ_CLI_VERSION }} 54 | inlineScript: | 55 | # Add container apps extension 56 | az extension add --name containerapp 57 | 58 | # Get id of the user-assigned identity 59 | - name: Get identity ID 60 | uses: azure/cli@v2 61 | with: 62 | azcliversion: ${{ env.AZ_CLI_VERSION }} 63 | inlineScript: | 64 | echo "USER_ASSIGNED_IDENTITY_ID=$(az identity show --resource-group ${{ inputs.resource_group }} \ 65 | --name ${{ inputs.user_assigned_identity_name }} | jq '.id' -r)" >> $GITHUB_ENV 66 | 67 | # Deploy the container app 68 | - name: Deploy container app 69 | uses: azure/cli@v2 70 | with: 71 | azcliversion: ${{ env.AZ_CLI_VERSION }} 72 | inlineScript: | 73 | # Create artifacts directory 74 | mkdir ${{ inputs.artifacts_directory }} 75 | 76 | # Check if container app exists 77 | if az containerapp show --name ${{ inputs.container_app_name }} --resource-group ${{ inputs.resource_group }} &> /dev/null; then 78 | echo "Container app exists. Executing code to update existing app..." 79 | az containerapp update \ 80 | --name ${{ inputs.container_app_name }} \ 81 | --resource-group ${{ inputs.resource_group }} \ 82 | --image ${{ inputs.container_registry_hostname }}/${{ inputs.container_image }} \ 83 | --revision-suffix ${{ inputs.revision_suffix }} > ${{ inputs.artifacts_directory }}/containerapp.json 84 | else 85 | echo "Container app does not exist. Executing code to create new app..." 86 | az containerapp create \ 87 | --name ${{ inputs.container_app_name }} \ 88 | --resource-group ${{ inputs.resource_group }} \ 89 | --environment ${{ inputs.container_app_environment_name }} \ 90 | --user-assigned $USER_ASSIGNED_IDENTITY_ID \ 91 | --registry-identity $USER_ASSIGNED_IDENTITY_ID \ 92 | --registry-server ${{ inputs.container_registry_hostname }} \ 93 | --image ${{ inputs.container_registry_hostname }}/${{ inputs.container_image }} \ 94 | --target-port ${{ inputs.target_port }} \ 95 | --ingress 'external' \ 96 | --revision-suffix ${{ inputs.revision_suffix }} > ${{ inputs.artifacts_directory }}/containerapp.json 97 | fi 98 | -------------------------------------------------------------------------------- /.github/templates/deploy-to-kubernetes/action.yml: -------------------------------------------------------------------------------- 1 | name: Deploy to Kubernetes Service 2 | 3 | description: This workflow deploys a container to an Azure Kubernetes Service. 4 | 5 | inputs: 6 | resource_group: 7 | description: The resource group where the container app is deployed. 8 | required: true 9 | 10 | environment_tag: 11 | description: The tag to identify the environment. 12 | required: true 13 | 14 | container_image: 15 | description: The container image to deploy. 16 | required: true 17 | 18 | kubernetes_manifest_file: 19 | description: The path to the Kubernetes manifest file. 20 | required: false 21 | default: kubernetes/manifest.yml 22 | 23 | runs: 24 | using: "composite" 25 | 26 | steps: 27 | # Set environment variables 28 | - name: Set resource names 29 | uses: azure/cli@v2 30 | with: 31 | azcliversion: ${{ env.AZ_CLI_VERSION }} 32 | inlineScript: | 33 | # Get container registry name 34 | echo "CONTAINER_REGISTRY_NAME=$(az resource list --resource-group ${{ inputs.resource_group }} \ 35 | | jq '.[] | select(.type == "Microsoft.ContainerRegistry/registries") | .name' -r)" >> $GITHUB_ENV 36 | 37 | # Get kubernetes service name 38 | echo "KUBERNETES_SERVICE_NAME=$(az resource list --resource-group ${{ inputs.resource_group }} \ 39 | | jq '.[] | select(.type == "Microsoft.ContainerService/managedClusters") | select(.tags.environment == "${{ inputs.environment_tag }}") | .name' -r)" >> $GITHUB_ENV 40 | 41 | # Update Kubernetes manifest file with image 42 | - name: Update manifest 43 | shell: bash 44 | run: | 45 | export CONTAINER_IMAGE=$CONTAINER_REGISTRY_NAME.azurecr.io/${{ inputs.container_image }} 46 | echo "CONTAINER_IMAGE=$CONTAINER_REGISTRY_NAME.azurecr.io/${{ inputs.container_image }}" >> $GITHUB_ENV 47 | envsubst < ${{ inputs.kubernetes_manifest_file }} > manifest.yml 48 | cat manifest.yml 49 | 50 | # Set the target Kubernetes cluster 51 | - name: Set target Kubernetes 52 | uses: azure/aks-set-context@v4 53 | with: 54 | resource-group: ${{ inputs.resource_group }} 55 | cluster-name: ${{ env.KUBERNETES_SERVICE_NAME }} 56 | 57 | # Deploy to Kubernetes cluster 58 | - name: Deploy to Kubernetes 59 | uses: azure/k8s-deploy@v5 60 | with: 61 | action: deploy 62 | manifests: manifest.yml 63 | -------------------------------------------------------------------------------- /.github/workflows/deploy-container-app.yml: -------------------------------------------------------------------------------- 1 | name: Deploy to Container Apps 2 | 3 | on: 4 | # Trigger the workflow manually 5 | workflow_dispatch: 6 | 7 | # Trigger the workflow on pull request 8 | # pull_request: 9 | # types: 10 | # - opened 11 | # - synchronize 12 | # branches: 13 | # - main 14 | 15 | # Trigger the workflow on push 16 | # push: 17 | # branches: 18 | # - main 19 | 20 | env: 21 | AZ_CLI_VERSION: 2.59.0 22 | APP_NAME: creditdefaultapi 23 | GITHUB_RUN_ID: ${{ github.run_id }} 24 | DEPLOYMENT_RESOURCE_GROUP_NAME: ${{ vars.DEPLOYMENT_RESOURCE_GROUP_NAME }} 25 | 26 | permissions: 27 | id-token: write 28 | contents: read 29 | 30 | jobs: 31 | train: 32 | name: Train Model 33 | runs-on: ubuntu-latest 34 | steps: 35 | # Checkout the repository to the GitHub Actions runner 36 | - name: Checkout repo 37 | uses: actions/checkout@v4 38 | 39 | # Authenticate to Az CLI using OIDC 40 | - name: Azure CLI login 41 | uses: azure/login@v2 42 | with: 43 | client-id: ${{ secrets.AZURE_CLIENT_ID }} 44 | tenant-id: ${{ secrets.AZURE_TENANT_ID }} 45 | subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 46 | 47 | # Deploy Databricks Bundle 48 | - name: Deploy Databricks Bundle 49 | uses: "./.github/templates/deploy-databricks-bundle" 50 | with: 51 | resource_group: ${{ env.DEPLOYMENT_RESOURCE_GROUP_NAME }} 52 | 53 | # Run train model workflow 54 | - name: Run workflow 55 | working-directory: databricks 56 | run: | 57 | # Create artifacts directory 58 | mkdir train-artifacts 59 | 60 | # Run train model workflow 61 | databricks bundle run train_register_model_job --output json > train-artifacts/workflow-output.json 62 | 63 | # Display workflow output 64 | cat train-artifacts/workflow-output.json 65 | 66 | # Upload output from workflow run 67 | - name: Upload artifacts 68 | uses: actions/upload-artifact@v4 69 | with: 70 | name: train-artifacts 71 | path: databricks/train-artifacts 72 | if-no-files-found: error 73 | 74 | containerize: 75 | name: Build Container 76 | runs-on: ubuntu-latest 77 | needs: [train] 78 | steps: 79 | # Checkout the repository to the GitHub Actions runner 80 | - name: Checkout repo 81 | uses: actions/checkout@v4 82 | 83 | # Download output from deployment 84 | - uses: actions/download-artifact@v4 85 | with: 86 | name: train-artifacts 87 | 88 | # Authenticate to Az CLI using OIDC 89 | - name: Azure CLI login 90 | uses: azure/login@v2 91 | with: 92 | client-id: ${{ secrets.AZURE_CLIENT_ID }} 93 | tenant-id: ${{ secrets.AZURE_TENANT_ID }} 94 | subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 95 | 96 | # Set up Python 97 | - uses: actions/setup-python@v5 98 | with: 99 | python-version: "3.10" 100 | 101 | # Install MLFlow 102 | - name: Install MLFlow 103 | run: | 104 | pip install mlflow==2.10.0 105 | 106 | # Set Databricks host and token environment variables and MLFlow tracking URI 107 | - name: Set Databricks environment variables 108 | uses: azure/cli@v2 109 | with: 110 | azcliversion: ${{ env.AZ_CLI_VERSION }} 111 | inlineScript: | 112 | DATABRICKS_WORKSPACE_NAME=$(az resource list --resource-group $DEPLOYMENT_RESOURCE_GROUP_NAME \ 113 | | jq '.[] | select(.type == "Microsoft.Databricks/workspaces") | .name' -r) 114 | 115 | echo "DATABRICKS_HOST=https://$(az databricks workspace show --name $DATABRICKS_WORKSPACE_NAME \ 116 | --resource-group $DEPLOYMENT_RESOURCE_GROUP_NAME | jq '.workspaceUrl' -r)" >> $GITHUB_ENV 117 | 118 | echo "DATABRICKS_TOKEN=$(az account get-access-token \ 119 | --resource 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d | jq .accessToken -r)" >> $GITHUB_ENV 120 | 121 | echo "MLFLOW_TRACKING_URI=databricks" >> $GITHUB_ENV 122 | 123 | # Set artifact model environment variables 124 | - name: Set model variables 125 | run: | 126 | MODEL_URI=$(jq \ 127 | '.task_outputs | map(select(.TaskKey | contains("register_model"))) | .[0].Output.result' \ 128 | workflow-output.json -r) 129 | 130 | echo "MODEL_NAME=$(echo $MODEL_URI | cut -d'/' -f2)" >> $GITHUB_ENV 131 | echo "MODEL_VERSION=$(echo $MODEL_URI | cut -d'/' -f3)" >> $GITHUB_ENV 132 | 133 | # Download model artifact 134 | - name: Download model 135 | run: | 136 | # Get model artifact uri 137 | curl -X GET "$DATABRICKS_HOST/api/2.0/preview/mlflow/model-versions/get-download-uri" \ 138 | -H "Authorization: Bearer ${DATABRICKS_TOKEN}" \ 139 | -d '{"name": "'"${MODEL_NAME}"'", "version": "'"${MODEL_VERSION}"'"}' > download-uri.json 140 | 141 | # Display download-uri.json 142 | cat download-uri.json 143 | 144 | # Set model artifact uri 145 | MODEL_ARTIFACT_URI=$(jq '.artifact_uri' download-uri.json -r) 146 | 147 | # Download model artifacts from databricks 148 | mlflow artifacts download --artifact-uri $MODEL_ARTIFACT_URI --dst-path ./app/ 149 | 150 | # Create artifacts directory 151 | mkdir containerize-artifacts 152 | 153 | # Copy model to artifacts directory 154 | cp ./app/model containerize-artifacts/model -r 155 | 156 | # Build and push container image to ACR 157 | - name: Build and push container image 158 | uses: azure/cli@v2 159 | with: 160 | azcliversion: ${{ env.AZ_CLI_VERSION }} 161 | inlineScript: | 162 | # Get container registry name 163 | CONTAINER_REGISTRY_NAME=$(az resource list --resource-group $DEPLOYMENT_RESOURCE_GROUP_NAME \ 164 | | jq '.[] | select(.type == "Microsoft.ContainerRegistry/registries") | .name' -r) 165 | 166 | # Log in to ACR 167 | az acr login --name $CONTAINER_REGISTRY_NAME -t 168 | 169 | # Build container image 170 | az acr build --image $APP_NAME:$GITHUB_RUN_ID --registry $CONTAINER_REGISTRY_NAME ./app 171 | 172 | # Upload model from workflow run 173 | - name: Upload artifacts 174 | uses: actions/upload-artifact@v4 175 | with: 176 | name: containerize-artifacts 177 | path: containerize-artifacts 178 | 179 | staging: 180 | name: Staging Deployment 181 | runs-on: ubuntu-latest 182 | needs: [containerize] 183 | environment: 184 | name: Staging 185 | steps: 186 | # Checkout the repository to the GitHub Actions runner 187 | - name: Checkout repo 188 | uses: actions/checkout@v4 189 | 190 | # Authenticate to Az CLI using OIDC 191 | - name: Azure CLI login 192 | uses: azure/login@v2 193 | with: 194 | client-id: ${{ secrets.AZURE_CLIENT_ID }} 195 | tenant-id: ${{ secrets.AZURE_TENANT_ID }} 196 | subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 197 | 198 | # Build and push container image to ACR 199 | - name: Set resource names 200 | uses: azure/cli@v2 201 | with: 202 | azcliversion: ${{ env.AZ_CLI_VERSION }} 203 | inlineScript: | 204 | # Get container registry name 205 | echo "CONTAINER_REGISTRY_NAME=$(az resource list --resource-group $DEPLOYMENT_RESOURCE_GROUP_NAME \ 206 | | jq '.[] | select(.type == "Microsoft.ContainerRegistry/registries") | .name' -r)" >> $GITHUB_ENV 207 | 208 | # Get container app environment name 209 | echo "CONTAINER_APP_ENVIRONMENT_NAME=$(az resource list --resource-group $DEPLOYMENT_RESOURCE_GROUP_NAME \ 210 | | jq '.[] | select(.type == "Microsoft.App/managedEnvironments") | select(.tags.environment == "staging") | .name' -r)" >> $GITHUB_ENV 211 | 212 | # Get user-assigned identity name 213 | echo "USER_ASSIGNED_IDENTITY_NAME=$(az resource list --resource-group $DEPLOYMENT_RESOURCE_GROUP_NAME \ 214 | | jq '.[] | select(.type == "Microsoft.ManagedIdentity/userAssignedIdentities") | .name' -r)" >> $GITHUB_ENV 215 | 216 | - name: Deploy container app 217 | uses: "./.github/templates/deploy-to-container-app" 218 | with: 219 | container_app_name: ca01${{ env.APP_NAME }} 220 | resource_group: ${{ env.DEPLOYMENT_RESOURCE_GROUP_NAME }} 221 | container_app_environment_name: ${{ env.CONTAINER_APP_ENVIRONMENT_NAME }} 222 | user_assigned_identity_name: ${{ env.USER_ASSIGNED_IDENTITY_NAME }} 223 | container_registry_hostname: ${{ env.CONTAINER_REGISTRY_NAME }}.azurecr.io 224 | container_image: ${{ env.APP_NAME }}:${{ env.GITHUB_RUN_ID }} 225 | artifacts_directory: staging-artifacts 226 | 227 | # Upload model from workflow run 228 | - name: Upload artifacts 229 | uses: actions/upload-artifact@v4 230 | with: 231 | name: staging-artifacts 232 | path: staging-artifacts 233 | if-no-files-found: error 234 | 235 | test: 236 | name: Smoke Test 237 | runs-on: ubuntu-latest 238 | needs: [staging] 239 | environment: 240 | name: Staging 241 | steps: 242 | # Checkout the repository to the GitHub Actions runner 243 | - name: Checkout repo 244 | uses: actions/checkout@v4 245 | 246 | # Download output from deployment 247 | - uses: actions/download-artifact@v4 248 | with: 249 | name: staging-artifacts 250 | 251 | # Smoke test the deployed container app 252 | - name: Smoke test 253 | run: | 254 | # Set the app endpoint 255 | CONTAINER_APP_ENDPOINT=$(cat containerapp.json | jq '.properties.latestRevisionFqdn' -r) 256 | 257 | # Exit on error 258 | set -e 259 | 260 | # Call the app endpoint 261 | STATUS_CODE=$(curl -X POST "https://$CONTAINER_APP_ENDPOINT/predict" \ 262 | -H 'accept: application/json' \ 263 | -H 'Content-Type: application/json' \ 264 | -d @app/sample-request.json \ 265 | -o response.json \ 266 | -w "%{http_code}" \ 267 | -s) 268 | 269 | # Check the status code 270 | if [ $STATUS_CODE -ne 200 ]; then 271 | echo "Got status code $status instead of expected 200" 272 | exit 1 273 | fi 274 | 275 | # Display the response 276 | cat response.json 277 | 278 | production: 279 | name: Production Deployment 280 | if: ${{ github.ref == 'refs/heads/main' }} 281 | runs-on: ubuntu-latest 282 | needs: [staging] 283 | environment: 284 | name: Production 285 | steps: 286 | # Checkout the repository to the GitHub Actions runner 287 | - name: Checkout repo 288 | uses: actions/checkout@v4 289 | 290 | # Authenticate to Az CLI using OIDC 291 | - name: Azure CLI login 292 | uses: azure/login@v2 293 | with: 294 | client-id: ${{ secrets.AZURE_CLIENT_ID }} 295 | tenant-id: ${{ secrets.AZURE_TENANT_ID }} 296 | subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 297 | 298 | # Build and push container image to ACR 299 | - name: Set resource names 300 | uses: azure/cli@v2 301 | with: 302 | azcliversion: ${{ env.AZ_CLI_VERSION }} 303 | inlineScript: | 304 | # Get container registry name 305 | echo "CONTAINER_REGISTRY_NAME=$(az resource list --resource-group $DEPLOYMENT_RESOURCE_GROUP_NAME \ 306 | | jq '.[] | select(.type == "Microsoft.ContainerRegistry/registries") | .name' -r)" >> $GITHUB_ENV 307 | 308 | # Get container app environment name 309 | echo "CONTAINER_APP_ENVIRONMENT_NAME=$(az resource list --resource-group $DEPLOYMENT_RESOURCE_GROUP_NAME \ 310 | | jq '.[] | select(.type == "Microsoft.App/managedEnvironments") | select(.tags.environment == "production") | .name' -r)" >> $GITHUB_ENV 311 | 312 | # Get user-assigned identity name 313 | echo "USER_ASSIGNED_IDENTITY_NAME=$(az resource list --resource-group $DEPLOYMENT_RESOURCE_GROUP_NAME \ 314 | | jq '.[] | select(.type == "Microsoft.ManagedIdentity/userAssignedIdentities") | .name' -r)" >> $GITHUB_ENV 315 | 316 | - name: Deploy container app 317 | uses: "./.github/templates/deploy-to-container-app" 318 | with: 319 | container_app_name: ca02${{ env.APP_NAME }} 320 | resource_group: ${{ env.DEPLOYMENT_RESOURCE_GROUP_NAME }} 321 | container_app_environment_name: ${{ env.CONTAINER_APP_ENVIRONMENT_NAME }} 322 | user_assigned_identity_name: ${{ env.USER_ASSIGNED_IDENTITY_NAME }} 323 | container_registry_hostname: ${{ env.CONTAINER_REGISTRY_NAME }}.azurecr.io 324 | container_image: ${{ env.APP_NAME }}:${{ env.GITHUB_RUN_ID }} 325 | artifacts_directory: production-artifacts 326 | 327 | # Upload model from workflow run 328 | - name: Upload artifacts 329 | uses: actions/upload-artifact@v4 330 | with: 331 | name: production-artifacts 332 | path: production-artifacts 333 | if-no-files-found: error 334 | -------------------------------------------------------------------------------- /.github/workflows/deploy-infrastructure.yml: -------------------------------------------------------------------------------- 1 | name: Deploy Azure Resources 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | env: 7 | AZ_CLI_VERSION: 2.59.0 8 | TEMPLATE_FILE: infrastructure/main.bicep 9 | DEPLOYMENT_LOCATION: ${{ vars.DEPLOYMENT_LOCATION }} 10 | DEPLOYMENT_RESOURCE_GROUP_NAME: ${{ vars.DEPLOYMENT_RESOURCE_GROUP_NAME }} 11 | DEPLOYMENT_DATARBICKS_MANAGED_RESOURCE_GROUP_NAME: ${{ vars.DEPLOYMENT_DATARBICKS_MANAGED_RESOURCE_GROUP_NAME }} 12 | DEPLOYMENT_KUBERNETES_MANAGED_RESOURCE_GROUP_NAME: ${{ vars.DEPLOYMENT_KUBERNETES_MANAGED_RESOURCE_GROUP_NAME }} 13 | DEPLOY_CONTAINER_APPS: ${{ vars.DEPLOY_CONTAINER_APPS == 'true' }} 14 | DEPLOY_KUBERNETES: ${{ vars.DEPLOY_KUBERNETES == 'true' }} 15 | GITHUB_RUN_ID: ${{ github.run_id }} 16 | 17 | permissions: 18 | id-token: write 19 | contents: read 20 | 21 | jobs: 22 | build: 23 | name: Bicep Build 24 | runs-on: ubuntu-latest 25 | steps: 26 | # Checkout the repository to the GitHub Actions runner 27 | - name: Checkout repo 28 | uses: actions/checkout@v4 29 | 30 | # Authenticate to Az CLI using OIDC 31 | - name: Azure CLI login 32 | uses: azure/login@v2 33 | with: 34 | client-id: ${{ secrets.AZURE_CLIENT_ID }} 35 | tenant-id: ${{ secrets.AZURE_TENANT_ID }} 36 | subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 37 | 38 | # Checks that all Bicep configuration files adhere to a canonical format 39 | - name: Bicep lint 40 | uses: azure/cli@v2 41 | with: 42 | azcliversion: ${{ env.AZ_CLI_VERSION }} 43 | inlineScript: az bicep build --file ${TEMPLATE_FILE} 44 | 45 | # Validate whether the template is valid at subscription scope 46 | - name: Bicep validate 47 | uses: azure/cli@v2 48 | with: 49 | azcliversion: ${{ env.AZ_CLI_VERSION }} 50 | inlineScript: | 51 | az deployment sub validate \ 52 | --name validate-${GITHUB_RUN_ID} \ 53 | --template-file ${TEMPLATE_FILE} \ 54 | --location ${DEPLOYMENT_LOCATION} \ 55 | --parameters resourceGroupName=${DEPLOYMENT_RESOURCE_GROUP_NAME} \ 56 | --parameters mrgDatabricksName=${DEPLOYMENT_DATARBICKS_MANAGED_RESOURCE_GROUP_NAME} \ 57 | --parameters mrgKubernetesName=${DEPLOYMENT_KUBERNETES_MANAGED_RESOURCE_GROUP_NAME} \ 58 | --parameters deployContainerAppsEnvironment=${DEPLOY_CONTAINER_APPS} \ 59 | --parameters deployKubernetesService=${DEPLOY_KUBERNETES} \ 60 | --parameters location=${DEPLOYMENT_LOCATION} 61 | 62 | deploy: 63 | name: Bicep Deploy 64 | runs-on: ubuntu-latest 65 | needs: [build] 66 | steps: 67 | # Checkout the repository to the GitHub Actions runner 68 | - name: Checkout repo 69 | uses: actions/checkout@v4 70 | 71 | # Authenticate to Az CLI using OIDC 72 | - name: Azure CLI login 73 | uses: azure/login@v2 74 | with: 75 | client-id: ${{ secrets.AZURE_CLIENT_ID }} 76 | tenant-id: ${{ secrets.AZURE_TENANT_ID }} 77 | subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 78 | 79 | # Deploy template to subscription 80 | - name: Bicep deploy 81 | uses: azure/cli@v2 82 | with: 83 | azcliversion: ${{ env.AZ_CLI_VERSION }} 84 | inlineScript: | 85 | # Create artifacts directory 86 | mkdir artifacts 87 | 88 | # Deploy the Bicep template 89 | az deployment sub create \ 90 | --name validate-${DEPLOYMENT_NAME} \ 91 | --template-file ${TEMPLATE_FILE} \ 92 | --location ${DEPLOYMENT_LOCATION} \ 93 | --parameters resourceGroupName=${DEPLOYMENT_RESOURCE_GROUP_NAME} \ 94 | --parameters mrgDatabricksName=${DEPLOYMENT_DATARBICKS_MANAGED_RESOURCE_GROUP_NAME} \ 95 | --parameters mrgKubernetesName=${DEPLOYMENT_KUBERNETES_MANAGED_RESOURCE_GROUP_NAME} \ 96 | --parameters deployContainerAppsEnvironment=${DEPLOY_CONTAINER_APPS} \ 97 | --parameters deployKubernetesService=${DEPLOY_KUBERNETES} \ 98 | --parameters location=${DEPLOYMENT_LOCATION} \ 99 | > artifacts/deployment-output.json 100 | 101 | # Upload output from deployment 102 | - name: Upload artifacts 103 | uses: actions/upload-artifact@v4 104 | with: 105 | name: artifacts 106 | path: artifacts 107 | 108 | databricks-setup: 109 | name: Databricks Setup 110 | runs-on: ubuntu-latest 111 | needs: [deploy] 112 | steps: 113 | # Checkout the repository to the GitHub Actions runner 114 | - name: Checkout repo 115 | uses: actions/checkout@v4 116 | 117 | # Download output from deployment 118 | - uses: actions/download-artifact@v4 119 | with: 120 | name: artifacts 121 | 122 | # Authenticate to Az CLI using OIDC 123 | - name: Azure CLI login 124 | uses: azure/login@v2 125 | with: 126 | client-id: ${{ secrets.AZURE_CLIENT_ID }} 127 | tenant-id: ${{ secrets.AZURE_TENANT_ID }} 128 | subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 129 | 130 | # Set Databricks host and token environment variables 131 | - name: Set Databricks environment variables 132 | uses: azure/cli@v2 133 | with: 134 | azcliversion: ${{ env.AZ_CLI_VERSION }} 135 | inlineScript: | 136 | echo "DATABRICKS_HOST=https://$(jq .properties.outputs.databricksHostname.value \ 137 | deployment-output.json -r)" >> $GITHUB_ENV 138 | 139 | echo "DATABRICKS_TOKEN=$(az account get-access-token \ 140 | --resource 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d | jq .accessToken -r)" >> $GITHUB_ENV 141 | 142 | # Install the Databricks CLI 143 | - name: Install Databricks CLI 144 | run: | 145 | curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh 146 | 147 | # Configure the Databricks CLI 148 | - name: Databricks CLI config 149 | run: | 150 | cat > ~/.databrickscfg << EOF 151 | [DEFAULT] 152 | host = $DATABRICKS_HOST 153 | token = $DATABRICKS_TOKEN 154 | EOF 155 | 156 | # Create Databricks cluster 157 | - name: Create Databricks cluster 158 | run: | 159 | # Create artifacts directory 160 | mkdir artifacts 161 | 162 | # Create Databricks cluster 163 | curl -X POST "$DATABRICKS_HOST/api/2.0/clusters/create" \ 164 | -H "Authorization: Bearer $DATABRICKS_TOKEN" \ 165 | -d '{ 166 | "num_workers": 0, 167 | "cluster_name": "default", 168 | "spark_version": "14.3.x-cpu-ml-scala2.12", 169 | "spark_conf": { 170 | "spark.master": "local[*, 4]", 171 | "spark.databricks.cluster.profile": "singleNode" 172 | }, 173 | "azure_attributes": { 174 | "first_on_demand": 1, 175 | "availability": "ON_DEMAND_AZURE", 176 | "spot_bid_max_price": -1 177 | }, 178 | "node_type_id": "Standard_D4ads_v5", 179 | "driver_node_type_id": "Standard_D4ads_v5", 180 | "autotermination_minutes": 60, 181 | "enable_elastic_disk": true, 182 | "enable_local_disk_encryption": false, 183 | "runtime_engine": "STANDARD" 184 | }' > artifacts/cluster-output.json 185 | 186 | # Display cluster output 187 | cat artifacts/cluster-output.json 188 | 189 | # Set Databricks cluster id environment variable 190 | - name: Set Databricks clister id 191 | run: | 192 | echo "DATABRICKS_CLUSTER_ID=$(jq .cluster_id artifacts/cluster-output.json -r)" >> $GITHUB_ENV 193 | 194 | # Upload files to DBFS 195 | - name: Upload data to dbfs 196 | run: | 197 | databricks fs mkdir dbfs:/FileStore/data/credit-card-default-uci-curated 198 | databricks fs cp -r databricks/data/curated.csv dbfs:/FileStore/data/credit-card-default-uci-curated/01.csv 199 | 200 | # Trigger notebook to create external tables 201 | - name: Create external tables 202 | uses: databricks/run-notebook@v0 203 | with: 204 | databricks-host: ${{ env.DATABRICKS_HOST }} 205 | databricks-token: ${{ env.DATABRICKS_TOKEN }} 206 | existing-cluster-id: ${{ env.DATABRICKS_CLUSTER_ID }} 207 | local-notebook-path: databricks/src/00-create-external-table.ipynb 208 | notebook-params-json: > 209 | { 210 | "path": "dbfs:/FileStore/data/credit-card-default-uci-curated" 211 | } 212 | 213 | kubernetes-setup: 214 | name: Kubernetes Setup 215 | runs-on: ubuntu-latest 216 | needs: [deploy] 217 | if: ${{ vars.DEPLOY_KUBERNETES == 'true' }} 218 | steps: 219 | # Checkout the repository to the GitHub Actions runner 220 | - name: Checkout repo 221 | uses: actions/checkout@v4 222 | 223 | # Download output from deployment 224 | - uses: actions/download-artifact@v4 225 | with: 226 | name: artifacts 227 | 228 | # Authenticate to Az CLI using OIDC 229 | - name: Azure CLI login 230 | uses: azure/login@v2 231 | with: 232 | client-id: ${{ secrets.AZURE_CLIENT_ID }} 233 | tenant-id: ${{ secrets.AZURE_TENANT_ID }} 234 | subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 235 | 236 | # Set Kubernetes Service and Container Registry environment variables 237 | - name: Set environment variables 238 | uses: azure/cli@v2 239 | with: 240 | azcliversion: ${{ env.AZ_CLI_VERSION }} 241 | inlineScript: | 242 | echo "KUBERNETES_SERVICE_STAGING=$(jq .properties.outputs.kubernetesServiceStagingName.value \ 243 | deployment-output.json -r)" >> $GITHUB_ENV 244 | 245 | echo "KUBERNETES_SERVICE_PRODUCTION=$(jq .properties.outputs.kubernetesServiceProductionName.value \ 246 | deployment-output.json -r)" >> $GITHUB_ENV 247 | 248 | echo "CONTAINER_REGISTRY=$(jq .properties.outputs.containerRegistryName.value \ 249 | deployment-output.json -r)" >> $GITHUB_ENV 250 | 251 | # Attach ACR to AKS 252 | - name: Attach ACR to AKS 253 | uses: azure/cli@v2 254 | with: 255 | azcliversion: ${{ env.AZ_CLI_VERSION }} 256 | inlineScript: | 257 | az aks update --name $KUBERNETES_SERVICE_STAGING --resource-group $DEPLOYMENT_RESOURCE_GROUP_NAME \ 258 | --attach-acr $CONTAINER_REGISTRY 259 | az aks update --name $KUBERNETES_SERVICE_PRODUCTION --resource-group $DEPLOYMENT_RESOURCE_GROUP_NAME \ 260 | --attach-acr $CONTAINER_REGISTRY 261 | -------------------------------------------------------------------------------- /.github/workflows/deploy-kubernetes.yml: -------------------------------------------------------------------------------- 1 | name: Deploy to Kubernetes Service 2 | 3 | on: 4 | # Trigger the workflow manually 5 | workflow_dispatch: 6 | 7 | # Trigger the workflow on pull request 8 | # pull_request: 9 | # types: 10 | # - opened 11 | # - synchronize 12 | # branches: 13 | # - main 14 | 15 | # Trigger the workflow on push 16 | # push: 17 | # branches: 18 | # - main 19 | 20 | env: 21 | AZ_CLI_VERSION: 2.59.0 22 | APP_NAME: creditdefaultapi 23 | GITHUB_RUN_ID: ${{ github.run_id }} 24 | DEPLOYMENT_RESOURCE_GROUP_NAME: ${{ vars.DEPLOYMENT_RESOURCE_GROUP_NAME }} 25 | 26 | permissions: 27 | id-token: write 28 | contents: read 29 | 30 | jobs: 31 | train: 32 | name: Train Model 33 | runs-on: ubuntu-latest 34 | steps: 35 | # Checkout the repository to the GitHub Actions runner 36 | - name: Checkout repo 37 | uses: actions/checkout@v4 38 | 39 | # Authenticate to Az CLI using OIDC 40 | - name: Azure CLI login 41 | uses: azure/login@v2 42 | with: 43 | client-id: ${{ secrets.AZURE_CLIENT_ID }} 44 | tenant-id: ${{ secrets.AZURE_TENANT_ID }} 45 | subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 46 | 47 | # Deploy Databricks Bundle 48 | - name: Deploy Databricks Bundle 49 | uses: "./.github/templates/deploy-databricks-bundle" 50 | with: 51 | resource_group: ${{ env.DEPLOYMENT_RESOURCE_GROUP_NAME }} 52 | 53 | # Run train model workflow 54 | - name: Run workflow 55 | working-directory: databricks 56 | run: | 57 | # Create artifacts directory 58 | mkdir train-artifacts 59 | 60 | # Run train model workflow 61 | databricks bundle run train_register_model_job --output json > train-artifacts/workflow-output.json 62 | 63 | # Display workflow output 64 | cat train-artifacts/workflow-output.json 65 | 66 | # Upload output from workflow run 67 | - name: Upload artifacts 68 | uses: actions/upload-artifact@v4 69 | with: 70 | name: train-artifacts 71 | path: databricks/train-artifacts 72 | if-no-files-found: error 73 | 74 | containerize: 75 | name: Build Container 76 | runs-on: ubuntu-latest 77 | needs: [train] 78 | steps: 79 | # Checkout the repository to the GitHub Actions runner 80 | - name: Checkout repo 81 | uses: actions/checkout@v4 82 | 83 | # Download output from deployment 84 | - uses: actions/download-artifact@v4 85 | with: 86 | name: train-artifacts 87 | 88 | # Authenticate to Az CLI using OIDC 89 | - name: Azure CLI login 90 | uses: azure/login@v2 91 | with: 92 | client-id: ${{ secrets.AZURE_CLIENT_ID }} 93 | tenant-id: ${{ secrets.AZURE_TENANT_ID }} 94 | subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 95 | 96 | # Set up Python 97 | - uses: actions/setup-python@v5 98 | with: 99 | python-version: "3.10" 100 | 101 | # Install MLFlow 102 | - name: Install MLFlow 103 | run: | 104 | pip install mlflow==2.10.0 105 | 106 | # Set Databricks host and token environment variables and MLFlow tracking URI 107 | - name: Set Databricks environment variables 108 | uses: azure/cli@v2 109 | with: 110 | azcliversion: ${{ env.AZ_CLI_VERSION }} 111 | inlineScript: | 112 | DATABRICKS_WORKSPACE_NAME=$(az resource list --resource-group $DEPLOYMENT_RESOURCE_GROUP_NAME \ 113 | | jq '.[] | select(.type == "Microsoft.Databricks/workspaces") | .name' -r) 114 | 115 | echo "DATABRICKS_HOST=https://$(az databricks workspace show --name $DATABRICKS_WORKSPACE_NAME \ 116 | --resource-group $DEPLOYMENT_RESOURCE_GROUP_NAME | jq '.workspaceUrl' -r)" >> $GITHUB_ENV 117 | 118 | echo "DATABRICKS_TOKEN=$(az account get-access-token \ 119 | --resource 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d | jq .accessToken -r)" >> $GITHUB_ENV 120 | 121 | echo "MLFLOW_TRACKING_URI=databricks" >> $GITHUB_ENV 122 | 123 | # Set artifact model environment variables 124 | - name: Set model variables 125 | run: | 126 | MODEL_URI=$(jq \ 127 | '.task_outputs | map(select(.TaskKey | contains("register_model"))) | .[0].Output.result' \ 128 | workflow-output.json -r) 129 | 130 | echo "MODEL_NAME=$(echo $MODEL_URI | cut -d'/' -f2)" >> $GITHUB_ENV 131 | echo "MODEL_VERSION=$(echo $MODEL_URI | cut -d'/' -f3)" >> $GITHUB_ENV 132 | 133 | # Download model artifact 134 | - name: Download model 135 | run: | 136 | # Get model artifact uri 137 | curl -X GET "$DATABRICKS_HOST/api/2.0/preview/mlflow/model-versions/get-download-uri" \ 138 | -H "Authorization: Bearer ${DATABRICKS_TOKEN}" \ 139 | -d '{"name": "'"${MODEL_NAME}"'", "version": "'"${MODEL_VERSION}"'"}' > download-uri.json 140 | 141 | # Display download-uri.json 142 | cat download-uri.json 143 | 144 | # Set model artifact uri 145 | MODEL_ARTIFACT_URI=$(jq '.artifact_uri' download-uri.json -r) 146 | 147 | # Download model artifacts from databricks 148 | mlflow artifacts download --artifact-uri $MODEL_ARTIFACT_URI --dst-path ./app/ 149 | 150 | # Create artifacts directory 151 | mkdir containerize-artifacts 152 | 153 | # Copy model to artifacts directory 154 | cp ./app/model containerize-artifacts/model -r 155 | 156 | # Build and push container image to ACR 157 | - name: Build and push container image 158 | uses: azure/cli@v2 159 | with: 160 | azcliversion: ${{ env.AZ_CLI_VERSION }} 161 | inlineScript: | 162 | # Get container registry name 163 | CONTAINER_REGISTRY_NAME=$(az resource list --resource-group $DEPLOYMENT_RESOURCE_GROUP_NAME \ 164 | | jq '.[] | select(.type == "Microsoft.ContainerRegistry/registries") | .name' -r) 165 | 166 | # Log in to ACR 167 | az acr login --name $CONTAINER_REGISTRY_NAME -t 168 | 169 | # Build container image 170 | az acr build --image $APP_NAME:$GITHUB_RUN_ID --registry $CONTAINER_REGISTRY_NAME ./app 171 | 172 | # Upload model from workflow run 173 | - name: Upload artifacts 174 | uses: actions/upload-artifact@v4 175 | with: 176 | name: containerize-artifacts 177 | path: containerize-artifacts 178 | 179 | staging: 180 | name: Staging Deployment 181 | runs-on: ubuntu-latest 182 | needs: [containerize] 183 | environment: 184 | name: Staging 185 | steps: 186 | # Checkout the repository to the GitHub Actions runner 187 | - name: Checkout repo 188 | uses: actions/checkout@v4 189 | 190 | # Authenticate to Az CLI using OIDC 191 | - name: Azure CLI login 192 | uses: azure/login@v2 193 | with: 194 | client-id: ${{ secrets.AZURE_CLIENT_ID }} 195 | tenant-id: ${{ secrets.AZURE_TENANT_ID }} 196 | subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 197 | 198 | # Deploy to Kubernetes cluster 199 | - name: Deploy to Kubernetes 200 | uses: "./.github/templates/deploy-to-kubernetes" 201 | with: 202 | resource_group: ${{ env.DEPLOYMENT_RESOURCE_GROUP_NAME }} 203 | environment_tag: staging 204 | container_image: ${{ env.APP_NAME }}:$GITHUB_RUN_ID 205 | 206 | test: 207 | name: Smoke Test 208 | runs-on: ubuntu-latest 209 | needs: [staging] 210 | environment: 211 | name: Staging 212 | steps: 213 | # Checkout the repository to the GitHub Actions runner 214 | - name: Checkout repo 215 | uses: actions/checkout@v4 216 | 217 | # Authenticate to Az CLI using OIDC 218 | - name: Azure CLI login 219 | uses: azure/login@v2 220 | with: 221 | client-id: ${{ secrets.AZURE_CLIENT_ID }} 222 | tenant-id: ${{ secrets.AZURE_TENANT_ID }} 223 | subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 224 | 225 | # Get kubernetes service name 226 | - name: Set resource names 227 | uses: azure/cli@v2 228 | with: 229 | azcliversion: ${{ env.AZ_CLI_VERSION }} 230 | inlineScript: | 231 | echo "KUBERNETES_SERVICE_NAME=$(az resource list --resource-group $DEPLOYMENT_RESOURCE_GROUP_NAME \ 232 | | jq '.[] | select(.type == "Microsoft.ContainerService/managedClusters") | select(.tags.environment == "staging") | .name' -r)" >> $GITHUB_ENV 233 | 234 | # Set the target Kubernetes cluster 235 | - name: Set target Kubernetes 236 | uses: azure/aks-set-context@v4 237 | with: 238 | resource-group: ${{ env.DEPLOYMENT_RESOURCE_GROUP_NAME }} 239 | cluster-name: ${{ env.KUBERNETES_SERVICE_NAME }} 240 | 241 | # Smoke test the deployed container app 242 | - name: Smoke test 243 | run: | 244 | # Set the app endpoint 245 | INGRESS_IP=$(kubectl get service --sort-by=.metadata.creationTimestamp --output json | jq '.items[-1].status.loadBalancer.ingress[0].ip' -r) 246 | TARGET_PORT=$(kubectl get service --sort-by=.metadata.creationTimestamp --output json | jq '.items[-1].spec.ports[0].targetPort' -r) 247 | APP_ENDPOINT="http://$INGRESS_IP:$TARGET_PORT/predict" 248 | 249 | # Display endpoint 250 | echo $APP_ENDPOINT 251 | 252 | # Exit on error 253 | set -e 254 | 255 | # Call the app endpoint 256 | STATUS_CODE=$(curl -X POST "$APP_ENDPOINT" \ 257 | -H 'accept: application/json' \ 258 | -H 'Content-Type: application/json' \ 259 | -d @app/sample-request.json \ 260 | -o response.json \ 261 | -w "%{http_code}" \ 262 | -s) 263 | 264 | # Check the status code 265 | if [ $STATUS_CODE -ne 200 ]; then 266 | echo "Got status code $status instead of expected 200" 267 | exit 1 268 | fi 269 | 270 | # Display the response 271 | cat response.json 272 | 273 | production: 274 | name: Production Deployment 275 | if: ${{ github.ref == 'refs/heads/main' }} 276 | runs-on: ubuntu-latest 277 | needs: [staging] 278 | environment: 279 | name: Production 280 | steps: 281 | # Checkout the repository to the GitHub Actions runner 282 | - name: Checkout repo 283 | uses: actions/checkout@v4 284 | 285 | # Authenticate to Az CLI using OIDC 286 | - name: Azure CLI login 287 | uses: azure/login@v2 288 | with: 289 | client-id: ${{ secrets.AZURE_CLIENT_ID }} 290 | tenant-id: ${{ secrets.AZURE_TENANT_ID }} 291 | subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 292 | 293 | # Deploy to Kubernetes cluster 294 | - name: Deploy to Kubernetes 295 | uses: "./.github/templates/deploy-to-kubernetes" 296 | with: 297 | resource_group: ${{ env.DEPLOYMENT_RESOURCE_GROUP_NAME }} 298 | environment_tag: production 299 | container_image: ${{ env.APP_NAME }}:$GITHUB_RUN_ID 300 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # Custom 132 | .DS_Store 133 | .databricks 134 | mlruns/ 135 | app/model/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Nicholas Moore 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Example Scenarios: MLOps with Azure Databricks using Containers for Online Inference 2 | 3 | ## :books: Overview 4 | 5 | This repository provides prescriptive guidance when building, deploying, and monitoring machine learning models with [Azure Databricks](https://learn.microsoft.com/azure/databricks/introduction/) for online inference scenarios in line with MLOps principles and practices. 6 | 7 | MLOps is a set of repeatable, automated, and collaborative workflows with best practices that empower teams of ML professionals to quickly and easily get their machine learning models deployed into production. 8 | 9 | ## :computer: Getting Started 10 | 11 | This repository will focus on online inference scenarios that integrate Azure Databricks with other Azure services to deploy machine learning models as web services. Out-of-the-box capabilities of Azure Databricks will be used to build machine learning models, but the deployment and monitoring of these models will be done using Azure Container Apps or Azure Kubernetes Service. 12 | 13 | All example scenarios will focus on classical machine learning problems. An adapted version of the `UCI Credit Card Client Default` [dataset](https://archive.ics.uci.edu/dataset/350/default+of+credit+card+clients) will be used to illustrate each example scenario. The data is available in the `core/data` directory of this repository. 14 | 15 | ### Setup 16 | 17 | Detailed instructions for deploying this proof-of-concept are outlined in the [Step-by-Step Setup](.github/docs/step-by-step-setup.md) section of this repository. This proof-of-concept will illustrate how to: 18 | 19 | - Build a machine learning model on Azure Databricks. 20 | - Containerize the machine learning model. 21 | - Deploy the machine learning model as a web service using Azure Container Apps or Azure Kubernetes Service. 22 | - Develop automated workflows to build and deploy models. 23 | - Monitor the machine learning model for usage, performance, and data drift. 24 | 25 | ### Example Scenarios 26 | 27 | This proof-of-concept will cover the following example scenarios: 28 | 29 | | Example Scenario | Description | 30 | | ---------------- | ----------- | 31 | | Azure Container Apps | Build a machine learning model on Azure Databricks, containerize it, and deploy it as a web service using Azure Container Apps. | 32 | | Azure Kubernetes Service | Build a machine learning model on Azure Databricks, containerize it, and deploy it as a web service using Azure Kubernetes Service. | 33 | 34 | For more information on the example scenarios are outlined in the [Getting Started](.github/docs/getting-started.md) section of this repository. 35 | 36 | ## :balance_scale: License 37 | 38 | Details on licensing for the project can be found in the [LICENSE](./LICENSE) file. 39 | -------------------------------------------------------------------------------- /app/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | WORKDIR /app 4 | 5 | COPY ./requirements.txt /app/requirements.txt 6 | 7 | RUN pip install --no-cache-dir --upgrade pip && \ 8 | pip install --no-cache-dir -r /app/requirements.txt 9 | 10 | RUN apt-get update && \ 11 | apt-get install -y ffmpeg libsm6 libxext6 && \ 12 | rm -rf /var/lib/apt/lists/* 13 | 14 | COPY ./model.py /app/model.py 15 | 16 | COPY ./main.py /app/main.py 17 | 18 | COPY ./model /app/model 19 | 20 | ENV MODEL_DIRECTORY=model 21 | 22 | EXPOSE 5000 23 | 24 | CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "5000"] 25 | -------------------------------------------------------------------------------- /app/main.py: -------------------------------------------------------------------------------- 1 | """Main module for the FastAPI application.""" 2 | 3 | import json 4 | import logging 5 | import os 6 | import uuid 7 | from typing import AsyncGenerator 8 | 9 | import mlflow 10 | import pandas as pd 11 | import uvicorn 12 | from fastapi import FastAPI 13 | from fastapi.concurrency import asynccontextmanager 14 | from model import LoanApplicant, ModelOutput 15 | 16 | # Initialize the ML models 17 | ml_models = {} 18 | 19 | 20 | @asynccontextmanager 21 | async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: 22 | """ 23 | A context manager to initialize and clean up the ML models. 24 | """ 25 | # Load the ML model 26 | ml_models["credit_default"] = mlflow.pyfunc.load_model( 27 | os.getenv("MODEL_DIRECTORY", "./app/model") 28 | ) 29 | yield 30 | # Clean up the ML models and release the resources 31 | ml_models.clear() 32 | 33 | 34 | # Initialize the FastAPI app 35 | app = FastAPI( 36 | title=os.environ.get("SERVICE_NAME", "credit-default-api"), 37 | docs_url="/", 38 | lifespan=lifespan, 39 | ) 40 | 41 | 42 | @app.post("/predict", response_model=ModelOutput) 43 | async def predict(data: list[LoanApplicant]) -> str: 44 | """ 45 | An endpoint to make predictions on the input data. 46 | 47 | Parameters: 48 | request (List[LoanApplicant]): A list of loan applicant data. 49 | 50 | Returns: 51 | response (str): A JSON response containing the model predictions. 52 | """ 53 | # Parse data 54 | input_df = pd.DataFrame(data) 55 | 56 | # Define UUID for the request 57 | request_id = uuid.uuid4().hex 58 | 59 | # Log inference data 60 | logging.info( 61 | json.dumps( 62 | { 63 | "service_name": os.environ.get("SERVICE_NAME", "credit-default-api"), 64 | "type": "InferenceData", 65 | "request_id": request_id, 66 | "data": input_df.to_json(orient="records"), 67 | } 68 | ) 69 | ) 70 | 71 | # Generate predictions 72 | model_output = ml_models["credit_default"].predict(input_df) 73 | 74 | # Log model outputs 75 | logging.info( 76 | json.dumps( 77 | { 78 | "service_name": os.environ.get("SERVICE_NAME", "credit-default-api"), 79 | "type": "ModelOutput", 80 | "request_id": request_id, 81 | "data": model_output, 82 | } 83 | ) 84 | ) 85 | 86 | return model_output 87 | 88 | 89 | # Configure logging 90 | logging.basicConfig(level=logging.INFO) 91 | 92 | if __name__ == "__main__": 93 | uvicorn.run(app, host="0.0.0.0", port=5000) 94 | -------------------------------------------------------------------------------- /app/model.py: -------------------------------------------------------------------------------- 1 | """Data model for loan applicant""" 2 | 3 | import dataclasses 4 | 5 | from pydantic import BaseModel 6 | 7 | 8 | @dataclasses.dataclass 9 | class LoanApplicant(BaseModel): 10 | """Load applicant data model""" 11 | 12 | sex: str = "male" 13 | education: str = "university" 14 | marriage: str = "married" 15 | repayment_status_1: str = "duly_paid" 16 | repayment_status_2: str = "duly_paid" 17 | repayment_status_3: str = "duly_paid" 18 | repayment_status_4: str = "duly_paid" 19 | repayment_status_5: str = "no_delay" 20 | repayment_status_6: str = "no_delay" 21 | credit_limit: float = 18000.0 22 | age: float = 18000.0 23 | bill_amount_1: float = 764.95 24 | bill_amount_2: float = 2221.95 25 | bill_amount_3: float = 1131.85 26 | bill_amount_4: float = 5074.85 27 | bill_amount_5: float = 18000.0 28 | bill_amount_6: float = 1419.95 29 | payment_amount_1: float = 2236.5 30 | payment_amount_2: float = 1137.55 31 | payment_amount_3: float = 5084.55 32 | payment_amount_4: float = 111.65 33 | payment_amount_5: float = 306.9 34 | payment_amount_6: float = 805.65 35 | 36 | 37 | @dataclasses.dataclass 38 | class FeatureBatchDrift(BaseModel): 39 | sex: float 40 | education: float 41 | marriage: float 42 | repayment_status_1: float 43 | repayment_status_2: float 44 | repayment_status_3: float 45 | repayment_status_4: float 46 | repayment_status_5: float 47 | repayment_status_6: float 48 | credit_limit: float 49 | age: float 50 | bill_amount_1: float 51 | bill_amount_2: float 52 | bill_amount_3: float 53 | bill_amount_4: float 54 | bill_amount_5: float 55 | bill_amount_6: float 56 | payment_amount_1: float 57 | payment_amount_2: float 58 | payment_amount_3: float 59 | payment_amount_4: float 60 | payment_amount_5: float 61 | payment_amount_6: float 62 | 63 | 64 | @dataclasses.dataclass 65 | class ModelOutput(BaseModel): 66 | """Model output data model""" 67 | 68 | predictions: list[float] 69 | outliers: list[float] 70 | feature_drift_batch: FeatureBatchDrift 71 | -------------------------------------------------------------------------------- /app/requirements.txt: -------------------------------------------------------------------------------- 1 | # API 2 | fastapi==0.110.2 3 | uvicorn==0.29.0 4 | 5 | # Model 6 | alibi-detect==0.12.0 7 | configparser==5.2.0 8 | mlflow==2.10.0 9 | psutil==5.9.0 10 | cloudpickle==2.0.0 11 | joblib==1.2.0 12 | numpy==1.23.5 13 | pandas==1.5.3 14 | scikit-learn==1.1.1 15 | typing-extensions==4.8.0 -------------------------------------------------------------------------------- /app/sample-request.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "sex": "male", 4 | "education": "university", 5 | "marriage": "married", 6 | "repayment_status_1": "duly_paid", 7 | "repayment_status_2": "duly_paid", 8 | "repayment_status_3": "duly_paid", 9 | "repayment_status_4": "duly_paid", 10 | "repayment_status_5": "no_delay", 11 | "repayment_status_6": "no_delay", 12 | "credit_limit": 18000, 13 | "age": 18000, 14 | "bill_amount_1": 764.95, 15 | "bill_amount_2": 2221.95, 16 | "bill_amount_3": 1131.85, 17 | "bill_amount_4": 5074.85, 18 | "bill_amount_5": 18000, 19 | "bill_amount_6": 1419.95, 20 | "payment_amount_1": 2236.5, 21 | "payment_amount_2": 1137.55, 22 | "payment_amount_3": 5084.55, 23 | "payment_amount_4": 111.65, 24 | "payment_amount_5": 306.9, 25 | "payment_amount_6": 805.65 26 | } 27 | ] -------------------------------------------------------------------------------- /databricks/data/inference.csv: -------------------------------------------------------------------------------- 1 | credit_limit,sex,education,marriage,age,repayment_status_1,repayment_status_2,repayment_status_3,repayment_status_4,repayment_status_5,repayment_status_6,bill_amount_1,bill_amount_2,bill_amount_3,bill_amount_4,bill_amount_5,bill_amount_6,payment_amount_1,payment_amount_2,payment_amount_3,payment_amount_4,payment_amount_5,payment_amount_6 2 | 40000,female,university,single,55,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,14610.95,7010.35,6770.95,7160.55,7470.4,7770.45,14250,7500,5000,5000,5250,7600 3 | 36000,male,university,married,37,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,4174.56,2002.96,1934.56,2045.87,2134.40,2220.13,4071.43,2142.86,1428.57,1428.57,1500.00,2171.43 4 | 13000,female,university,married,47,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,9.90,0.00,0.00,0.00,235.30,0.00,0.00,0.00,0.00,235.30,0.00,0.00 5 | 10000,male,graduate_school,married,55,duly_paid,duly_paid,duly_paid,duly_paid,delay_2_months,delay_2_months,7625.95,7437.55,7203.8,408.7,409.9,395.9,0,0,411.1,15,0,50 6 | 1000,male,university,married,53,delay_2_months,delay_2_months,delay_2_months,no_delay,no_delay,no_delay,104.85,209.65,198.9,203.1,209.8,216.3,115,0,7.5,10,10,8 7 | 10500,female,graduate_school,single,26,delay_2_months,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,30,30,115.9,228,30,425,30,115.9,228,30,425,90.9 8 | 6500,female,high_school,single,27,delay_1_month,duly_paid,duly_paid,duly_paid,delay_2_months,duly_paid,-9.5,-492.5,-492.5,515.55,508.05,365.95,0,0,1008.05,0,365.95,694.95 9 | 1000,male,university,married,39,no_delay,no_delay,no_delay,no_delay,no_delay,duly_paid,898.65,968.35,977.95,912,896.4,7.5,84.95,73,31.3,87.5,7.5,0 10 | 7000,male,graduate_school,single,36,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,67.2,1021.2,85,41.5,10,143,1021.2,85,41.5,10,143,0 11 | 19000,female,graduate_school,single,30,duly_paid,duly_paid,duly_paid,no_delay,no_delay,no_delay,-4.05,-15.15,1623.75,1644.55,1678.2,1702.8,11.15,1658.9,58.55,59.85,62.5,250 12 | 24000,male,graduate_school,married,61,no_delay,no_delay,no_delay,delay_2_months,delay_2_months,no_delay,21103.45,21567.1,23971.6,24353.3,23557.25,23498.05,803.9,2784.65,850,0,900,1210 13 | 2500,female,high_school,single,17,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,2234.9,2112.7,1917.35,1624.8,1173.85,1204.7,88.35,68.1,50.1,42,49.75,45.2 14 | 3000,female,university,single,25,delay_2_months,delay_2_months,delay_2_months,delay_2_months,delay_2_months,no_delay,5668.5,5520.8,5917.5,6021.8,5544.7,5530.5,0,500,251.1,0.6,300,300 15 | 7000,female,university,single,20,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,558,0,0,0,0,2652.9,0,0,0,0,2652.9,200 16 | 4000,female,university,married,36,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,305.4,143.05,163.85,165.95,57.5,57.5,143.05,163.95,165.95,57.5,57.5,51.75 17 | 35000,male,graduate_school,single,48,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,35.65,113.6,36.1,43.35,57.5,263.15,113.6,36.1,43.35,57.5,263.15,250.55 18 | 6500,male,university,single,42,no_delay,no_delay,no_delay,duly_paid,duly_paid,duly_paid,17143.8,17838.2,3994,12048.3,4412.7,12656.8,1090.8,0,13365.7,456.6,13384.1,479.6 19 | 36000,male,university,married,36,delay_1_month,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,-5.15,-5.15,-5.15,-5.15,-5.15,-5.15,0,0,0,0,0,0 20 | 16500,female,graduate_school,married,36,no_delay,no_delay,delay_2_months,no_delay,no_delay,no_delay,10587.9,10843.1,10559.4,10589.6,10649.1,10728.9,926,0,359.3,410,1579.4,0 21 | 5000,male,high_school,married,43,no_delay,no_delay,delay_2_months,no_delay,no_delay,no_delay,1324.4,1472.2,1518.1,1592.8,1667.1,1739.3,200,100,100,100,100,100 22 | 14000,male,university,married,38,delay_2_months,delay_2_months,delay_2_months,delay_2_months,delay_2_months,delay_3_months,6783.65,6926.6,6740.65,7220.05,7608.7,7470.75,325,0,712.7,742.5,0,250 23 | 5000,female,graduate_school,single,27,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,5212.8,5269.2,5447.7,5607.6,6010,5971.3,200,267.7,307.6,508,300,203.3 24 | 5000,male,university,single,37,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,950.75,964.7,1012.95,1013.7,1015.55,997.85,67,65.25,35,35.9,36.2,34.2 25 | 3000,male,graduate_school,single,27,duly_paid,delay_2_months,no_delay,no_delay,delay_3_months,delay_2_months,909.95,880.9,931.55,1065.95,1034.6,1060.05,0,65.6,150,0,50,50 26 | 24000,male,graduate_school,single,31,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,16.3,16.3,16.3,283.8,23.8,16.3,16.3,16.3,283.8,23.8,16.3,26.3 27 | 8000,male,university,single,24,delay_2_months,no_delay,no_delay,no_delay,no_delay,no_delay,1402.9,1549.3,1663,1705.5,1762.9,1818.6,200,170,100,100,100,100 28 | 40000,male,university,married,35,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,983,483.3,593.35,391.95,741.85,397.95,483.85,593.35,391.95,741.85,397.95,285.6 29 | 12000,female,university,single,34,no_delay,no_delay,no_delay,no_delay,duly_paid,duly_paid,2512.7,2572.25,2650.75,2623.95,65.35,60.15,100,150,150,65.35,60.15,28.15 30 | 5000,male,high_school,single,40,delay_2_months,delay_2_months,delay_2_months,delay_3_months,delay_2_months,delay_2_months,2300.2,2298.8,2447.65,2442.55,2465.9,2557.15,50,201.75,50,70,140,0 31 | 45000,male,graduate_school,married,38,delay_1_month,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,0,0,0,0,0,0,0,0,0,0,0,0 32 | 5500,female,graduate_school,married,52,delay_1_month,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,0,0,0,0,0,0,0,0,0,0,0,0 33 | 15500,female,university,married,33,delay_2_months,no_delay,no_delay,no_delay,no_delay,no_delay,30499.1,31124.3,30631.4,25861,24649.1,19888.9,1301.9,1112.8,840.7,859.9,683.3,598.7 34 | 2000,male,graduate_school,single,30,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,1911.5,1896.2,1929.8,1937.8,1971.7,1563,140.4,113,60,86.1,31.3,0 35 | 2000,male,university,single,23,delay_1_month,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,0,0,0,0,0,0,0,0,0,0,0,0 36 | 2500,male,university,single,30,delay_1_month,delay_2_months,no_delay,no_delay,no_delay,delay_2_months,2443,2390.05,2418.15,1511.05,1143.85,1118.05,0,75,50,100,0,100 37 | 1000,female,university,single,23,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,1600.1,1262.2,1322.1,1313,1403.4,1490.6,121.2,120.1,50,150,150,100 38 | 8000,female,university,married,32,no_delay,no_delay,delay_2_months,no_delay,no_delay,no_delay,3894.15,4090.55,4012.5,3073.35,533.1,574.3,290,50,30,20,50,0 39 | 24000,male,graduate_school,single,38,duly_paid,duly_paid,delay_2_months,no_delay,no_delay,duly_paid,610.6,1328.9,1266.55,1330.25,1313.95,62.8,750,0,100,0,62.8,3296.75 40 | 8000,female,high_school,single,30,no_delay,duly_paid,no_delay,no_delay,no_delay,no_delay,2480.4,620.6,743.65,868.2,888.5,873,625,325,150,100,150,100 41 | 25000,female,graduate_school,married,46,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,2821.1,5530.8,5517,6148.35,5441.7,3503.2,3500.5,1517.85,1500,1000,2609.15,1000 42 | 3000,female,university,married,28,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,2901.2,2894.55,2441.95,948.55,966.15,969.75,125,80,150,50,36.85,100 43 | 1000,male,university,single,23,no_delay,no_delay,no_delay,no_delay,no_delay,duly_paid,1064.2,1167.7,1307,1228,161.5,162,120,159.3,60.1,13.5,182.4,0 44 | 10000,female,university,married,36,delay_1_month,delay_2_months,no_delay,no_delay,delay_2_months,no_delay,1448.3,1396.1,1532.3,1626.8,1586.8,1644.8,0,160,150,0,100,150 45 | 18000,female,graduate_school,single,37,delay_1_month,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,130.8,2853.85,264.35,3422.25,694.05,812,2854.35,264.75,3422.7,694.45,812.5,1915.65 46 | 10000,female,high_school,single,42,delay_2_months,delay_2_months,delay_2_months,delay_2_months,delay_2_months,delay_2_months,9971.8,10147.35,9696.8,9809.3,10008.1,9495.75,410.7,350,340,356.7,0,341.8 47 | 25000,female,graduate_school,married,31,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,20.6,6.9,114.95,62.55,60.3,57.55,6.9,114.95,62.55,60.3,57.55,790.8 48 | 3000,female,high_school,single,27,delay_1_month,delay_2_months,delay_2_months,no_delay,no_delay,no_delay,2901,2925.6,2812.2,2983.6,163,0,100,8.5,171.4,10.4,0,0 49 | 18000,female,graduate_school,married,35,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,75,0,0,0,0,0,0,0,0,0,0,0 50 | 7000,male,graduate_school,single,26,no_delay,no_delay,delay_2_months,no_delay,no_delay,delay_2_months,1356.15,1364.95,899.25,1011.25,1370.35,541.65,1250,0,150,390,0,125 51 | 7000,female,graduate_school,single,23,duly_paid,no_delay,duly_paid,duly_paid,duly_paid,duly_paid,671.2,900,650.05,290.9,675.3,24.35,500,650.05,292.05,675.3,24.35,350.2 52 | 12000,male,university,single,21,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,10731.4,11057.8,11373.6,11600,11913.1,12213.5,500,500,415.2,500,500,500 53 | 36000,female,university,married,50,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,22643,23187.8,23419.2,19021.1,17355.7,14651.1,910,730,580,560,560,410 54 | 2000,male,high_school,married,47,duly_paid,duly_paid,no_delay,no_delay,no_delay,no_delay,11.35,1017.55,1011.85,966.95,972.5,966.85,1117.95,65.25,62.9,72.95,172.85,85 55 | 10000,male,graduate_school,married,50,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,39,147.3,39,39,39,0,147.3,39,39,39,0,238 56 | 21000,female,university,married,42,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,154.4,79.3,17.15,188.15,43.85,0,79.3,17.15,188.15,43.85,0,20.95 57 | 4000,female,university,single,23,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,4081.25,4077.7,3979.95,2477.6,2475.45,2478.4,160,150,101.9,100,90,90 58 | 33000,female,graduate_school,married,47,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,443.6,216.85,142.7,84.5,0,85.45,217.25,142.7,85.1,0,85.45,12.9 59 | 22000,male,graduate_school,single,24,no_delay,no_delay,no_delay,duly_paid,duly_paid,duly_paid,10560.7,10891.7,11152,830.3,0,8563.2,500,505,830.3,0,8563.2,0 60 | 10500,female,graduate_school,married,36,delay_1_month,duly_paid,duly_paid,no_delay,duly_paid,duly_paid,0,0,25,12.3,78.9,122.2,0,25,0,78.9,122.2,961.6 61 | 2000,female,high_school,married,38,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,3825.7,3890.1,3810.3,3620.7,3313.8,3133.9,170,150.4,120,150,150,100 62 | 1500,male,university,single,36,no_delay,no_delay,no_delay,delay_2_months,no_delay,no_delay,1417.35,1405.4,919.4,649.6,386.8,401.6,151.8,115.45,0,50,100,7.4 63 | 47000,male,university,single,27,delay_2_months,delay_2_months,delay_2_months,delay_2_months,no_delay,no_delay,29657.3,30332,30784.3,47997.8,30514.5,30995.9,1300,1100.1,0,1048.4,1083.8,1036.7 64 | 1500,male,high_school,married,41,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,delay_2_months,19.5,19.5,19.5,12,66,39,19.5,19.5,12,73.5,0,0 65 | 17000,female,graduate_school,single,23,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,6518.5,6559.95,6445,6275.7,6364,6441.95,326.5,243,1125,241,243.85,248.1 66 | 17500,male,high_school,married,43,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,326.5,268.6,186.4,56.9,0,0,268.6,186.4,56.9,0,0,0 67 | 1000,male,university,married,40,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,1645.5,1651.1,1890.2,1876.8,1865,1917.5,160.9,300,66.3,67.6,83.4,78.2 68 | 2500,male,university,single,27,delay_2_months,no_delay,no_delay,no_delay,no_delay,no_delay,2487.9,2422.8,2205.8,1062.35,1003.3,942.9,120.05,112.7,100.2,35.2,35.35,50.2 69 | 1000,female,high_school,single,21,delay_1_month,delay_2_months,no_delay,no_delay,no_delay,no_delay,1915.4,1816.5,1723.3,763,173,0,0,133.3,50,10,0,320 70 | 5000,female,graduate_school,single,22,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,3508.4,3579.6,3093.7,1942,1038.5,130.8,209,200.2,100.2,23.8,101.1,38.9 71 | 2000,male,university,single,28,delay_1_month,delay_2_months,no_delay,no_delay,delay_2_months,no_delay,2023.5,1713.2,1685.6,1687.5,1345.4,1010.4,0,120,100,0,100,1000 72 | 5000,male,university,married,21,no_delay,no_delay,duly_paid,duly_paid,duly_paid,no_delay,10280,0,0,262.4,4556.7,4655.7,0,0,262.4,4556.7,174.7,200 73 | 9500,male,university,single,38,delay_2_months,no_delay,no_delay,no_delay,delay_2_months,delay_2_months,6490.05,6569.15,6718.95,7116.15,7006,7502.6,250,250,500,0,605.9,138.45 74 | 6000,female,university,married,31,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,2910.15,1818.35,1046.7,1034.7,1008.75,971.05,75.3,64.9,50,36.05,40.55,29.95 75 | 4000,male,graduate_school,single,39,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,399.4,198.85,342.4,85.95,68.9,997.1,199.15,342.65,85.95,68.9,997.1,120.9 76 | 7500,female,university,single,24,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,6370.1,5502.5,2577.35,2219.2,1845,1474.85,225,87.25,78.3,60.4,53.85,126.45 77 | 21000,female,graduate_school,single,33,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,duly_paid,144,0,742.2,89.3,608.2,229.3,0,742.2,89.3,608.2,229.3,329.9 78 | 58000,female,graduate_school,married,32,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,7988,8109.45,8306.35,8468.25,8437.75,8398.2,321.1,328.25,297.55,300.3,294.7,297.3 79 | 18000,male,high_school,married,59,no_delay,no_delay,duly_paid,duly_paid,duly_paid,duly_paid,921,848.5,1723.1,177,0,0,601.5,1745.4,177.5,0,0,0 80 | 13500,female,university,married,35,delay_1_month,delay_2_months,delay_2_months,delay_2_months,delay_2_months,delay_2_months,11737.6,11651.8,11464.3,10882.2,11184.5,11070.65,500,400,0,800,400,0 81 | 6000,female,graduate_school,single,23,duly_paid,duly_paid,duly_paid,duly_paid,delay_2_months,duly_paid,1407.2,0,0,30,15,15,0,0,30,0,15,20 82 | 5500,male,high_school,married,42,no_delay,no_delay,no_delay,no_delay,no_delay,no_delay,3090.35,3124.95,3108.1,3030.25,3013.6,3007.25,115,110,115,110,115,115 -------------------------------------------------------------------------------- /databricks/databricks.yml: -------------------------------------------------------------------------------- 1 | bundle: 2 | name: azure_databricks_containers_mlops_example_scenarios 3 | 4 | include: 5 | - resources/*.yml 6 | 7 | workspace: 8 | host: 9 | -------------------------------------------------------------------------------- /databricks/resources/train_register_model.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | curated_dataset_table: 3 | default: hive_metastore.default.credit_default_uci_curated 4 | experiment_name: 5 | default: /azure_databricks_containers_mlops_example_scenarios 6 | model_name: 7 | default: credit-default-uci-custom 8 | 9 | resources: 10 | jobs: 11 | train_register_model_job: 12 | name: train_register_model 13 | job_clusters: 14 | - job_cluster_key: train_register_model_job_cluster 15 | new_cluster: 16 | node_type_id: Standard_D4ads_v5 17 | num_workers: 1 18 | spark_version: 14.3.x-cpu-ml-scala2.12 19 | tasks: 20 | - task_key: train_model 21 | job_cluster_key: train_register_model_job_cluster 22 | notebook_task: 23 | notebook_path: ../src/01-train-model.ipynb 24 | base_parameters: 25 | curated_dataset_table: ${var.curated_dataset_table} 26 | experiment_name: ${var.experiment_name} 27 | - task_key: register_model 28 | job_cluster_key: train_register_model_job_cluster 29 | depends_on: 30 | - task_key: train_model 31 | libraries: 32 | - pypi: 33 | package: alibi-detect==0.12.0 34 | notebook_task: 35 | notebook_path: ../src/02-register-model.ipynb 36 | base_parameters: 37 | curated_dataset_table: ${var.curated_dataset_table} 38 | experiment_name: ${var.experiment_name} 39 | model_name: ${var.model_name} 40 | -------------------------------------------------------------------------------- /databricks/src/00-create-external-table.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "application/vnd.databricks.v1+cell": { 7 | "cellMetadata": {}, 8 | "inputWidgets": {}, 9 | "nuid": "e8e90692-e20e-4bba-8537-2aa489465a31", 10 | "showTitle": false, 11 | "title": "" 12 | } 13 | }, 14 | "source": [ 15 | "# Create external table\n", 16 | "\n", 17 | "This notebook will create an external tables in Unity Catalog based on the `UCI Credit Card Client Default` [dataset](https://archive.ics.uci.edu/dataset/350/default+of+credit+card+clients) that was uploaded during setup.\n" 18 | ] 19 | }, 20 | { 21 | "cell_type": "markdown", 22 | "metadata": { 23 | "application/vnd.databricks.v1+cell": { 24 | "cellMetadata": {}, 25 | "inputWidgets": {}, 26 | "nuid": "73d373cd-e604-4e34-ae75-0175b6dd3933", 27 | "showTitle": false, 28 | "title": "" 29 | } 30 | }, 31 | "source": [ 32 | "#### Define notebook parameters\n" 33 | ] 34 | }, 35 | { 36 | "cell_type": "code", 37 | "execution_count": null, 38 | "metadata": { 39 | "application/vnd.databricks.v1+cell": { 40 | "cellMetadata": { 41 | "byteLimit": 2048000, 42 | "rowLimit": 10000 43 | }, 44 | "inputWidgets": {}, 45 | "nuid": "00270229-6e5d-4ceb-83ec-eef0f7483bb7", 46 | "showTitle": false, 47 | "title": "" 48 | } 49 | }, 50 | "outputs": [], 51 | "source": [ 52 | "dbutils.widgets.text(\"path\", \"dbfs:/FileStore/tables/credit-card-default-uci-curated\")" 53 | ] 54 | }, 55 | { 56 | "cell_type": "markdown", 57 | "metadata": { 58 | "application/vnd.databricks.v1+cell": { 59 | "cellMetadata": {}, 60 | "inputWidgets": {}, 61 | "nuid": "11b73742-9e80-4ce9-9e59-13f266ba884c", 62 | "showTitle": false, 63 | "title": "" 64 | } 65 | }, 66 | "source": [ 67 | "#### Create external table\n" 68 | ] 69 | }, 70 | { 71 | "cell_type": "code", 72 | "execution_count": null, 73 | "metadata": { 74 | "application/vnd.databricks.v1+cell": { 75 | "cellMetadata": { 76 | "byteLimit": 2048000, 77 | "implicitDf": true, 78 | "rowLimit": 10000 79 | }, 80 | "inputWidgets": {}, 81 | "nuid": "d7ec3de5-b1ab-4438-85f9-b91d91b711e1", 82 | "showTitle": false, 83 | "title": "" 84 | } 85 | }, 86 | "outputs": [], 87 | "source": [ 88 | "%sql\n", 89 | "\n", 90 | "DROP TABLE IF EXISTS credit_default_uci_curated;\n", 91 | "\n", 92 | "CREATE TABLE credit_default_uci_curated\n", 93 | " USING csv\n", 94 | " OPTIONS (path \"${path}\", header \"true\", inferSchema \"true\");" 95 | ] 96 | } 97 | ], 98 | "metadata": { 99 | "application/vnd.databricks.v1+notebook": { 100 | "dashboards": [], 101 | "language": "python", 102 | "notebookMetadata": { 103 | "mostRecentlyExecutedCommandWithImplicitDF": { 104 | "commandId": 1068276009428663, 105 | "dataframes": [ 106 | "_sqldf" 107 | ] 108 | }, 109 | "pythonIndentUnit": 4, 110 | "widgetLayout": [ 111 | { 112 | "breakBefore": false, 113 | "name": "path", 114 | "width": 381 115 | } 116 | ] 117 | }, 118 | "notebookName": "00-create-external-table", 119 | "widgets": { 120 | "path": { 121 | "currentValue": "dbfs:/FileStore/tables/credit-card-default-uci-curated", 122 | "nuid": "fdfcee50-89c4-43cc-851a-69f13ccfbc7b", 123 | "typedWidgetInfo": null, 124 | "widgetInfo": { 125 | "defaultValue": "dbfs:/FileStore/tables/credit-card-default-uci-curated", 126 | "label": null, 127 | "name": "path", 128 | "options": { 129 | "autoCreated": null, 130 | "validationRegex": null, 131 | "widgetType": "text" 132 | }, 133 | "widgetType": "text" 134 | } 135 | } 136 | } 137 | }, 138 | "language_info": { 139 | "name": "python" 140 | } 141 | }, 142 | "nbformat": 4, 143 | "nbformat_minor": 0 144 | } 145 | -------------------------------------------------------------------------------- /databricks/src/01-train-model.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "application/vnd.databricks.v1+cell": { 7 | "cellMetadata": { 8 | "byteLimit": 2048000, 9 | "rowLimit": 10000 10 | }, 11 | "inputWidgets": {}, 12 | "nuid": "1c25ea4c-a3f8-4d7a-a605-d883d4278d98", 13 | "showTitle": false, 14 | "title": "" 15 | } 16 | }, 17 | "source": [ 18 | "# Train machine learning model\n", 19 | "\n", 20 | "This notebook outlines a workflow for training a machine learning model with the goal of identifying optimal hyperparameters. The `UCI Credit Card Client Default` [dataset](https://archive.ics.uci.edu/dataset/350/default+of+credit+card+clients) will be used to develop a machine learning model to predict the liklihood of credit default.\n" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": { 26 | "application/vnd.databricks.v1+cell": { 27 | "cellMetadata": { 28 | "byteLimit": 2048000, 29 | "rowLimit": 10000 30 | }, 31 | "inputWidgets": {}, 32 | "nuid": "ce9e41b8-c45b-44b5-8081-739f85051e44", 33 | "showTitle": false, 34 | "title": "" 35 | } 36 | }, 37 | "source": [ 38 | "#### Import dependencies, define notebook parameters and constants\n" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "metadata": { 45 | "application/vnd.databricks.v1+cell": { 46 | "cellMetadata": { 47 | "byteLimit": 2048000, 48 | "rowLimit": 10000 49 | }, 50 | "inputWidgets": {}, 51 | "nuid": "8056838a-a538-4f17-9ee6-ef5d0e71dcf2", 52 | "showTitle": false, 53 | "title": "" 54 | } 55 | }, 56 | "outputs": [], 57 | "source": [ 58 | "import json\n", 59 | "from typing import Dict, Tuple, Union\n", 60 | "\n", 61 | "import mlflow\n", 62 | "import pandas as pd\n", 63 | "from hyperopt import STATUS_OK, fmin, hp, tpe\n", 64 | "from mlflow.models.signature import infer_signature\n", 65 | "from sklearn.compose import ColumnTransformer\n", 66 | "from sklearn.ensemble import RandomForestClassifier\n", 67 | "from sklearn.impute import SimpleImputer\n", 68 | "from sklearn.metrics import (\n", 69 | " accuracy_score,\n", 70 | " f1_score,\n", 71 | " precision_score,\n", 72 | " recall_score,\n", 73 | " roc_auc_score,\n", 74 | ")\n", 75 | "from sklearn.model_selection import train_test_split\n", 76 | "from sklearn.pipeline import Pipeline\n", 77 | "from sklearn.preprocessing import OneHotEncoder" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": null, 83 | "metadata": { 84 | "application/vnd.databricks.v1+cell": { 85 | "cellMetadata": { 86 | "byteLimit": 2048000, 87 | "rowLimit": 10000 88 | }, 89 | "inputWidgets": {}, 90 | "nuid": "c977f935-1edf-497c-91c1-7d19c0cd2b06", 91 | "showTitle": false, 92 | "title": "" 93 | } 94 | }, 95 | "outputs": [], 96 | "source": [ 97 | "# define notebook parameters\n", 98 | "dbutils.widgets.text(\"experiment_name\", \"/online-inference-containers-examples\")\n", 99 | "\n", 100 | "dbutils.widgets.text(\n", 101 | " \"curated_dataset_table\", \"hive_metastore.default.credit_default_uci_curated\"\n", 102 | ")" 103 | ] 104 | }, 105 | { 106 | "cell_type": "code", 107 | "execution_count": null, 108 | "metadata": { 109 | "application/vnd.databricks.v1+cell": { 110 | "cellMetadata": { 111 | "byteLimit": 2048000, 112 | "rowLimit": 10000 113 | }, 114 | "inputWidgets": {}, 115 | "nuid": "dda4b332-f396-479b-aa52-bbd182b08c17", 116 | "showTitle": false, 117 | "title": "" 118 | } 119 | }, 120 | "outputs": [], 121 | "source": [ 122 | "# define target column\n", 123 | "TARGET = [\"default_payment_next_month\"]\n", 124 | "\n", 125 | "# define categorical feature columns\n", 126 | "CATEGORICAL_FEATURES = [\n", 127 | " \"sex\",\n", 128 | " \"education\",\n", 129 | " \"marriage\",\n", 130 | " \"repayment_status_1\",\n", 131 | " \"repayment_status_2\",\n", 132 | " \"repayment_status_3\",\n", 133 | " \"repayment_status_4\",\n", 134 | " \"repayment_status_5\",\n", 135 | " \"repayment_status_6\",\n", 136 | "]\n", 137 | "\n", 138 | "# define numeric feature columns\n", 139 | "NUMERIC_FEATURES = [\n", 140 | " \"credit_limit\",\n", 141 | " \"age\",\n", 142 | " \"bill_amount_1\",\n", 143 | " \"bill_amount_2\",\n", 144 | " \"bill_amount_3\",\n", 145 | " \"bill_amount_4\",\n", 146 | " \"bill_amount_5\",\n", 147 | " \"bill_amount_6\",\n", 148 | " \"payment_amount_1\",\n", 149 | " \"payment_amount_2\",\n", 150 | " \"payment_amount_3\",\n", 151 | " \"payment_amount_4\",\n", 152 | " \"payment_amount_5\",\n", 153 | " \"payment_amount_6\",\n", 154 | "]\n", 155 | "\n", 156 | "# define all features\n", 157 | "FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES" 158 | ] 159 | }, 160 | { 161 | "cell_type": "markdown", 162 | "metadata": { 163 | "application/vnd.databricks.v1+cell": { 164 | "cellMetadata": { 165 | "byteLimit": 2048000, 166 | "rowLimit": 10000 167 | }, 168 | "inputWidgets": {}, 169 | "nuid": "80db3cfe-6b32-44ce-a93d-c95ed42592f0", 170 | "showTitle": false, 171 | "title": "" 172 | } 173 | }, 174 | "source": [ 175 | "#### Define functions to build the model\n" 176 | ] 177 | }, 178 | { 179 | "cell_type": "code", 180 | "execution_count": null, 181 | "metadata": { 182 | "application/vnd.databricks.v1+cell": { 183 | "cellMetadata": { 184 | "byteLimit": 2048000, 185 | "rowLimit": 10000 186 | }, 187 | "inputWidgets": {}, 188 | "nuid": "e5f6cd74-7251-4d5b-b4ab-9547e740793a", 189 | "showTitle": false, 190 | "title": "" 191 | } 192 | }, 193 | "outputs": [], 194 | "source": [ 195 | "def make_classifer_pipeline(params: Dict[str, Union[str, int]]) -> Pipeline:\n", 196 | " \"\"\"Create sklearn pipeline to apply transforms and a final estimator\"\"\"\n", 197 | " # categorical features transformations\n", 198 | " categorical_transformer = Pipeline(\n", 199 | " steps=[\n", 200 | " (\"imputer\", SimpleImputer(strategy=\"constant\", fill_value=\"missing\")),\n", 201 | " (\n", 202 | " \"ohe\",\n", 203 | " OneHotEncoder(\n", 204 | " handle_unknown=\"ignore\",\n", 205 | " ),\n", 206 | " ),\n", 207 | " ]\n", 208 | " )\n", 209 | "\n", 210 | " # numeric features transformations\n", 211 | " numeric_transformer = Pipeline(\n", 212 | " steps=[(\"imputer\", SimpleImputer(strategy=\"median\"))]\n", 213 | " )\n", 214 | "\n", 215 | " # preprocessing pipeline\n", 216 | " preprocessor = ColumnTransformer(\n", 217 | " transformers=[\n", 218 | " (\"categorical\", categorical_transformer, CATEGORICAL_FEATURES),\n", 219 | " (\"numeric\", numeric_transformer, NUMERIC_FEATURES),\n", 220 | " ]\n", 221 | " )\n", 222 | "\n", 223 | " # model training pipeline\n", 224 | " classifer_pipeline = Pipeline(\n", 225 | " [\n", 226 | " (\"preprocessor\", preprocessor),\n", 227 | " (\"classifier\", RandomForestClassifier(**params, n_jobs=-1)),\n", 228 | " ]\n", 229 | " )\n", 230 | "\n", 231 | " return classifer_pipeline" 232 | ] 233 | }, 234 | { 235 | "cell_type": "code", 236 | "execution_count": null, 237 | "metadata": { 238 | "application/vnd.databricks.v1+cell": { 239 | "cellMetadata": { 240 | "byteLimit": 2048000, 241 | "rowLimit": 10000 242 | }, 243 | "inputWidgets": {}, 244 | "nuid": "0db1b591-3790-4233-81b7-9588754f58e0", 245 | "showTitle": false, 246 | "title": "" 247 | } 248 | }, 249 | "outputs": [], 250 | "source": [ 251 | "# define objective function\n", 252 | "def hyperparameter_tuning(params):\n", 253 | " mlflow.sklearn.autolog(silent=True)\n", 254 | "\n", 255 | " with mlflow.start_run(nested=True):\n", 256 | " # read and process curated data\n", 257 | " df = spark.read.table(dbutils.widgets.get(\"curated_dataset_table\")).toPandas()\n", 258 | "\n", 259 | " # split into train and test datasets\n", 260 | " df_train, df_test = train_test_split(\n", 261 | " df[CATEGORICAL_FEATURES + NUMERIC_FEATURES + TARGET],\n", 262 | " test_size=0.20,\n", 263 | " random_state=2024,\n", 264 | " )\n", 265 | "\n", 266 | " # seperate features and target variables\n", 267 | " x_train, y_train = (\n", 268 | " df_train[CATEGORICAL_FEATURES + NUMERIC_FEATURES],\n", 269 | " df_train[TARGET],\n", 270 | " )\n", 271 | " x_test, y_test = (\n", 272 | " df_test[CATEGORICAL_FEATURES + NUMERIC_FEATURES],\n", 273 | " df_test[TARGET],\n", 274 | " )\n", 275 | "\n", 276 | " # train model\n", 277 | " estimator = make_classifer_pipeline(params)\n", 278 | " estimator.fit(x_train, y_train.values.ravel())\n", 279 | "\n", 280 | " # train and model\n", 281 | " estimator = make_classifer_pipeline(params)\n", 282 | " estimator = estimator.fit(x_train, y_train.values.ravel())\n", 283 | " y_predict_proba = estimator.predict_proba(x_test)\n", 284 | "\n", 285 | " # train model\n", 286 | " estimator = make_classifer_pipeline(params)\n", 287 | " estimator.fit(x_train, y_train.values.ravel())\n", 288 | "\n", 289 | " # calculate evaluation metrics\n", 290 | " y_pred = estimator.predict(x_test)\n", 291 | " validation_accuracy_score = accuracy_score(y_test.values.ravel(), y_pred)\n", 292 | " validation_roc_auc_score = roc_auc_score(y_test.values.ravel(), y_pred)\n", 293 | " validation_f1_score = f1_score(y_test.values.ravel(), y_pred)\n", 294 | " validation_precision_score = precision_score(y_test.values.ravel(), y_pred)\n", 295 | " validation_recall_score = recall_score(y_test.values.ravel(), y_pred)\n", 296 | "\n", 297 | " # log evaluation metrics\n", 298 | " mlflow.log_metric(\"validation_accuracy_score\", validation_accuracy_score)\n", 299 | " mlflow.log_metric(\"validation_roc_auc_score\", validation_roc_auc_score)\n", 300 | " mlflow.log_metric(\"validation_f1_score\", validation_f1_score)\n", 301 | " mlflow.log_metric(\"validation_precision_score\", validation_precision_score)\n", 302 | " mlflow.log_metric(\"validation_recall_score\", validation_recall_score)\n", 303 | "\n", 304 | " # log model\n", 305 | " signature = infer_signature(x_train, y_pred)\n", 306 | " mlflow.sklearn.log_model(\n", 307 | " estimator,\n", 308 | " \"model\",\n", 309 | " signature=signature,\n", 310 | " input_example=x_test.iloc[0].to_dict(),\n", 311 | " )\n", 312 | "\n", 313 | " return {\"loss\": -validation_roc_auc_score, \"status\": STATUS_OK}" 314 | ] 315 | }, 316 | { 317 | "cell_type": "code", 318 | "execution_count": null, 319 | "metadata": { 320 | "application/vnd.databricks.v1+cell": { 321 | "cellMetadata": { 322 | "byteLimit": 2048000, 323 | "rowLimit": 10000 324 | }, 325 | "inputWidgets": {}, 326 | "nuid": "0d8bb8fb-9767-411b-acc4-b5ecf273f445", 327 | "showTitle": false, 328 | "title": "" 329 | } 330 | }, 331 | "outputs": [], 332 | "source": [ 333 | "def train_model():\n", 334 | " # set mlflow tracking uri\n", 335 | " mlflow_client = mlflow.tracking.MlflowClient(tracking_uri=\"databricks\")\n", 336 | " mlflow.set_tracking_uri(\"databricks\")\n", 337 | "\n", 338 | " # start model training run\n", 339 | " mlflow.set_experiment(dbutils.widgets.get(\"experiment_name\"))\n", 340 | " with mlflow.start_run(run_name=\"credit-default-uci-train\") as run:\n", 341 | " # define search space\n", 342 | " search_space = {\n", 343 | " \"n_estimators\": hp.choice(\"n_estimators\", range(100, 1000)),\n", 344 | " \"max_depth\": hp.choice(\"max_depth\", range(1, 25)),\n", 345 | " \"criterion\": hp.choice(\"criterion\", [\"gini\", \"entropy\"]),\n", 346 | " }\n", 347 | "\n", 348 | " # hyperparameter tuning\n", 349 | " best_params = fmin(\n", 350 | " fn=hyperparameter_tuning,\n", 351 | " space=search_space,\n", 352 | " algo=tpe.suggest,\n", 353 | " max_evals=10,\n", 354 | " )\n", 355 | "\n", 356 | " # end run\n", 357 | " mlflow.end_run()\n", 358 | "\n", 359 | " return run" 360 | ] 361 | }, 362 | { 363 | "cell_type": "markdown", 364 | "metadata": { 365 | "application/vnd.databricks.v1+cell": { 366 | "cellMetadata": { 367 | "byteLimit": 2048000, 368 | "rowLimit": 10000 369 | }, 370 | "inputWidgets": {}, 371 | "nuid": "71bc4da5-d840-487f-bf52-03797c7b0164", 372 | "showTitle": false, 373 | "title": "" 374 | } 375 | }, 376 | "source": [ 377 | "#### Train the machine learning model\n" 378 | ] 379 | }, 380 | { 381 | "cell_type": "code", 382 | "execution_count": null, 383 | "metadata": { 384 | "application/vnd.databricks.v1+cell": { 385 | "cellMetadata": { 386 | "byteLimit": 2048000, 387 | "rowLimit": 10000 388 | }, 389 | "inputWidgets": {}, 390 | "nuid": "d5d2edb4-363f-483f-813c-fcc0375272c1", 391 | "showTitle": false, 392 | "title": "" 393 | } 394 | }, 395 | "outputs": [], 396 | "source": [ 397 | "# train model\n", 398 | "run = train_model()\n", 399 | "\n", 400 | "# retreive model from best run\n", 401 | "best_run = mlflow.search_runs(\n", 402 | " filter_string=f\"tags.mlflow.parentRunId='{run.info.run_id}'\",\n", 403 | " order_by=[\"metrics.validation_roc_auc_score DESC\"],\n", 404 | ").iloc[0]" 405 | ] 406 | }, 407 | { 408 | "cell_type": "markdown", 409 | "metadata": { 410 | "application/vnd.databricks.v1+cell": { 411 | "cellMetadata": {}, 412 | "inputWidgets": {}, 413 | "nuid": "75a1e085-ec60-4958-9ec0-013ab9409642", 414 | "showTitle": false, 415 | "title": "" 416 | } 417 | }, 418 | "source": [ 419 | "#### Return notebook outputs\n" 420 | ] 421 | }, 422 | { 423 | "cell_type": "code", 424 | "execution_count": null, 425 | "metadata": { 426 | "application/vnd.databricks.v1+cell": { 427 | "cellMetadata": {}, 428 | "inputWidgets": {}, 429 | "nuid": "78018db8-cd0e-45ae-8a7a-3d68c0946fa9", 430 | "showTitle": false, 431 | "title": "" 432 | } 433 | }, 434 | "outputs": [], 435 | "source": [ 436 | "# set best run id for task values\n", 437 | "dbutils.jobs.taskValues.set(key=\"best_run_id\", value=best_run.run_id)" 438 | ] 439 | } 440 | ], 441 | "metadata": { 442 | "application/vnd.databricks.v1+notebook": { 443 | "dashboards": [], 444 | "language": "python", 445 | "notebookMetadata": { 446 | "pythonIndentUnit": 4 447 | }, 448 | "notebookName": "01-train-model", 449 | "widgets": { 450 | "curated_dataset_table": { 451 | "currentValue": "hive_metastore.default.credit_default_uci_curated", 452 | "nuid": "6a1769f3-7d95-4fc3-a19f-fe27769f2a7a", 453 | "typedWidgetInfo": null, 454 | "widgetInfo": { 455 | "defaultValue": "hive_metastore.default.credit_default_uci_curated", 456 | "label": null, 457 | "name": "curated_dataset_table", 458 | "options": { 459 | "autoCreated": null, 460 | "validationRegex": null, 461 | "widgetType": "text" 462 | }, 463 | "widgetType": "text" 464 | } 465 | }, 466 | "experiment_name": { 467 | "currentValue": "/online-inference-containers-examples", 468 | "nuid": "3576f59b-6263-44f1-b065-b050bc08a208", 469 | "typedWidgetInfo": null, 470 | "widgetInfo": { 471 | "defaultValue": "/online-inference-containers", 472 | "label": null, 473 | "name": "experiment_name", 474 | "options": { 475 | "autoCreated": null, 476 | "validationRegex": null, 477 | "widgetType": "text" 478 | }, 479 | "widgetType": "text" 480 | } 481 | } 482 | } 483 | }, 484 | "language_info": { 485 | "name": "python" 486 | } 487 | }, 488 | "nbformat": 4, 489 | "nbformat_minor": 0 490 | } 491 | -------------------------------------------------------------------------------- /databricks/src/02-register-model.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "application/vnd.databricks.v1+cell": { 7 | "cellMetadata": { 8 | "byteLimit": 2048000, 9 | "rowLimit": 10000 10 | }, 11 | "inputWidgets": {}, 12 | "nuid": "58d875ff-2666-4e00-b1e6-e79b67fc8a74", 13 | "showTitle": false, 14 | "title": "" 15 | } 16 | }, 17 | "source": [ 18 | "# Register machine learning model\n", 19 | "\n", 20 | "This notebook outlines a workflow for registering a machine learning model from a MLFlow run. A `python_function` MLFlow model object will be created to perform classification, drift detection and outlier detection.\n" 21 | ] 22 | }, 23 | { 24 | "cell_type": "markdown", 25 | "metadata": { 26 | "application/vnd.databricks.v1+cell": { 27 | "cellMetadata": { 28 | "byteLimit": 2048000, 29 | "rowLimit": 10000 30 | }, 31 | "inputWidgets": {}, 32 | "nuid": "61647b92-b0d0-427a-a833-5afceb5850da", 33 | "showTitle": false, 34 | "title": "" 35 | } 36 | }, 37 | "source": [ 38 | "#### Import dependencies, define notebook parameters and constants\n" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": null, 44 | "metadata": { 45 | "application/vnd.databricks.v1+cell": { 46 | "cellMetadata": { 47 | "byteLimit": 2048000, 48 | "rowLimit": 10000 49 | }, 50 | "inputWidgets": {}, 51 | "nuid": "4519bb7d-d0d5-43d8-aef8-4685782f0766", 52 | "showTitle": false, 53 | "title": "" 54 | } 55 | }, 56 | "outputs": [], 57 | "source": [ 58 | "import os\n", 59 | "import json\n", 60 | "import yaml\n", 61 | "import joblib\n", 62 | "import mlflow\n", 63 | "import pandas as pd\n", 64 | "import importlib.metadata\n", 65 | "\n", 66 | "from mlflow.tracking import MlflowClient\n", 67 | "from alibi_detect.od import IForest\n", 68 | "from alibi_detect.cd import TabularDrift" 69 | ] 70 | }, 71 | { 72 | "cell_type": "code", 73 | "execution_count": null, 74 | "metadata": { 75 | "application/vnd.databricks.v1+cell": { 76 | "cellMetadata": { 77 | "byteLimit": 2048000, 78 | "rowLimit": 10000 79 | }, 80 | "inputWidgets": {}, 81 | "nuid": "83300ef4-ae69-4323-a887-be582a2b5c30", 82 | "showTitle": false, 83 | "title": "" 84 | } 85 | }, 86 | "outputs": [], 87 | "source": [ 88 | "# define notebook parameters\n", 89 | "dbutils.widgets.text(\"model_name\", \"credit-default-uci-custom\")\n", 90 | "\n", 91 | "dbutils.widgets.text(\"experiment_name\", \"/online-inference-containers-examples\")\n", 92 | "\n", 93 | "dbutils.widgets.text(\n", 94 | " \"curated_dataset_table\", \"hive_metastore.default.credit_default_uci_curated\"\n", 95 | ")" 96 | ] 97 | }, 98 | { 99 | "cell_type": "code", 100 | "execution_count": null, 101 | "metadata": { 102 | "application/vnd.databricks.v1+cell": { 103 | "cellMetadata": { 104 | "byteLimit": 2048000, 105 | "rowLimit": 10000 106 | }, 107 | "inputWidgets": {}, 108 | "nuid": "aeedfe4f-aecf-4390-9353-78e7fbdb06a9", 109 | "showTitle": false, 110 | "title": "" 111 | } 112 | }, 113 | "outputs": [], 114 | "source": [ 115 | "# define target column\n", 116 | "TARGET = [\"default_payment_next_month\"]\n", 117 | "\n", 118 | "# define categorical feature columns\n", 119 | "CATEGORICAL_FEATURES = [\n", 120 | " \"sex\",\n", 121 | " \"education\",\n", 122 | " \"marriage\",\n", 123 | " \"repayment_status_1\",\n", 124 | " \"repayment_status_2\",\n", 125 | " \"repayment_status_3\",\n", 126 | " \"repayment_status_4\",\n", 127 | " \"repayment_status_5\",\n", 128 | " \"repayment_status_6\",\n", 129 | "]\n", 130 | "\n", 131 | "# define numeric feature columns\n", 132 | "NUMERIC_FEATURES = [\n", 133 | " \"credit_limit\",\n", 134 | " \"age\",\n", 135 | " \"bill_amount_1\",\n", 136 | " \"bill_amount_2\",\n", 137 | " \"bill_amount_3\",\n", 138 | " \"bill_amount_4\",\n", 139 | " \"bill_amount_5\",\n", 140 | " \"bill_amount_6\",\n", 141 | " \"payment_amount_1\",\n", 142 | " \"payment_amount_2\",\n", 143 | " \"payment_amount_3\",\n", 144 | " \"payment_amount_4\",\n", 145 | " \"payment_amount_5\",\n", 146 | " \"payment_amount_6\",\n", 147 | "]\n", 148 | "\n", 149 | "# define all features\n", 150 | "FEATURES = CATEGORICAL_FEATURES + NUMERIC_FEATURES\n", 151 | "\n", 152 | "# define sample data for inference\n", 153 | "INPUT_SAMPLE = [\n", 154 | " {\n", 155 | " \"sex\": \"male\",\n", 156 | " \"education\": \"university\",\n", 157 | " \"marriage\": \"married\",\n", 158 | " \"repayment_status_1\": \"duly_paid\",\n", 159 | " \"repayment_status_2\": \"duly_paid\",\n", 160 | " \"repayment_status_3\": \"duly_paid\",\n", 161 | " \"repayment_status_4\": \"duly_paid\",\n", 162 | " \"repayment_status_5\": \"no_delay\",\n", 163 | " \"repayment_status_6\": \"no_delay\",\n", 164 | " \"credit_limit\": 18000.0,\n", 165 | " \"age\": 33.0,\n", 166 | " \"bill_amount_1\": 764.95,\n", 167 | " \"bill_amount_2\": 2221.95,\n", 168 | " \"bill_amount_3\": 1131.85,\n", 169 | " \"bill_amount_4\": 5074.85,\n", 170 | " \"bill_amount_5\": 3448.0,\n", 171 | " \"bill_amount_6\": 1419.95,\n", 172 | " \"payment_amount_1\": 2236.5,\n", 173 | " \"payment_amount_2\": 1137.55,\n", 174 | " \"payment_amount_3\": 5084.55,\n", 175 | " \"payment_amount_4\": 111.65,\n", 176 | " \"payment_amount_5\": 306.9,\n", 177 | " \"payment_amount_6\": 805.65,\n", 178 | " }\n", 179 | "]\n", 180 | "\n", 181 | "# define sample response for inference\n", 182 | "OUTPUT_SAMPLE = {\"predictions\": [0.02]}" 183 | ] 184 | }, 185 | { 186 | "cell_type": "markdown", 187 | "metadata": { 188 | "application/vnd.databricks.v1+cell": { 189 | "cellMetadata": { 190 | "byteLimit": 2048000, 191 | "rowLimit": 10000 192 | }, 193 | "inputWidgets": {}, 194 | "nuid": "44bc7566-4bff-4bd2-8753-a8b58a8d7e32", 195 | "showTitle": false, 196 | "title": "" 197 | } 198 | }, 199 | "source": [ 200 | "#### Build drift detector and write models\n" 201 | ] 202 | }, 203 | { 204 | "cell_type": "code", 205 | "execution_count": null, 206 | "metadata": { 207 | "application/vnd.databricks.v1+cell": { 208 | "cellMetadata": { 209 | "byteLimit": 2048000, 210 | "rowLimit": 10000 211 | }, 212 | "inputWidgets": {}, 213 | "nuid": "1eb25547-2788-4488-8e04-785eff1d67ab", 214 | "showTitle": false, 215 | "title": "" 216 | } 217 | }, 218 | "outputs": [], 219 | "source": [ 220 | "# read and process curated data\n", 221 | "df = spark.read.table(dbutils.widgets.get(\"curated_dataset_table\")).toPandas()\n", 222 | "\n", 223 | "# build drift model\n", 224 | "categories_per_feature = {i: None for i in range(len(CATEGORICAL_FEATURES))}\n", 225 | "drift = TabularDrift(\n", 226 | " df[CATEGORICAL_FEATURES + NUMERIC_FEATURES].values,\n", 227 | " p_val=0.05,\n", 228 | " categories_per_feature=categories_per_feature,\n", 229 | ")\n", 230 | "\n", 231 | "# build outlier model\n", 232 | "outlier = IForest(threshold=0.95)\n", 233 | "outlier.fit(df[NUMERIC_FEATURES].values)" 234 | ] 235 | }, 236 | { 237 | "cell_type": "code", 238 | "execution_count": null, 239 | "metadata": { 240 | "application/vnd.databricks.v1+cell": { 241 | "cellMetadata": { 242 | "byteLimit": 2048000, 243 | "rowLimit": 10000 244 | }, 245 | "inputWidgets": {}, 246 | "nuid": "8e7e754e-a3c7-4a12-b4e6-d6f99d40603b", 247 | "showTitle": false, 248 | "title": "" 249 | } 250 | }, 251 | "outputs": [], 252 | "source": [ 253 | "# get best run id from task values\n", 254 | "best_run_id = dbutils.jobs.taskValues.get(\n", 255 | " taskKey=\"train_model\", key=\"best_run_id\", debugValue=\"your-run-id\"\n", 256 | ")\n", 257 | "\n", 258 | "# load best model\n", 259 | "classifier = mlflow.pyfunc.load_model(f\"runs:/{best_run_id}/model\")\n", 260 | "\n", 261 | "# write drift model and outlier model\n", 262 | "os.makedirs(\"/tmp/models\", exist_ok=True)\n", 263 | "joblib.dump(drift, \"/tmp/models/drift.pkl\")\n", 264 | "joblib.dump(outlier, \"/tmp/models/outlier.pkl\")\n", 265 | "\n", 266 | "# write classifier model\n", 267 | "client = MlflowClient()\n", 268 | "classifier_model_path = \"/tmp/models/classifier\"\n", 269 | "os.makedirs(classifier_model_path, exist_ok=True)\n", 270 | "client.download_artifacts(best_run_id, \"model\", classifier_model_path)" 271 | ] 272 | }, 273 | { 274 | "cell_type": "markdown", 275 | "metadata": { 276 | "application/vnd.databricks.v1+cell": { 277 | "cellMetadata": {}, 278 | "inputWidgets": {}, 279 | "nuid": "6a68fb1c-3fac-44db-9b95-6e78e1ef4826", 280 | "showTitle": false, 281 | "title": "" 282 | } 283 | }, 284 | "source": [ 285 | "#### Create custom MLFlow Pyfunc model\n" 286 | ] 287 | }, 288 | { 289 | "cell_type": "code", 290 | "execution_count": null, 291 | "metadata": { 292 | "application/vnd.databricks.v1+cell": { 293 | "cellMetadata": { 294 | "byteLimit": 2048000, 295 | "rowLimit": 10000 296 | }, 297 | "inputWidgets": {}, 298 | "nuid": "04d2f883-9af1-4e34-ae72-4ddc8c6e8ed4", 299 | "showTitle": false, 300 | "title": "" 301 | } 302 | }, 303 | "outputs": [], 304 | "source": [ 305 | "class CustomModel(mlflow.pyfunc.PythonModel):\n", 306 | " \"\"\"\n", 307 | " Custom model for classification outlier and feature drift detection.\n", 308 | " \"\"\"\n", 309 | "\n", 310 | " def __init__(\n", 311 | " self, categorical_feature_names: list[str], numeric_feature_names: list[str]\n", 312 | " ):\n", 313 | " self.categorical_features = categorical_feature_names\n", 314 | " self.numeric_features = numeric_feature_names\n", 315 | " self.all_features = categorical_feature_names + numeric_feature_names\n", 316 | "\n", 317 | " def load_context(self, context):\n", 318 | " self.classifier = joblib.load(\n", 319 | " os.path.join(\n", 320 | " context.artifacts[\"artifacts_path\"], \"classifier/model/model.pkl\"\n", 321 | " )\n", 322 | " )\n", 323 | " self.drift = joblib.load(\n", 324 | " os.path.join(context.artifacts[\"artifacts_path\"], \"drift.pkl\")\n", 325 | " )\n", 326 | " self.outliers = joblib.load(\n", 327 | " os.path.join(context.artifacts[\"artifacts_path\"], \"outlier.pkl\")\n", 328 | " )\n", 329 | "\n", 330 | " def predict(self, context, model_input):\n", 331 | " # convert to pandas dataframe\n", 332 | " df = pd.DataFrame(model_input)\n", 333 | "\n", 334 | " # generate predictions, drift results, and outlier results\n", 335 | " predictions = self.classifier.predict_proba(df[self.all_features])[\n", 336 | " :, 1\n", 337 | " ].tolist()\n", 338 | " drift_results = self.drift.predict(df[self.all_features].values)\n", 339 | " outlier_results = self.outliers.predict(df[self.numeric_features].values)\n", 340 | "\n", 341 | " # format response\n", 342 | " response = {\n", 343 | " \"predictions\": predictions,\n", 344 | " \"outliers\": outlier_results[\"data\"][\"is_outlier\"].tolist(),\n", 345 | " \"feature_drift_batch\": dict(\n", 346 | " zip(\n", 347 | " CATEGORICAL_FEATURES + NUMERIC_FEATURES,\n", 348 | " (1 - drift_results[\"data\"][\"p_val\"]).tolist(),\n", 349 | " )\n", 350 | " ),\n", 351 | " }\n", 352 | "\n", 353 | " return response" 354 | ] 355 | }, 356 | { 357 | "cell_type": "markdown", 358 | "metadata": { 359 | "application/vnd.databricks.v1+cell": { 360 | "cellMetadata": {}, 361 | "inputWidgets": {}, 362 | "nuid": "23761f9c-b3ac-4059-83d2-7c616b3fdba3", 363 | "showTitle": false, 364 | "title": "" 365 | } 366 | }, 367 | "source": [ 368 | "#### Register custom MLFlow model\n" 369 | ] 370 | }, 371 | { 372 | "cell_type": "code", 373 | "execution_count": null, 374 | "metadata": { 375 | "application/vnd.databricks.v1+cell": { 376 | "cellMetadata": { 377 | "byteLimit": 2048000, 378 | "rowLimit": 10000 379 | }, 380 | "inputWidgets": {}, 381 | "nuid": "ed93f0b0-f257-4bd9-a618-1ace25996397", 382 | "showTitle": false, 383 | "title": "" 384 | } 385 | }, 386 | "outputs": [], 387 | "source": [ 388 | "# load base conda file\n", 389 | "with open(\"/tmp/models/classifier/model/conda.yaml\", \"r\") as f:\n", 390 | " base_conda_env = yaml.safe_load(f)\n", 391 | "\n", 392 | "# define extra pip dependencies\n", 393 | "extra_pip_dependencies = [\n", 394 | " f\"{library}=={importlib.metadata.version(library)}\"\n", 395 | " for library in [\"alibi-detect\", \"joblib\", \"numpy\", \"pandas\"]\n", 396 | "]\n", 397 | "\n", 398 | "# update base conda file\n", 399 | "updated_conda_env = base_conda_env.copy()\n", 400 | "updated_conda_env[\"dependencies\"][-1][\"pip\"] = (\n", 401 | " base_conda_env[\"dependencies\"][-1][\"pip\"] + extra_pip_dependencies\n", 402 | ")" 403 | ] 404 | }, 405 | { 406 | "cell_type": "code", 407 | "execution_count": null, 408 | "metadata": { 409 | "application/vnd.databricks.v1+cell": { 410 | "cellMetadata": { 411 | "byteLimit": 2048000, 412 | "rowLimit": 10000 413 | }, 414 | "inputWidgets": {}, 415 | "nuid": "a05545b3-c9f1-4b87-8328-c7fcc3e68ec5", 416 | "showTitle": false, 417 | "title": "" 418 | } 419 | }, 420 | "outputs": [], 421 | "source": [ 422 | "mlflow.set_experiment(dbutils.widgets.get(\"experiment_name\"))\n", 423 | "with mlflow.start_run(run_name=\"credit-default-uci-register\") as run:\n", 424 | " # create instance of custom model\n", 425 | " model_artifact = CustomModel(\n", 426 | " categorical_feature_names=CATEGORICAL_FEATURES,\n", 427 | " numeric_feature_names=NUMERIC_FEATURES,\n", 428 | " )\n", 429 | "\n", 430 | " # log model\n", 431 | " mlflow.pyfunc.log_model(\n", 432 | " artifact_path=\"model\",\n", 433 | " python_model=model_artifact,\n", 434 | " artifacts={\"artifacts_path\": \"/tmp/models\"},\n", 435 | " conda_env=updated_conda_env,\n", 436 | " input_example=INPUT_SAMPLE,\n", 437 | " signature=False,\n", 438 | " )\n", 439 | "\n", 440 | " mlflow.end_run()" 441 | ] 442 | }, 443 | { 444 | "cell_type": "code", 445 | "execution_count": null, 446 | "metadata": { 447 | "application/vnd.databricks.v1+cell": { 448 | "cellMetadata": { 449 | "byteLimit": 2048000, 450 | "rowLimit": 10000 451 | }, 452 | "inputWidgets": {}, 453 | "nuid": "5088acec-45d8-4750-b2d4-1b37ad68aa02", 454 | "showTitle": false, 455 | "title": "" 456 | } 457 | }, 458 | "outputs": [], 459 | "source": [ 460 | "# get best run id from task values\n", 461 | "best_run_id = dbutils.jobs.taskValues.get(\n", 462 | " taskKey=\"train_model\", key=\"best_run_id\", debugValue=\"your-run-id\"\n", 463 | ")\n", 464 | "\n", 465 | "# register drift model to MLFlow model registry\n", 466 | "registered_model = mlflow.register_model(\n", 467 | " f\"runs:/{run.info.run_id}/model\",\n", 468 | " dbutils.widgets.get(\"model_name\"),\n", 469 | " tags={\"best_classifier_model_run_id\": best_run_id},\n", 470 | ")" 471 | ] 472 | }, 473 | { 474 | "cell_type": "markdown", 475 | "metadata": { 476 | "application/vnd.databricks.v1+cell": { 477 | "cellMetadata": {}, 478 | "inputWidgets": {}, 479 | "nuid": "19e6e87e-4e0d-4eb9-8475-2be9a0481eaa", 480 | "showTitle": false, 481 | "title": "" 482 | } 483 | }, 484 | "source": [ 485 | "#### Return notebook outputs\n" 486 | ] 487 | }, 488 | { 489 | "cell_type": "code", 490 | "execution_count": null, 491 | "metadata": { 492 | "application/vnd.databricks.v1+cell": { 493 | "cellMetadata": {}, 494 | "inputWidgets": {}, 495 | "nuid": "8e39f375-cb5a-4809-8cbb-77ef65211f5c", 496 | "showTitle": false, 497 | "title": "" 498 | } 499 | }, 500 | "outputs": [], 501 | "source": [ 502 | "# return notebook output\n", 503 | "model_uri = f\"models:/{registered_model.name}/{registered_model.version}\"\n", 504 | "dbutils.notebook.exit(model_uri)" 505 | ] 506 | } 507 | ], 508 | "metadata": { 509 | "application/vnd.databricks.v1+notebook": { 510 | "dashboards": [], 511 | "language": "python", 512 | "notebookMetadata": { 513 | "pythonIndentUnit": 4 514 | }, 515 | "notebookName": "02-register-model", 516 | "widgets": { 517 | "best_run_id": { 518 | "currentValue": "a4b9e7fb2ed6424587282825c13fb92b", 519 | "nuid": "0c44fa31-9c1f-4bdc-a26d-fefafed1f949", 520 | "typedWidgetInfo": null, 521 | "widgetInfo": { 522 | "defaultValue": "", 523 | "label": null, 524 | "name": "best_run_id", 525 | "options": { 526 | "autoCreated": null, 527 | "validationRegex": null, 528 | "widgetType": "text" 529 | }, 530 | "widgetType": "text" 531 | } 532 | }, 533 | "curated_dataset_table": { 534 | "currentValue": "hive_metastore.default.credit_default_uci_curated", 535 | "nuid": "de532abb-35dc-471e-a638-99e89136e3f1", 536 | "typedWidgetInfo": null, 537 | "widgetInfo": { 538 | "defaultValue": "hive_metastore.default.credit_default_uci_curated", 539 | "label": null, 540 | "name": "curated_dataset_table", 541 | "options": { 542 | "autoCreated": null, 543 | "validationRegex": null, 544 | "widgetType": "text" 545 | }, 546 | "widgetType": "text" 547 | } 548 | }, 549 | "experiment_name": { 550 | "currentValue": "/online-inference-containers-examples", 551 | "nuid": "1b06d156-90f3-4241-a342-8eb65c599f46", 552 | "typedWidgetInfo": null, 553 | "widgetInfo": { 554 | "defaultValue": "/online-inference-containers-examples", 555 | "label": null, 556 | "name": "experiment_name", 557 | "options": { 558 | "autoCreated": null, 559 | "validationRegex": null, 560 | "widgetType": "text" 561 | }, 562 | "widgetType": "text" 563 | } 564 | }, 565 | "model_name": { 566 | "currentValue": "credit-default-uci-custom", 567 | "nuid": "7158a77e-3fce-4a95-9d99-2c3a16e88d48", 568 | "typedWidgetInfo": null, 569 | "widgetInfo": { 570 | "defaultValue": "credit-default-uci-custom", 571 | "label": null, 572 | "name": "model_name", 573 | "options": { 574 | "autoCreated": null, 575 | "validationRegex": null, 576 | "widgetType": "text" 577 | }, 578 | "widgetType": "text" 579 | } 580 | } 581 | } 582 | }, 583 | "language_info": { 584 | "name": "python" 585 | } 586 | }, 587 | "nbformat": 4, 588 | "nbformat_minor": 0 589 | } 590 | -------------------------------------------------------------------------------- /infrastructure/main.bicep: -------------------------------------------------------------------------------- 1 | targetScope = 'subscription' 2 | 3 | //******************************************************** 4 | // Parameters 5 | //******************************************************** 6 | 7 | @description('Resource group name') 8 | param resourceGroupName string = 'rg-example-scenario-azure-databricks-online-inference-containers' 9 | 10 | @description('Databricks managed resource group name') 11 | param mrgDatabricksName string = 'rgm-example-scenario-azure-databricks-online-inference-containers-databricks' 12 | 13 | @description('Kubernetes managed resource group name') 14 | param mrgKubernetesName string = 'rgm-example-scenario-azure-databricks-online-inference-containers-kubernetes' 15 | 16 | @description('Location for resources') 17 | param location string = 'australiaeast' 18 | 19 | @description('Deploy Container Apps Environment') 20 | param deployContainerAppsEnvironment bool = true 21 | 22 | @description('Deploy Kubernetes service') 23 | param deployKubernetesService bool = true 24 | 25 | //******************************************************** 26 | // Variables 27 | //******************************************************** 28 | 29 | var serviceSuffix = substring(uniqueString(resourceGroupName), 0, 5) 30 | 31 | var resources = { 32 | applicationInsightsName: 'appi01${serviceSuffix}' 33 | containerRegistryName: 'cr01${serviceSuffix}' 34 | databricksName: 'dbw01${serviceSuffix}' 35 | logAnalyticsWorkspaceName: 'log01${serviceSuffix}' 36 | storageAccountName: 'st01${serviceSuffix}' 37 | userAssignedIdentityName: 'id01${serviceSuffix}' 38 | containerAppEnvironmnetStagingName: 'cae01${serviceSuffix}' 39 | containerAppEnvironmnetProductionName: 'cae02${serviceSuffix}' 40 | kubernetesServiceStagingName: 'aks01${serviceSuffix}' 41 | kubernetesServiceProductionName: 'aks02${serviceSuffix}' 42 | } 43 | 44 | //******************************************************** 45 | // Resources 46 | //******************************************************** 47 | 48 | resource resourceGroup 'Microsoft.Resources/resourceGroups@2023-07-01' = { 49 | name: resourceGroupName 50 | location: location 51 | } 52 | 53 | // ******************************************************** 54 | // Modules 55 | // ******************************************************** 56 | 57 | module userAssignedIdentity './modules/user-assigned-identity.bicep' = { 58 | name: '${resources.userAssignedIdentityName}-deployment' 59 | scope: resourceGroup 60 | params: { 61 | name: resources.userAssignedIdentityName 62 | location: location 63 | tags: { 64 | environment: 'shared' 65 | } 66 | } 67 | } 68 | 69 | module storageAccount './modules/storage-account.bicep' = { 70 | name: '${resources.storageAccountName}-deployment' 71 | scope: resourceGroup 72 | params: { 73 | name: resources.storageAccountName 74 | location: location 75 | tags: { 76 | environment: 'shared' 77 | } 78 | } 79 | } 80 | 81 | module logAnalyticsWorkspace './modules/log-analytics-workspace.bicep' = { 82 | name: '${resources.logAnalyticsWorkspaceName}-deployment' 83 | scope: resourceGroup 84 | params: { 85 | name: resources.logAnalyticsWorkspaceName 86 | location: location 87 | tags: { 88 | environment: 'shared' 89 | } 90 | storageAccountId: storageAccount.outputs.id 91 | } 92 | } 93 | 94 | module applicationInsights './modules/application-insights.bicep' = { 95 | name: '${resources.applicationInsightsName}-deployment' 96 | scope: resourceGroup 97 | params: { 98 | name: resources.applicationInsightsName 99 | location: location 100 | tags: { 101 | environment: 'shared' 102 | } 103 | logAnalyticsWorkspaceId: logAnalyticsWorkspace.outputs.id 104 | } 105 | } 106 | 107 | module containerRegistry './modules/container-registry.bicep' = { 108 | name: '${resources.containerRegistryName}-deployment' 109 | scope: resourceGroup 110 | params: { 111 | name: resources.containerRegistryName 112 | location: location 113 | tags: { 114 | environment: 'shared' 115 | } 116 | logAnalyticsWorkspaceId: logAnalyticsWorkspace.outputs.id 117 | roles: [ 118 | { 119 | principalId: userAssignedIdentity.outputs.principalId 120 | id: '7f951dda-4ed3-4680-a7ca-43fe172d538d' // ACR Pull role 121 | } 122 | ] 123 | } 124 | } 125 | 126 | module databricks './modules/databricks.bicep' = { 127 | name: '${resources.databricksName}-deployment' 128 | scope: resourceGroup 129 | params: { 130 | name: resources.databricksName 131 | location: location 132 | tags: { 133 | environment: 'shared' 134 | } 135 | managedResourceGroupName: mrgDatabricksName 136 | logAnalyticsWorkspaceId: logAnalyticsWorkspace.outputs.id 137 | } 138 | } 139 | 140 | var containerAppEnvironments = [ 141 | { name: resources.containerAppEnvironmnetStagingName, environment: 'staging' } 142 | { name: resources.containerAppEnvironmnetProductionName, environment: 'production' } 143 | ] 144 | 145 | module containerAppsEnvironment './modules/container-app-environment.bicep' = [ 146 | for containerAppEnvironment in containerAppEnvironments: if (deployContainerAppsEnvironment) { 147 | name: '${containerAppEnvironment.name}-deployment' 148 | scope: resourceGroup 149 | params: { 150 | name: containerAppEnvironment.name 151 | location: location 152 | tags: { 153 | environment: containerAppEnvironment.environment 154 | } 155 | logAnalyticsWorkspaceName: logAnalyticsWorkspace.outputs.name 156 | logAnalyticsWorkspaceResourceGroupName: resourceGroup.name 157 | } 158 | } 159 | ] 160 | 161 | var kubernetesServices = [ 162 | { name: resources.kubernetesServiceStagingName, environment: 'staging' } 163 | { name: resources.kubernetesServiceProductionName, environment: 'production' } 164 | ] 165 | 166 | module kubernetesService './modules/kubernetes-service.bicep' = [ 167 | for kubernetesService in kubernetesServices: if (deployKubernetesService) { 168 | name: '${kubernetesService.name}-deployment' 169 | scope: resourceGroup 170 | params: { 171 | name: kubernetesService.name 172 | location: location 173 | tags: { 174 | environment: kubernetesService.environment 175 | } 176 | nodeResourceGroup: kubernetesService.environment == 'staging' 177 | ? '${mrgKubernetesName}-01' 178 | : '${mrgKubernetesName}-02' 179 | logAnalyticsWorkspaceId: logAnalyticsWorkspace.outputs.id 180 | } 181 | } 182 | ] 183 | 184 | //******************************************************** 185 | // Outputs 186 | //******************************************************** 187 | 188 | output storageAccountName string = storageAccount.outputs.name 189 | output logAnalyticsWorkspaceName string = logAnalyticsWorkspace.outputs.name 190 | output applicationInsightsName string = applicationInsights.outputs.name 191 | output containerRegistryName string = containerRegistry.outputs.name 192 | output databricksName string = databricks.outputs.name 193 | output databricksHostname string = databricks.outputs.hostname 194 | output userAssignedIdentityName string = userAssignedIdentity.outputs.name 195 | output containerAppEnvironmnetStagingName string = containerAppsEnvironment[0].outputs.name 196 | output containerAppEnvironmnetProductionName string = containerAppsEnvironment[1].outputs.name 197 | output kubernetesServiceStagingName string = kubernetesService[0].outputs.name 198 | output kubernetesServiceProductionName string = kubernetesService[1].outputs.name 199 | -------------------------------------------------------------------------------- /infrastructure/modules/application-insights.bicep: -------------------------------------------------------------------------------- 1 | //******************************************************** 2 | // Parameters 3 | //******************************************************** 4 | 5 | @description('Name of the Application Insights service') 6 | param name string 7 | 8 | @description('Location for Application Insights service') 9 | param location string = resourceGroup().location 10 | 11 | @description('Tags for the Application Insights service') 12 | param tags object = {} 13 | 14 | @description('Role assignments for the Application Insights service') 15 | param roles array = [] 16 | 17 | @description('Log Analytics workspace ID for diagnostics') 18 | param logAnalyticsWorkspaceId string = '' 19 | 20 | //******************************************************** 21 | // Resources 22 | //******************************************************** 23 | 24 | resource appiNew 'Microsoft.Insights/components@2020-02-02' = { 25 | name: name 26 | location: location 27 | tags: tags 28 | kind: 'web' 29 | properties: { 30 | Application_Type: 'web' 31 | Flow_Type: 'Bluefield' 32 | IngestionMode: 'LogAnalytics' 33 | RetentionInDays: 30 34 | WorkspaceResourceId: logAnalyticsWorkspaceId 35 | } 36 | } 37 | 38 | resource roleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [ 39 | for role in roles: { 40 | name: guid(name, role.principalId, role.id) 41 | scope: appiNew 42 | properties: { 43 | roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', role.id) 44 | principalId: role.principalId 45 | principalType: contains(role, 'type') ? role.type : 'ServicePrincipal' 46 | } 47 | } 48 | ] 49 | 50 | //******************************************************** 51 | // Outputs 52 | //******************************************************** 53 | 54 | output name string = appiNew.name 55 | output id string = appiNew.id 56 | -------------------------------------------------------------------------------- /infrastructure/modules/container-app-environment.bicep: -------------------------------------------------------------------------------- 1 | //******************************************************** 2 | // Parameters 3 | //******************************************************** 4 | 5 | @description('Name of the Application Insights service') 6 | param name string 7 | 8 | @description('Location for Application Insights service') 9 | param location string = resourceGroup().location 10 | 11 | @description('Tags for the Application Insights service') 12 | param tags object = {} 13 | 14 | @description('Role assignments for the Application Insights service') 15 | param roles array = [] 16 | 17 | @description('Log Analytics workspace name') 18 | param logAnalyticsWorkspaceName string = '' 19 | 20 | @description('Log Analytics workspace resource group name') 21 | param logAnalyticsWorkspaceResourceGroupName string = resourceGroup().name 22 | 23 | //******************************************************** 24 | // Resources 25 | //******************************************************** 26 | 27 | resource logExisting 'Microsoft.OperationalInsights/workspaces@2022-10-01' existing = { 28 | scope: resourceGroup(logAnalyticsWorkspaceResourceGroupName) 29 | name: logAnalyticsWorkspaceName 30 | } 31 | 32 | resource caeNew 'Microsoft.App/managedEnvironments@2023-05-01' = { 33 | name: name 34 | location: location 35 | tags: tags 36 | properties: { 37 | appLogsConfiguration: { 38 | destination: 'log-analytics' 39 | logAnalyticsConfiguration: { 40 | customerId: logExisting.properties.customerId 41 | sharedKey: logExisting.listKeys().primarySharedKey 42 | } 43 | } 44 | } 45 | } 46 | 47 | resource roleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [ 48 | for role in roles: { 49 | name: guid(name, role.principalId, role.id) 50 | scope: caeNew 51 | properties: { 52 | roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', role.id) 53 | principalId: role.principalId 54 | principalType: contains(role, 'type') ? role.type : 'ServicePrincipal' 55 | } 56 | } 57 | ] 58 | 59 | //******************************************************** 60 | // Outputs 61 | //******************************************************** 62 | 63 | output name string = caeNew.name 64 | output id string = caeNew.id 65 | -------------------------------------------------------------------------------- /infrastructure/modules/container-registry.bicep: -------------------------------------------------------------------------------- 1 | //******************************************************** 2 | // Parameters 3 | //******************************************************** 4 | 5 | @description('Name of the Container Registry service') 6 | param name string 7 | 8 | @description('Location for Container Registry service') 9 | param location string = resourceGroup().location 10 | 11 | @description('Tags for the Container Registry service') 12 | param tags object = {} 13 | 14 | @description('Role assignments for the Container Registry service') 15 | param roles array = [] 16 | 17 | @description('Log Analytics workspace ID for diagnostics') 18 | param logAnalyticsWorkspaceId string = '' 19 | 20 | //******************************************************** 21 | // Resources 22 | //******************************************************** 23 | 24 | resource crNew 'Microsoft.ContainerRegistry/registries@2022-12-01' = { 25 | name: name 26 | location: location 27 | tags: tags 28 | sku: { 29 | name: 'Standard' 30 | } 31 | properties: { 32 | adminUserEnabled: true 33 | } 34 | } 35 | 36 | resource diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { 37 | name: 'all-logs-all-metrics' 38 | scope: crNew 39 | properties: { 40 | workspaceId: logAnalyticsWorkspaceId 41 | logs: [ 42 | { 43 | categoryGroup: 'allLogs' 44 | enabled: true 45 | } 46 | ] 47 | metrics: [ 48 | { 49 | category: 'AllMetrics' 50 | enabled: true 51 | } 52 | ] 53 | } 54 | } 55 | 56 | resource roleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [ 57 | for role in roles: { 58 | name: guid(name, role.principalId, role.id) 59 | scope: crNew 60 | properties: { 61 | roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', role.id) 62 | principalId: role.principalId 63 | principalType: contains(role, 'type') ? role.type : 'ServicePrincipal' 64 | } 65 | } 66 | ] 67 | 68 | //******************************************************** 69 | // Outputs 70 | //******************************************************** 71 | 72 | output name string = crNew.name 73 | output id string = crNew.id 74 | -------------------------------------------------------------------------------- /infrastructure/modules/databricks.bicep: -------------------------------------------------------------------------------- 1 | //******************************************************** 2 | // Parameters 3 | //******************************************************** 4 | 5 | @description('Name of the Databricks service') 6 | param name string 7 | 8 | @description('Managed resource group for the Databricks service') 9 | param managedResourceGroupName string 10 | 11 | @description('Location for the Databricks service') 12 | param location string = resourceGroup().location 13 | 14 | @description('Tags for the Databricks service') 15 | param tags object = {} 16 | 17 | @description('Role assignments for the Databricks service') 18 | param roles array = [] 19 | 20 | @description('Log Analytics workspace ID for diagnostics') 21 | param logAnalyticsWorkspaceId string = '' 22 | 23 | //******************************************************** 24 | // Resources 25 | //******************************************************** 26 | 27 | resource managedRg 'Microsoft.Resources/resourceGroups@2020-06-01' existing = { 28 | scope: subscription() 29 | name: managedResourceGroupName 30 | } 31 | 32 | resource dbwNew 'Microsoft.Databricks/workspaces@2023-02-01' = { 33 | name: name 34 | location: location 35 | tags: tags 36 | sku: { 37 | name: 'premium' 38 | } 39 | properties: { 40 | parameters: {} 41 | managedResourceGroupId: managedRg.id 42 | } 43 | } 44 | 45 | resource diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { 46 | name: 'all-logs-all-metrics' 47 | scope: dbwNew 48 | properties: { 49 | workspaceId: logAnalyticsWorkspaceId 50 | logs: [ 51 | { 52 | categoryGroup: 'allLogs' 53 | enabled: true 54 | } 55 | ] 56 | } 57 | } 58 | 59 | resource roleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [ 60 | for role in roles: { 61 | name: guid(name, role.principalId, role.id) 62 | scope: dbwNew 63 | properties: { 64 | roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', role.id) 65 | principalId: role.principalId 66 | principalType: contains(role, 'type') ? role.type : 'ServicePrincipal' 67 | } 68 | } 69 | ] 70 | 71 | //******************************************************** 72 | // Outputs 73 | //******************************************************** 74 | 75 | output name string = dbwNew.name 76 | output id string = dbwNew.id 77 | output hostname string = dbwNew.properties.workspaceUrl 78 | -------------------------------------------------------------------------------- /infrastructure/modules/kubernetes-service.bicep: -------------------------------------------------------------------------------- 1 | //******************************************************** 2 | // Parameters 3 | //******************************************************** 4 | 5 | @description('Name of the Kubernetes Service service') 6 | param name string 7 | 8 | @description('Location for Kubernetes Service service') 9 | param location string = resourceGroup().location 10 | 11 | @description('Tags for the Kubernetes Service service') 12 | param tags object = {} 13 | 14 | @description('Role assignments for the Kubernetes Service service') 15 | param roles array = [] 16 | 17 | @description('Managed resource group for the node resources') 18 | param nodeResourceGroup string 19 | 20 | @description('Log Analytics workspace ID for diagnostics') 21 | param logAnalyticsWorkspaceId string 22 | 23 | //******************************************************** 24 | // Resources 25 | //******************************************************** 26 | 27 | resource aksNew 'Microsoft.ContainerService/managedClusters@2024-01-01' = { 28 | name: name 29 | location: location 30 | tags: tags 31 | sku: { 32 | name: 'Base' 33 | tier: 'Free' 34 | } 35 | identity: { 36 | type: 'SystemAssigned' 37 | } 38 | properties: { 39 | dnsPrefix: '${name}-dns' 40 | agentPoolProfiles: [ 41 | { 42 | name: 'agentpool' 43 | osDiskSizeGB: 128 44 | count: 2 45 | vmSize: 'Standard_B2s' 46 | osType: 'Linux' 47 | mode: 'System' 48 | osSKU: 'AzureLinux' 49 | } 50 | ] 51 | enableRBAC: true 52 | nodeResourceGroup: nodeResourceGroup 53 | addonProfiles: { 54 | omsagent: { 55 | enabled: true 56 | config: { 57 | logAnalyticsWorkspaceResourceID: logAnalyticsWorkspaceId 58 | useAADAuth: 'true' 59 | } 60 | } 61 | } 62 | } 63 | } 64 | 65 | resource roleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [ 66 | for role in roles: { 67 | name: guid(name, role.principalId, role.id) 68 | scope: aksNew 69 | properties: { 70 | roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', role.id) 71 | principalId: role.principalId 72 | principalType: contains(role, 'type') ? role.type : 'ServicePrincipal' 73 | } 74 | } 75 | ] 76 | 77 | //******************************************************** 78 | // Outputs 79 | //******************************************************** 80 | 81 | output name string = aksNew.name 82 | output id string = aksNew.id 83 | -------------------------------------------------------------------------------- /infrastructure/modules/log-analytics-workspace.bicep: -------------------------------------------------------------------------------- 1 | //******************************************************** 2 | // Parameters 3 | //******************************************************** 4 | 5 | @description('Name of the Log Analytics workspace') 6 | param name string 7 | 8 | @description('Location for Log Analytics workspace') 9 | param location string = resourceGroup().location 10 | 11 | @description('Tags for the Log Analytics workspace') 12 | param tags object = {} 13 | 14 | @description('Role assignments for the Log Analytics workspace') 15 | param roles array = [] 16 | 17 | @description('Storage account ID to link to the Log Analytics workspace') 18 | param storageAccountId string = '' 19 | 20 | //******************************************************** 21 | // Resources 22 | //******************************************************** 23 | 24 | resource logNew 'Microsoft.OperationalInsights/workspaces@2022-10-01' = { 25 | name: name 26 | location: location 27 | tags: tags 28 | properties: { 29 | retentionInDays: 30 30 | sku: { 31 | name: 'Standalone' 32 | } 33 | } 34 | 35 | resource linkedStAlerts 'linkedStorageAccounts@2020-08-01' = { 36 | name: 'Alerts' 37 | properties: { 38 | storageAccountIds: [ 39 | storageAccountId 40 | ] 41 | } 42 | } 43 | 44 | resource linkedStCustomLogs 'linkedStorageAccounts@2020-08-01' = { 45 | name: 'CustomLogs' 46 | properties: { 47 | storageAccountIds: [ 48 | storageAccountId 49 | ] 50 | } 51 | } 52 | 53 | resource linkedStIngestion 'linkedStorageAccounts@2020-08-01' = { 54 | name: 'Ingestion' 55 | properties: { 56 | storageAccountIds: [ 57 | storageAccountId 58 | ] 59 | } 60 | } 61 | 62 | resource linkedStQuery 'linkedStorageAccounts@2020-08-01' = { 63 | name: 'Query' 64 | properties: { 65 | storageAccountIds: [ 66 | storageAccountId 67 | ] 68 | } 69 | } 70 | } 71 | 72 | resource roleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [ 73 | for role in roles: { 74 | name: guid(name, role.principalId, role.id) 75 | scope: logNew 76 | properties: { 77 | roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', role.id) 78 | principalId: role.principalId 79 | principalType: contains(role, 'type') ? role.type : 'ServicePrincipal' 80 | } 81 | } 82 | ] 83 | 84 | //******************************************************** 85 | // Outputs 86 | //******************************************************** 87 | 88 | output name string = logNew.name 89 | output id string = logNew.id 90 | -------------------------------------------------------------------------------- /infrastructure/modules/storage-account.bicep: -------------------------------------------------------------------------------- 1 | //******************************************************** 2 | // Parameters 3 | //******************************************************** 4 | 5 | @description('Name of the Storage service') 6 | param name string 7 | 8 | @description('Location for Storage service') 9 | param location string = resourceGroup().location 10 | 11 | @description('Tags for the Storage service') 12 | param tags object = {} 13 | 14 | @description('Role assignments for the Storage service') 15 | param roles array = [] 16 | 17 | @description('Enable hierarchical namespace') 18 | param enableHns bool = false 19 | 20 | //******************************************************** 21 | // Resources 22 | //******************************************************** 23 | 24 | resource stNew 'Microsoft.Storage/storageAccounts@2022-05-01' = { 25 | name: name 26 | location: location 27 | tags: tags 28 | sku: { 29 | name: 'Standard_LRS' 30 | } 31 | kind: 'StorageV2' 32 | properties: { 33 | isHnsEnabled: enableHns 34 | minimumTlsVersion: 'TLS1_2' 35 | } 36 | } 37 | 38 | resource roleAssignments 'Microsoft.Authorization/roleAssignments@2022-04-01' = [ 39 | for role in roles: { 40 | name: guid(name, role.principalId, role.id) 41 | scope: stNew 42 | properties: { 43 | roleDefinitionId: resourceId('Microsoft.Authorization/roleDefinitions', role.id) 44 | principalId: role.principalId 45 | principalType: contains(role, 'type') ? role.type : 'ServicePrincipal' 46 | } 47 | } 48 | ] 49 | 50 | //******************************************************** 51 | // Outputs 52 | //******************************************************** 53 | 54 | output name string = stNew.name 55 | output id string = stNew.id 56 | -------------------------------------------------------------------------------- /infrastructure/modules/user-assigned-identity.bicep: -------------------------------------------------------------------------------- 1 | //******************************************************** 2 | // Parameters 3 | //******************************************************** 4 | 5 | @description('Name of the User Assigned Identity service') 6 | param name string 7 | 8 | @description('Location for User Assigned Identity service') 9 | param location string = resourceGroup().location 10 | 11 | @description('Tags for the User Assigned Identity service') 12 | param tags object = {} 13 | 14 | //******************************************************** 15 | // Resources 16 | //******************************************************** 17 | 18 | resource idNew 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { 19 | name: name 20 | location: location 21 | tags: tags 22 | } 23 | 24 | //******************************************************** 25 | // Outputs 26 | //******************************************************** 27 | 28 | output id string = idNew.id 29 | output name string = idNew.name 30 | output principalId string = idNew.properties.principalId 31 | -------------------------------------------------------------------------------- /kubernetes/manifest.yml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | app: credit-default-api 6 | name: credit-default-api 7 | spec: 8 | selector: 9 | matchLabels: 10 | app: credit-default-api 11 | template: 12 | metadata: 13 | labels: 14 | app: credit-default-api 15 | spec: 16 | containers: 17 | - image: ${CONTAINER_IMAGE} 18 | imagePullPolicy: Always 19 | name: credit-default-api 20 | ports: 21 | - containerPort: 5000 22 | --- 23 | apiVersion: v1 24 | kind: Service 25 | metadata: 26 | labels: 27 | app: credit-default-api 28 | name: credit-default-api 29 | spec: 30 | ports: 31 | - protocol: "TCP" 32 | port: 5000 33 | targetPort: 5000 34 | selector: 35 | app: credit-default-api 36 | type: LoadBalancer 37 | --- 38 | apiVersion: networking.k8s.io/v1 39 | kind: Ingress 40 | metadata: 41 | name: credit-default-api 42 | annotations: 43 | kubernetes.io/ingress.class: nginx 44 | spec: 45 | rules: 46 | - http: 47 | paths: 48 | - path: / 49 | pathType: Prefix 50 | backend: 51 | service: 52 | name: credit-default-api 53 | port: 54 | number: 5000 --------------------------------------------------------------------------------