├── .devcontainer └── devcontainer.json ├── .dockerignore ├── .env.template ├── .eslintrc.json ├── .github └── workflows │ ├── CODEOWNERS │ └── ci.yml ├── .gitignore ├── .mocharc.yml ├── .ngrok.yml ├── .nycrc ├── .prettierrc.js ├── .vscode ├── extensions.json ├── launch.json └── settings.json ├── CODE_OF_CONDUCT.md ├── Dockerfile ├── LICENSE ├── README.md ├── SECURITY.md ├── deploy ├── arm │ ├── acr.json │ ├── azuredeploy.dev.json │ ├── azuredeploy.json │ ├── firewall.json │ ├── keyvault.json │ ├── laboratory.json │ ├── monitoring.json │ ├── network.json │ ├── parameters.dev.json │ ├── parameters.json │ ├── queue.json │ ├── rbac.json │ └── worker.json ├── deploy-acr.sh ├── deploy-aks.sh ├── deploy-arm.sh └── helm │ ├── .gitignore │ ├── .helmignore │ ├── Chart.yaml │ ├── crds │ └── argo.yaml │ ├── templates │ ├── argo-controller-cm.yaml │ ├── argo-controller.yaml │ ├── argo-server.yaml │ ├── runs-namespace.yaml │ └── sds-worker.yaml │ ├── values.dev.yaml │ └── values.yaml ├── docker-compose.yml ├── docs ├── development.md └── images │ └── .gitkeep ├── lerna.json ├── package-lock.json ├── package.json ├── packages ├── cli │ ├── .eslintrc.json │ ├── .nycrc │ ├── .prettierignore │ ├── .prettierrc.js │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── conn.ts │ │ ├── decode_error.ts │ │ ├── demo.ts │ │ ├── formatting.ts │ │ ├── index.ts │ │ └── shims.d.ts │ ├── test │ │ ├── .mocharc.yml │ │ ├── conn.test.ts │ │ ├── formatting.test.ts │ │ ├── index.ts │ │ └── sds.test.ts │ ├── tsconfig.build.json │ └── tsconfig.json ├── laboratory │ ├── .env.template │ ├── .eslintrc.json │ ├── .nycrc │ ├── .prettierrc.js │ ├── openapi.yml │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── app.ts │ │ ├── auth.ts │ │ ├── configuration.ts │ │ ├── database.ts │ │ ├── errors.ts │ │ ├── index.ts │ │ ├── main.ts │ │ ├── routes │ │ │ ├── benchmarks.ts │ │ │ ├── candidates.ts │ │ │ ├── index.ts │ │ │ ├── runs.ts │ │ │ └── suites.ts │ │ ├── sequelize_laboratory │ │ │ ├── benchmark.ts │ │ │ ├── candidate.ts │ │ │ ├── index.ts │ │ │ ├── laboratory.ts │ │ │ ├── models │ │ │ │ ├── benchmark.ts │ │ │ │ ├── candidate.ts │ │ │ │ ├── decorators.ts │ │ │ │ ├── index.ts │ │ │ │ ├── result.ts │ │ │ │ ├── run.ts │ │ │ │ └── suite.ts │ │ │ ├── run.ts │ │ │ ├── sequelize.ts │ │ │ └── suite.ts │ │ ├── shims.d.ts │ │ └── telemetry.ts │ ├── test │ │ ├── .mocharc.yml │ │ ├── auth.test.ts │ │ ├── index.ts │ │ ├── samples.test.ts │ │ ├── sequelize_laboratory │ │ │ ├── benchmarks.test.ts │ │ │ ├── candidates.test.ts │ │ │ ├── models │ │ │ │ └── models.test.ts │ │ │ ├── runs.test.ts │ │ │ ├── shared.ts │ │ │ └── suites.test.ts │ │ └── server.test.ts │ ├── tsconfig.build.json │ └── tsconfig.json ├── sds │ ├── .eslintrc.json │ ├── .nycrc │ ├── .prettierignore │ ├── .prettierrc.js │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── configuration.ts │ │ ├── index.ts │ │ ├── laboratory │ │ │ ├── client │ │ │ │ ├── client.ts │ │ │ │ └── index.ts │ │ │ ├── index.ts │ │ │ ├── interfaces.ts │ │ │ ├── normalize.ts │ │ │ └── validate.ts │ │ ├── messages.ts │ │ ├── queue │ │ │ ├── azure.ts │ │ │ ├── index.ts │ │ │ ├── inmemory.ts │ │ │ └── processor.ts │ │ ├── shims.d.ts │ │ └── telemetry.ts │ ├── test │ │ ├── .mocharc.yml │ │ ├── functional │ │ │ ├── .mocharc.yml │ │ │ ├── configuration.ts │ │ │ └── queue │ │ │ │ └── azure.test.ts │ │ ├── index.ts │ │ ├── laboratory │ │ │ ├── client │ │ │ │ └── client.test.ts │ │ │ └── data.ts │ │ └── queue │ │ │ ├── azure.test.ts │ │ │ ├── index.ts │ │ │ └── processor.test.ts │ ├── tsconfig.build.json │ └── tsconfig.json └── worker │ ├── .env.template │ ├── .eslintrc.json │ ├── .nycrc │ ├── .prettierignore │ ├── .prettierrc.js │ ├── package-lock.json │ ├── package.json │ ├── src │ ├── argo.ts │ ├── argoWorker.ts │ ├── configuration.ts │ ├── index.ts │ ├── shims.d.ts │ └── telemetry.ts │ ├── test │ ├── .mocharc.yml │ ├── argoWorker.test.ts │ └── index.ts │ ├── tsconfig.build.json │ └── tsconfig.json ├── sample-data ├── benchmark1.yaml ├── candidate1.yaml └── suite1.yaml ├── samples └── catdetection │ ├── README.md │ ├── benchmark-eval │ ├── Dockerfile │ └── start.sh │ ├── benchmark-prep │ ├── Dockerfile │ └── start.sh │ ├── benchmark.yml │ ├── candidate.yml │ ├── candidate │ ├── Dockerfile │ └── start.sh │ └── suite.yml ├── scripts ├── README.md ├── demo-catdetection.sh ├── demo-sample-data.sh ├── dev-npm-audit-fix.sh ├── dev-npm-update.sh ├── dev-vm-setup.sh ├── dev-worker.sh └── laboratory.rest ├── tsconfig.build.json └── tsconfig.json /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "forwardPorts": [ 3 | 2746, // argo-server 4 | 3000, // laboratory 5 | 7777 // octant 6 | ], 7 | "postCreateCommand": "npm install" 8 | } 9 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .env 2 | .git 3 | .github 4 | .vscode 5 | docs 6 | scripts 7 | 8 | **/.coverage 9 | **/.nyc_output 10 | **/dist 11 | **/node_modules 12 | 13 | *Dockerfile* 14 | *.tgz 15 | *.tsbuildinfo 16 | -------------------------------------------------------------------------------- /.env.template: -------------------------------------------------------------------------------- 1 | 2 | # To use in bash: 3 | # set -o allexport; source .env; set +o allexport 4 | 5 | # Service Principal configuration 6 | # Note: outside of development, Managed Identities are used 7 | AZURE_TENANT_ID=00000000-0000-0000-0000-000000000000 8 | AZURE_CLIENT_ID=00000000-0000-0000-0000-000000000000 9 | AZURE_CLIENT_SECRET=00000000-0000-0000-0000-000000000000 10 | 11 | # Laboratory configuration 12 | QUEUE_MODE=azure 13 | QUEUE_ENDPOINT=https://.queue.core.windows.net/runs 14 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./node_modules/gts/", 3 | "root": true, 4 | "ignorePatterns": [ 5 | "dist/**/*" 6 | ], 7 | "overrides": [ 8 | { 9 | "files": [ 10 | "test/**/*.ts" 11 | ], 12 | "env": { 13 | "mocha": true 14 | } 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /.github/workflows/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @MikeHopcroft @noelbundick 2 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | jobs: 3 | build: 4 | runs-on: ubuntu-latest 5 | steps: 6 | - uses: actions/checkout@v2 7 | 8 | - uses: actions/setup-node@v1 9 | with: 10 | node-version: '12.x' 11 | 12 | - uses: actions/cache@v1 13 | env: 14 | cache-name: cache-node-modules 15 | with: 16 | path: ~/.npm 17 | key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} 18 | restore-keys: | 19 | ${{ runner.os }}-build-${{ env.cache-name }}- 20 | ${{ runner.os }}-build- 21 | ${{ runner.os }}- 22 | 23 | - run: npm install 24 | 25 | - run: npm run test 26 | 27 | - uses: codecov/codecov-action@v1 28 | 29 | - run: docker-compose up --exit-code-from cli 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | .nyc_output 3 | *.tgz 4 | *.tsbuildinfo 5 | build 6 | coverage 7 | dist 8 | node_modules 9 | /hack 10 | -------------------------------------------------------------------------------- /.mocharc.yml: -------------------------------------------------------------------------------- 1 | spec: test/**/*.ts 2 | recursive: true 3 | ignore: 4 | - "**/functional/**/*.ts" # don't run functional tests by default 5 | -------------------------------------------------------------------------------- /.ngrok.yml: -------------------------------------------------------------------------------- 1 | tunnels: 2 | arm: 3 | addr: 9001 4 | proto: http 5 | bind_tls: true 6 | -------------------------------------------------------------------------------- /.nycrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@istanbuljs/nyc-config-typescript", 3 | "all": true, 4 | "exclude": [ 5 | "**/*.js", 6 | "**/*.d.ts", 7 | "**/test/**/*.ts" 8 | ], 9 | "reporter": [ 10 | "text", 11 | "lcovonly", 12 | "cobertura" 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | ...require('gts/.prettierrc.json'), 3 | bracketSpacing: true, 4 | } 5 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "humao.rest-client", 4 | "ms-azuretools.vscode-azureappservice", 5 | "msazurermtools.azurerm-vscode-tools", 6 | "visualstudioexptteam.vscodeintellicode" 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "laboratory", 9 | "request": "launch", 10 | "runtimeArgs": [ 11 | "run", 12 | "laboratory" 13 | ], 14 | "runtimeExecutable": "npm", 15 | "skipFiles": [ 16 | "/**" 17 | ], 18 | "type": "pwa-node" 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.tabSize": 2, 3 | "editor.detectIndentation": false, 4 | "files.insertFinalNewline": true, 5 | "typescript.tsdk": "node_modules/typescript/lib", 6 | 7 | "rest-client.environmentVariables": { 8 | "$shared": { 9 | "laboratoryHost": "http://localhost:3000" 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:lts-slim AS build 2 | ENV PATH="${PATH}:node_modules/.bin" 3 | RUN npm set unsafe-perm true 4 | 5 | WORKDIR /app 6 | COPY package*.json lerna.json tsconfig*.json ./ 7 | RUN npm ci --ignore-scripts 8 | 9 | # Node packages 10 | FROM build AS build-sds 11 | COPY packages/sds ./packages/sds 12 | RUN lerna bootstrap --ci --ignore-scripts \ 13 | && cd packages/sds \ 14 | && npm pack 15 | 16 | FROM build-sds AS build-laboratory 17 | COPY packages/laboratory ./packages/laboratory 18 | RUN lerna bootstrap --ci --ignore-scripts \ 19 | && cd packages/laboratory \ 20 | && npm pack 21 | 22 | FROM build-sds AS build-worker 23 | COPY packages/worker ./packages/worker 24 | RUN lerna bootstrap --ci --ignore-scripts \ 25 | && cd packages/worker \ 26 | && npm pack 27 | 28 | FROM build-sds AS build-cli 29 | COPY packages/cli ./packages/cli 30 | RUN lerna bootstrap --ci --ignore-scripts \ 31 | && cd packages/cli \ 32 | && npm pack 33 | 34 | # Application images 35 | FROM node:lts-slim AS app 36 | ENV NODE_ENV=production PATH="${PATH}:node_modules/.bin" 37 | WORKDIR /app 38 | RUN chown node:node . 39 | USER node 40 | 41 | FROM app AS laboratory 42 | COPY --from=build-laboratory /app/packages/sds/*.tgz /app/packages/laboratory/*.tgz /packages/ 43 | RUN npm install /packages/*.tgz 44 | CMD sds-laboratory 45 | 46 | FROM app AS worker 47 | COPY --from=build-worker /app/packages/sds/*.tgz /app/packages/worker/*.tgz /packages/ 48 | RUN npm install /packages/*.tgz 49 | CMD sds-worker 50 | 51 | FROM app AS cli 52 | COPY --from=build-cli /app/packages/sds/*.tgz /app/packages/cli/*.tgz /packages/ 53 | RUN npm install /packages/*.tgz 54 | CMD sds-cli 55 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets Microsoft's [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)) of a security vulnerability, please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /deploy/arm/acr.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "acrPrivateDnsZoneId": { 6 | "type": "string" 7 | }, 8 | "subnetId": { 9 | "type": "string" 10 | }, 11 | "tags": { 12 | "type": "object", 13 | "defaultValue": {} 14 | } 15 | }, 16 | "variables": { 17 | "infraAcr": "[concat('infra', variables('suffix'))]", 18 | "userAcr": "[concat('user', variables('suffix'))]", 19 | "suffix": "[toLower(take(uniqueString(resourceGroup().id), 6))]" 20 | }, 21 | "resources": [ 22 | { 23 | "type": "Microsoft.ContainerRegistry/registries", 24 | "apiVersion": "2019-05-01", 25 | "location": "[resourceGroup().location]", 26 | "name": "[variables('infraAcr')]", 27 | "sku": { 28 | "name": "Premium" 29 | }, 30 | "tags": "[parameters('tags')]", 31 | "properties": { 32 | "adminUserEnabled": false 33 | } 34 | }, 35 | { 36 | "type": "Microsoft.ContainerRegistry/registries", 37 | "apiVersion": "2019-05-01", 38 | "location": "[resourceGroup().location]", 39 | "name": "[variables('userAcr')]", 40 | "sku": { 41 | "name": "Premium" 42 | }, 43 | "tags": "[parameters('tags')]", 44 | "properties": { 45 | "adminUserEnabled": false 46 | } 47 | }, 48 | { 49 | "dependsOn": [ 50 | "[resourceId('Microsoft.ContainerRegistry/registries', variables('infraAcr'))]" 51 | ], 52 | "type": "Microsoft.Network/privateEndpoints", 53 | "apiVersion": "2020-05-01", 54 | "location": "[resourceGroup().location]", 55 | "name": "[variables('infraAcr')]", 56 | "tags": "[parameters('tags')]", 57 | "properties": { 58 | "privateLinkServiceConnections": [ 59 | { 60 | "name": "[variables('infraAcr')]", 61 | "properties": { 62 | "groupIds": [ 63 | "registry" 64 | ], 65 | "privateLinkServiceId": "[resourceId('Microsoft.ContainerRegistry/registries', variables('infraAcr'))]" 66 | } 67 | } 68 | ], 69 | "subnet": { 70 | "id": "[parameters('subnetId')]" 71 | } 72 | }, 73 | "resources": [ 74 | { 75 | "dependsOn": [ 76 | "[resourceId('Microsoft.Network/privateEndpoints', variables('infraAcr'))]" 77 | ], 78 | "type": "privateDnsZoneGroups", 79 | "apiVersion": "2020-05-01", 80 | "location": "[resourceGroup().location]", 81 | "name": "default", 82 | "properties": { 83 | "privateDnsZoneConfigs": [ 84 | { 85 | "name": "default", 86 | "properties": { 87 | "privateDnsZoneId": "[parameters('acrPrivateDnsZoneId')]" 88 | } 89 | } 90 | ] 91 | } 92 | } 93 | ] 94 | }, 95 | { 96 | "dependsOn": [ 97 | "[resourceId('Microsoft.ContainerRegistry/registries', variables('userAcr'))]" 98 | ], 99 | "type": "Microsoft.Network/privateEndpoints", 100 | "apiVersion": "2020-05-01", 101 | "location": "[resourceGroup().location]", 102 | "name": "[variables('userAcr')]", 103 | "tags": "[parameters('tags')]", 104 | "properties": { 105 | "privateLinkServiceConnections": [ 106 | { 107 | "name": "[variables('userAcr')]", 108 | "properties": { 109 | "groupIds": [ 110 | "registry" 111 | ], 112 | "privateLinkServiceId": "[resourceId('Microsoft.ContainerRegistry/registries', variables('userAcr'))]" 113 | } 114 | } 115 | ], 116 | "subnet": { 117 | "id": "[parameters('subnetId')]" 118 | } 119 | }, 120 | "resources": [ 121 | { 122 | "dependsOn": [ 123 | "[resourceId('Microsoft.Network/privateEndpoints', variables('userAcr'))]" 124 | ], 125 | "type": "privateDnsZoneGroups", 126 | "apiVersion": "2020-05-01", 127 | "location": "[resourceGroup().location]", 128 | "name": "default", 129 | "properties": { 130 | "privateDnsZoneConfigs": [ 131 | { 132 | "name": "default", 133 | "properties": { 134 | "privateDnsZoneId": "[parameters('acrPrivateDnsZoneId')]" 135 | } 136 | } 137 | ] 138 | } 139 | } 140 | ] 141 | } 142 | ], 143 | "outputs": { 144 | "infraAcrId": { 145 | "type": "string", 146 | "value": "[resourceId('Microsoft.ContainerRegistry/registries', variables('infraAcr'))]" 147 | }, 148 | "infraAcrLoginServer": { 149 | "type": "string", 150 | "value": "[reference(resourceId('Microsoft.ContainerRegistry/registries', variables('infraAcr'))).loginServer]" 151 | }, 152 | "userAcrId": { 153 | "type": "string", 154 | "value": "[resourceId('Microsoft.ContainerRegistry/registries', variables('userAcr'))]" 155 | }, 156 | "userAcrLoginServer": { 157 | "type": "string", 158 | "value": "[reference(resourceId('Microsoft.ContainerRegistry/registries', variables('userAcr'))).loginServer]" 159 | } 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /deploy/arm/keyvault.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "tags": { 6 | "type": "object", 7 | "defaultValue": {} 8 | } 9 | }, 10 | "variables": { 11 | "defaultTags": { 12 | "keyVaultExists": false 13 | }, 14 | "updatedTags": { 15 | "keyVaultExists": true 16 | }, 17 | "keyVault": "[concat('sds', variables('suffix'))]", 18 | "existingTags": "[if(contains(resourceGroup(), 'tags'), resourceGroup().tags, json('{}'))]", 19 | "keyVaultExists": "[bool(union(variables('defaultTags'), variables('existingTags'))['keyVaultExists'])]", 20 | "suffix": "[toLower(take(uniqueString(resourceGroup().id), 6))]" 21 | }, 22 | "resources": [ 23 | { 24 | "type": "Microsoft.Resources/deployments", 25 | "apiVersion": "2019-10-01", 26 | "name": "getAccessPolicies", 27 | "tags": "[parameters('tags')]", 28 | "properties": { 29 | "mode": "Incremental", 30 | "template": { 31 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", 32 | "contentVersion": "1.0.0.0", 33 | "resources": [ 34 | ], 35 | "outputs": { 36 | "accessPolicies": { 37 | "type": "array", 38 | "value": "[if(variables('keyVaultExists'), reference(resourceId('Microsoft.KeyVault/vaults', variables('keyVault')), '2016-10-01').accessPolicies, json('[]'))]" 39 | } 40 | } 41 | } 42 | } 43 | }, 44 | { 45 | "dependsOn": [ 46 | "[resourceId('Microsoft.Resources/deployments', 'getAccessPolicies')]" 47 | ], 48 | "type": "Microsoft.Resources/deployments", 49 | "apiVersion": "2019-10-01", 50 | "name": "keyvault-deploy", 51 | "tags": "[parameters('tags')]", 52 | "properties": { 53 | "mode": "Incremental", 54 | "template": { 55 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", 56 | "contentVersion": "1.0.0.0", 57 | "resources": [ 58 | { 59 | "type": "Microsoft.KeyVault/vaults", 60 | "apiVersion": "2016-10-01", 61 | "location": "[resourceGroup().location]", 62 | "name": "[variables('keyVault')]", 63 | "tags": "[parameters('tags')]", 64 | "properties": { 65 | "enabledForDeployment": false, 66 | "enabledForDiskEncryption": false, 67 | "enabledForTemplateDeployment": false, 68 | "sku": { 69 | "family": "A", 70 | "name": "standard" 71 | }, 72 | "enableSoftDelete": true, 73 | "tenantId": "[subscription().tenantId]", 74 | "accessPolicies": "[reference(resourceId('Microsoft.Resources/deployments', 'getAccessPolicies')).outputs.accessPolicies.value]" 75 | } 76 | }, 77 | { 78 | "dependsOn": [ 79 | "[resourceId('Microsoft.KeyVault/vaults', variables('keyVault'))]" 80 | ], 81 | "type": "Microsoft.Resources/tags", 82 | "apiVersion": "2019-10-01", 83 | "name": "default", 84 | "properties": { 85 | "tags": "[union(variables('existingTags'), variables('updatedTags'))]" 86 | } 87 | } 88 | ] 89 | } 90 | } 91 | } 92 | ], 93 | "outputs": { 94 | "keyVault": { 95 | "type": "string", 96 | "value": "[variables('keyVault')]" 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /deploy/arm/monitoring.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "tags": { 6 | "type": "object", 7 | "defaultValue": {} 8 | } 9 | }, 10 | "variables": { 11 | "appInsights": "[concat('sds', variables('suffix'))]", 12 | "suffix": "[toLower(take(uniqueString(resourceGroup().id), 6))]" 13 | }, 14 | "resources": [ 15 | { 16 | "type": "Microsoft.Insights/components", 17 | "apiVersion": "2020-02-02-preview", 18 | "location": "[resourceGroup().location]", 19 | "name": "[variables('appInsights')]", 20 | "tags": "[parameters('tags')]", 21 | "properties": { 22 | "Application_Type": "web" 23 | } 24 | } 25 | ], 26 | "outputs": { 27 | "instrumentationKey": { 28 | "type": "string", 29 | "value": "[reference(resourceId('Microsoft.Insights/components', variables('appInsights'))).InstrumentationKey]" 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /deploy/arm/parameters.dev.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "allowedExternalEndpoints": { 6 | "value": [ 7 | "placekitten.com", 8 | "*.cognitiveservices.azure.com" 9 | ] 10 | }, 11 | "mode": { 12 | "value": "development" 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /deploy/arm/parameters.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "allowedExternalEndpoints": { 6 | "value": [] 7 | }, 8 | "mode": { 9 | "value": "production" 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /deploy/arm/queue.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "queuePrivateDnsZoneId": { 6 | "type": "string" 7 | }, 8 | "subnetId": { 9 | "type": "string" 10 | }, 11 | "tags": { 12 | "type": "object", 13 | "defaultValue": {} 14 | } 15 | }, 16 | "variables": { 17 | "runsQueue": "runs", 18 | "privateEndpoint": "[concat(variables('storageAccount'), '-queue')]", 19 | "storageAccount": "[concat('queue', variables('suffix'))]", 20 | "suffix": "[toLower(take(uniqueString(resourceGroup().id), 6))]" 21 | }, 22 | "resources": [ 23 | { 24 | "type": "Microsoft.Storage/storageAccounts", 25 | "apiVersion": "2019-06-01", 26 | "location": "[resourceGroup().location]", 27 | "name": "[variables('storageAccount')]", 28 | "kind": "StorageV2", 29 | "sku": { 30 | "name": "Standard_LRS" 31 | }, 32 | "tags": "[parameters('tags')]", 33 | "properties": { 34 | "accessTier": "Hot", 35 | "supportsHttpsTrafficOnly": true 36 | }, 37 | "resources": [ 38 | { 39 | "dependsOn": [ 40 | "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccount'))]" 41 | ], 42 | "type": "queueServices/queues", 43 | "apiVersion": "2019-06-01", 44 | "name": "[concat('default/', variables('runsQueue'))]", 45 | "properties": { 46 | } 47 | } 48 | ] 49 | }, 50 | { 51 | "dependsOn": [ 52 | "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccount'))]" 53 | ], 54 | "type": "Microsoft.Network/privateEndpoints", 55 | "apiVersion": "2020-05-01", 56 | "location": "[resourceGroup().location]", 57 | "name": "[variables('privateEndpoint')]", 58 | "tags": "[parameters('tags')]", 59 | "properties": { 60 | "privateLinkServiceConnections": [ 61 | { 62 | "name": "[variables('privateEndpoint')]", 63 | "properties": { 64 | "groupIds": [ 65 | "queue" 66 | ], 67 | "privateLinkServiceId": "[resourceId('Microsoft.Storage/storageAccounts', variables('storageAccount'))]" 68 | } 69 | } 70 | ], 71 | "subnet": { 72 | "id": "[parameters('subnetId')]" 73 | } 74 | }, 75 | "resources": [ 76 | { 77 | "dependsOn": [ 78 | "[resourceId('Microsoft.Network/privateEndpoints', variables('privateEndpoint'))]" 79 | ], 80 | "type": "privateDnsZoneGroups", 81 | "apiVersion": "2020-05-01", 82 | "location": "[resourceGroup().location]", 83 | "name": "default", 84 | "properties": { 85 | "privateDnsZoneConfigs": [ 86 | { 87 | "name": "default", 88 | "properties": { 89 | "privateDnsZoneId": "[parameters('queuePrivateDnsZoneId')]" 90 | } 91 | } 92 | ] 93 | } 94 | } 95 | ] 96 | } 97 | ], 98 | "outputs": { 99 | "runsResourceId": { 100 | "type": "string", 101 | "value": "[resourceId('Microsoft.Storage/storageAccounts/queueServices/queues', variables('storageAccount'), 'default', variables('runsQueue'))]" 102 | }, 103 | "runsQueueEndpoint": { 104 | "type": "string", 105 | "value": "[concat(reference(resourceId('Microsoft.Storage/storageAccounts', variables('storageAccount'))).primaryEndpoints.queue, variables('runsQueue'))]" 106 | } 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /deploy/arm/rbac.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "kubeletIdentityId": { 6 | "type": "string" 7 | }, 8 | "workerIdentityId": { 9 | "type": "string" 10 | }, 11 | "laboratoryIdentityId": { 12 | "type": "string" 13 | }, 14 | "infraAcrId": { 15 | "type": "string" 16 | }, 17 | "userAcrId": { 18 | "type": "string" 19 | }, 20 | "runsQueueId": { 21 | "type": "string" 22 | }, 23 | "nodeResourceGroup": { 24 | "type": "string" 25 | }, 26 | "tags": { 27 | "type": "object", 28 | "defaultValue": {} 29 | } 30 | }, 31 | "variables": { 32 | "infraAcrName": "[last(split(parameters('infraAcrId'), '/'))]", 33 | "userAcrName": "[last(split(parameters('userAcrId'), '/'))]", 34 | "runsStorageAccountName": "[split(parameters('runsQueueId'), '/')[8]]", 35 | "runsQueueName": "[split(parameters('runsQueueId'), '/')[12]]", 36 | "benchmarkIdentity": "benchmark", 37 | "candidateIdentity": "candidate", 38 | "workerIdentity": "worker", 39 | "acrPullRoleId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d')]", 40 | "managedIdentityOperatorRoleId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'f1a07417-d97a-45cb-824c-7a7467783830')]", 41 | "storageQueueDataMessageSenderRoleId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'c6a89b2d-59bc-44d0-9896-0f6e12d7b80a')]", 42 | "storageQueueDataMessageProcessorRoleId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '8a0f0c08-91a1-4084-bc3d-661d67233fed')]", 43 | "virtualMachineContributorRoleId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '9980e02c-c2be-4d73-94e8-173b1dc7cf3c')]" 44 | }, 45 | "resources": [ 46 | { 47 | "type": "Microsoft.ManagedIdentity/userAssignedIdentities/providers/roleAssignments", 48 | "apiVersion": "2018-09-01-preview", 49 | "location": "[resourceGroup().location]", 50 | "name": "[concat(variables('workerIdentity'), '/Microsoft.Authorization/', guid(concat(resourceGroup().id, 'aks', 'worker', 'ManagedIdentityOperator')))]", 51 | "properties": { 52 | "principalId": "[parameters('kubeletIdentityId')]", 53 | "roleDefinitionId": "[variables('managedIdentityOperatorRoleId')]" 54 | } 55 | }, 56 | { 57 | "type": "Microsoft.ManagedIdentity/userAssignedIdentities/providers/roleAssignments", 58 | "apiVersion": "2018-09-01-preview", 59 | "location": "[resourceGroup().location]", 60 | "name": "[concat(variables('benchmarkIdentity'), '/Microsoft.Authorization/', guid(concat(resourceGroup().id, 'aks', 'benchmark', 'ManagedIdentityOperator')))]", 61 | "properties": { 62 | "principalId": "[parameters('kubeletIdentityId')]", 63 | "roleDefinitionId": "[variables('managedIdentityOperatorRoleId')]" 64 | } 65 | }, 66 | { 67 | "type": "Microsoft.ManagedIdentity/userAssignedIdentities/providers/roleAssignments", 68 | "apiVersion": "2018-09-01-preview", 69 | "location": "[resourceGroup().location]", 70 | "name": "[concat(variables('candidateIdentity'), '/Microsoft.Authorization/', guid(concat(resourceGroup().id, 'aks', 'candidate', 'ManagedIdentityOperator')))]", 71 | "properties": { 72 | "principalId": "[parameters('kubeletIdentityId')]", 73 | "roleDefinitionId": "[variables('managedIdentityOperatorRoleId')]" 74 | } 75 | }, 76 | { 77 | "type": "Microsoft.ContainerRegistry/registries/providers/roleAssignments", 78 | "apiVersion": "2018-09-01-preview", 79 | "location": "[resourceGroup().location]", 80 | "name": "[concat(variables('infraAcrName'), '/Microsoft.Authorization/', guid(concat(resourceGroup().id, parameters('kubeletIdentityId'), parameters('infraAcrId'), 'AcrPull')))]", 81 | "properties": { 82 | "principalId": "[parameters('kubeletIdentityId')]", 83 | "roleDefinitionId": "[variables('acrPullRoleId')]" 84 | } 85 | }, 86 | { 87 | "type": "Microsoft.ContainerRegistry/registries/providers/roleAssignments", 88 | "apiVersion": "2018-09-01-preview", 89 | "location": "[resourceGroup().location]", 90 | "name": "[concat(variables('userAcrName'), '/Microsoft.Authorization/', guid(concat(resourceGroup().id, parameters('kubeletIdentityId'), parameters('userAcrId'), 'AcrPull')))]", 91 | "properties": { 92 | "principalId": "[parameters('kubeletIdentityId')]", 93 | "roleDefinitionId": "[variables('acrPullRoleId')]" 94 | } 95 | }, 96 | { 97 | "type": "Microsoft.Storage/storageAccounts/queueServices/queues/providers/roleAssignments", 98 | "apiVersion": "2018-09-01-preview", 99 | "location": "[resourceGroup().location]", 100 | "name": "[concat(variables('runsStorageAccountName'), '/default/', variables('runsQueueName'), '/Microsoft.Authorization/', guid(concat(resourceGroup().id, parameters('workerIdentityId'), parameters('runsQueueId'), 'StorageQueueDataMessageProcessor')))]", 101 | "properties": { 102 | "principalId": "[parameters('workerIdentityId')]", 103 | "roleDefinitionId": "[variables('storageQueueDataMessageProcessorRoleId')]", 104 | "type": "ServicePrincipal" 105 | } 106 | }, 107 | { 108 | "type": "Microsoft.Storage/storageAccounts/queueServices/queues/providers/roleAssignments", 109 | "apiVersion": "2018-09-01-preview", 110 | "location": "[resourceGroup().location]", 111 | "name": "[concat(variables('runsStorageAccountName'), '/default/', variables('runsQueueName'), '/Microsoft.Authorization/', guid(concat(resourceGroup().id, parameters('laboratoryIdentityId'), parameters('runsQueueId'), 'StorageQueueDataMessageSender')))]", 112 | "properties": { 113 | "principalId": "[parameters('laboratoryIdentityId')]", 114 | "roleDefinitionId": "[variables('storageQueueDataMessageSenderRoleId')]", 115 | "type": "ServicePrincipal" 116 | } 117 | }, 118 | { 119 | "type": "Microsoft.Resources/deployments", 120 | "apiVersion": "2019-10-01", 121 | "name": "[parameters('nodeResourceGroup')]", 122 | "resourceGroup": "[parameters('nodeResourceGroup')]", 123 | "tags": "[parameters('tags')]", 124 | "properties": { 125 | "mode": "Incremental", 126 | "template": { 127 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", 128 | "contentVersion": "1.0.0.0", 129 | "resources": [ 130 | { 131 | "type": "Microsoft.Authorization/roleAssignments", 132 | "apiVersion": "2018-09-01-preview", 133 | "location": "[resourceGroup().location]", 134 | "name": "[guid(concat(resourceGroup().id, parameters('kubeletIdentityId'), 'VirtualMachineContributor'))]", 135 | "properties": { 136 | "principalId": "[parameters('kubeletIdentityId')]", 137 | "roleDefinitionId": "[variables('virtualMachineContributorRoleId')]", 138 | "type": "ServicePrincipal" 139 | } 140 | } 141 | ] 142 | } 143 | } 144 | } 145 | ], 146 | "outputs": {} 147 | } 148 | -------------------------------------------------------------------------------- /deploy/deploy-acr.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | RESOURCE_GROUP=${1:-sds} 5 | DEPLOYMENT_NAME=${2:-azuredeploy} 6 | TAG=${3:-latest} 7 | 8 | # Versions 9 | ARGO_VERSION="v2.11.0" 10 | 11 | export DOCKER_BUILDKIT=1 12 | 13 | OUTPUTS=$(az deployment group show -g $RESOURCE_GROUP -n $DEPLOYMENT_NAME --query properties.outputs -o json) 14 | LOGIN_SERVER=$(echo $OUTPUTS | jq -r .infraAcrLoginServer.value) 15 | az acr login -n $LOGIN_SERVER 16 | 17 | # Import public images 18 | az acr import -n $LOGIN_SERVER --source "docker.io/argoproj/argoexec:$ARGO_VERSION" --force 19 | az acr import -n $LOGIN_SERVER --source "docker.io/argoproj/workflow-controller:$ARGO_VERSION" --force 20 | az acr import -n $LOGIN_SERVER --source "docker.io/argoproj/argocli:$ARGO_VERSION" --force 21 | 22 | # Push SDS images 23 | LABORATORY_IMAGE="${LOGIN_SERVER}/sds-laboratory:${TAG}" 24 | docker build --target laboratory -t $LABORATORY_IMAGE . 25 | docker push $LABORATORY_IMAGE 26 | 27 | WORKER_IMAGE="${LOGIN_SERVER}/sds-worker:${TAG}" 28 | docker build --target worker -t $WORKER_IMAGE . 29 | docker push $WORKER_IMAGE 30 | -------------------------------------------------------------------------------- /deploy/deploy-aks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | RESOURCE_GROUP=${1:-sds} 5 | DEPLOYMENT_NAME=${2:-azuredeploy} 6 | 7 | # Versions 8 | AAD_POD_IDENTITY_VERSION="2.0.2" 9 | AZUREFILE_CSI_DRIVER_VERSION="v0.9.0" 10 | ARGO_VERSION="v2.11.0" 11 | 12 | # aad-pod-identity 13 | if ! helm repo list | grep '^aad-pod-identity'; then 14 | helm repo add aad-pod-identity https://raw.githubusercontent.com/Azure/aad-pod-identity/master/charts 15 | fi 16 | helm upgrade --install aad-pod-identity aad-pod-identity/aad-pod-identity --namespace kube-system --version $AAD_POD_IDENTITY_VERSION --set nmi.metadataHeaderRequired=true 17 | 18 | # azurefile-csi-driver 19 | if ! helm repo list | grep '^azurefile-csi-driver'; then 20 | helm repo add azurefile-csi-driver https://raw.githubusercontent.com/kubernetes-sigs/azurefile-csi-driver/master/charts 21 | fi 22 | helm upgrade --install azurefile-csi-driver azurefile-csi-driver/azurefile-csi-driver --namespace kube-system --version $AZUREFILE_CSI_DRIVER_VERSION 23 | 24 | OUTPUTS=$(az deployment group show -g $RESOURCE_GROUP -n $DEPLOYMENT_NAME --query properties.outputs -o json) 25 | INFRA_ACR=$(echo $OUTPUTS | jq -r .infraAcrLoginServer.value) 26 | helm upgrade --install sds deploy/helm \ 27 | --set "instrumentationKey=$(echo $OUTPUTS | jq -r .instrumentationKey.value)" \ 28 | --set "argoController.executorImage=$INFRA_ACR/argoproj/argoexec:$ARGO_VERSION" \ 29 | --set "argoController.image=$INFRA_ACR/argoproj/workflow-controller:$ARGO_VERSION" \ 30 | --set "argoServer.image=$INFRA_ACR/argoproj/argocli:$ARGO_VERSION" \ 31 | --set "worker.image=$INFRA_ACR/sds-worker" \ 32 | --set "worker.queueEndpoint=$(echo $OUTPUTS | jq -r .runsQueueEndpoint.value)" \ 33 | --set "identities.worker.resourceId=$(echo $OUTPUTS | jq -r .workerIdentityResourceId.value)" \ 34 | --set "identities.worker.clientId=$(echo $OUTPUTS | jq -r .workerIdentityClientId.value)" \ 35 | --set "identities.benchmark.resourceId=$(echo $OUTPUTS | jq -r .benchmarkIdentityResourceId.value)" \ 36 | --set "identities.benchmark.clientId=$(echo $OUTPUTS | jq -r .benchmarkIdentityClientId.value)" \ 37 | --set "identities.candidate.resourceId=$(echo $OUTPUTS | jq -r .candidateIdentityResourceId.value)" \ 38 | --set "identities.candidate.clientId=$(echo $OUTPUTS | jq -r .candidateIdentityClientId.value)" \ 39 | --set "worker.storage.storageAccount=$(echo $OUTPUTS | jq -r .runsTransientStorageAccount.value)" \ 40 | --set "worker.storage.resourceGroup=$RESOURCE_GROUP" 41 | -------------------------------------------------------------------------------- /deploy/deploy-arm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eo pipefail 3 | 4 | show_usage() { 5 | echo 'Usage: deploy.sh -g [--assets ] [--sas ] [--dev]' 6 | } 7 | 8 | DEV=false 9 | SAS="" 10 | 11 | parse_arguments() { 12 | PARAMS="" 13 | while (( $# )); do 14 | case "$1" in 15 | -h|--help) 16 | show_usage 17 | exit 0 18 | ;; 19 | -g|--resource-group) 20 | RESOURCE_GROUP=$2 21 | shift 2 22 | ;; 23 | -a|--assets) 24 | ASSETS_BASE=$2 25 | shift 2 26 | ;; 27 | --sas) 28 | SAS=$2 29 | shift 2 30 | ;; 31 | --dev) 32 | DEV=true 33 | shift 34 | ;; 35 | --) 36 | shift 37 | break 38 | ;; 39 | -*|--*) 40 | echo "Unsupported flag $1" >&2 41 | exit 1 42 | ;; 43 | *) 44 | PARAMS="$PARAMS $1" 45 | shift 46 | ;; 47 | esac 48 | done 49 | } 50 | 51 | validate_arguments() { 52 | if [[ -z "$RESOURCE_GROUP" ]]; then 53 | show_usage 54 | exit 1 55 | fi 56 | 57 | if [[ -z "$ASSETS_BASE" && "$DEV" = true ]]; then 58 | ASSETS_BASE=$(curl -s http://127.0.0.1:4040/api/tunnels | jq -r '.tunnels[] | select(.name == "arm").public_url') 59 | fi 60 | ASSETS_BASE=${ASSETS_BASE:-'https://raw.githubusercontent.com/microsoft/secure-data-sandbox/main/deploy/'} 61 | } 62 | 63 | deploy_environment() { 64 | PARAMS_FILE="parameters.json" 65 | 66 | if [ "$DEV" = true ]; then 67 | PARAMS_FILE="parameters.dev.json" 68 | fi 69 | 70 | DEPLOY_ARGS=( 71 | "-g" "$RESOURCE_GROUP" 72 | "-p" "${ASSETS_BASE}/arm/${PARAMS_FILE}?${SAS}" 73 | "-u" "${ASSETS_BASE}/arm/azuredeploy.json?${SAS}" 74 | "-p" 75 | "assetsBaseUrl=$ASSETS_BASE" 76 | "deploymentSas=$SAS" 77 | ) 78 | 79 | az deployment group create "${DEPLOY_ARGS[@]}" 80 | } 81 | 82 | deploy_dev() { 83 | if [ "$DEV" = true ]; then 84 | az deployment group create -g $RESOURCE_GROUP -u "${ASSETS_BASE}/arm/azuredeploy.dev.json?${SAS}" -p "sshPublicKey=$(cat ~/.ssh/id_rsa.pub)" 85 | fi 86 | } 87 | 88 | parse_arguments "$@" 89 | validate_arguments 90 | 91 | set -u 92 | deploy_environment 93 | deploy_dev 94 | -------------------------------------------------------------------------------- /deploy/helm/.gitignore: -------------------------------------------------------------------------------- 1 | charts 2 | -------------------------------------------------------------------------------- /deploy/helm/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *.orig 18 | *~ 19 | # Various IDEs 20 | .project 21 | .idea/ 22 | *.tmproj 23 | .vscode/ 24 | -------------------------------------------------------------------------------- /deploy/helm/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: sds-worker 3 | description: A Helm chart for Kubernetes 4 | 5 | # A chart can be either an 'application' or a 'library' chart. 6 | # 7 | # Application charts are a collection of templates that can be packaged into versioned archives 8 | # to be deployed. 9 | # 10 | # Library charts provide useful utilities or functions for the chart developer. They're included as 11 | # a dependency of application charts to inject those utilities and functions into the rendering 12 | # pipeline. Library charts do not define any templates and therefore cannot be deployed. 13 | type: application 14 | 15 | # This is the chart version. This version number should be incremented each time you make changes 16 | # to the chart and its templates, including the app version. 17 | # Versions are expected to follow Semantic Versioning (https://semver.org/) 18 | version: 0.1.0 19 | 20 | # This is the version number of the application being deployed. This version number should be 21 | # incremented each time you make changes to the application. Versions are not expected to 22 | # follow Semantic Versioning. They should reflect the version the application is using. 23 | appVersion: 0.1.0 24 | -------------------------------------------------------------------------------- /deploy/helm/crds/argo.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apiextensions.k8s.io/v1beta1 2 | kind: CustomResourceDefinition 3 | metadata: 4 | name: clusterworkflowtemplates.argoproj.io 5 | spec: 6 | group: argoproj.io 7 | names: 8 | kind: ClusterWorkflowTemplate 9 | listKind: ClusterWorkflowTemplateList 10 | plural: clusterworkflowtemplates 11 | shortNames: 12 | - clusterwftmpl 13 | - cwft 14 | singular: clusterworkflowtemplate 15 | scope: Cluster 16 | version: v1alpha1 17 | versions: 18 | - name: v1alpha1 19 | served: true 20 | storage: true 21 | --- 22 | apiVersion: apiextensions.k8s.io/v1beta1 23 | kind: CustomResourceDefinition 24 | metadata: 25 | name: cronworkflows.argoproj.io 26 | spec: 27 | group: argoproj.io 28 | names: 29 | kind: CronWorkflow 30 | listKind: CronWorkflowList 31 | plural: cronworkflows 32 | shortNames: 33 | - cwf 34 | - cronwf 35 | singular: cronworkflow 36 | scope: Namespaced 37 | version: v1alpha1 38 | versions: 39 | - name: v1alpha1 40 | served: true 41 | storage: true 42 | --- 43 | apiVersion: apiextensions.k8s.io/v1beta1 44 | kind: CustomResourceDefinition 45 | metadata: 46 | name: workfloweventbindings.argoproj.io 47 | spec: 48 | group: argoproj.io 49 | names: 50 | kind: WorkflowEventBinding 51 | listKind: WorkflowEventBindingList 52 | plural: workfloweventbindings 53 | shortNames: 54 | - wfeb 55 | singular: workfloweventbinding 56 | scope: Namespaced 57 | version: v1alpha1 58 | versions: 59 | - name: v1alpha1 60 | served: true 61 | storage: true 62 | --- 63 | apiVersion: apiextensions.k8s.io/v1beta1 64 | kind: CustomResourceDefinition 65 | metadata: 66 | name: workflows.argoproj.io 67 | spec: 68 | additionalPrinterColumns: 69 | - JSONPath: .status.phase 70 | description: Status of the workflow 71 | name: Status 72 | type: string 73 | - JSONPath: .status.startedAt 74 | description: When the workflow was started 75 | format: date-time 76 | name: Age 77 | type: date 78 | group: argoproj.io 79 | names: 80 | kind: Workflow 81 | listKind: WorkflowList 82 | plural: workflows 83 | shortNames: 84 | - wf 85 | singular: workflow 86 | scope: Namespaced 87 | subresources: {} 88 | version: v1alpha1 89 | versions: 90 | - name: v1alpha1 91 | served: true 92 | storage: true 93 | --- 94 | apiVersion: apiextensions.k8s.io/v1beta1 95 | kind: CustomResourceDefinition 96 | metadata: 97 | name: workflowtemplates.argoproj.io 98 | spec: 99 | group: argoproj.io 100 | names: 101 | kind: WorkflowTemplate 102 | listKind: WorkflowTemplateList 103 | plural: workflowtemplates 104 | shortNames: 105 | - wftmpl 106 | singular: workflowtemplate 107 | scope: Namespaced 108 | version: v1alpha1 109 | versions: 110 | - name: v1alpha1 111 | served: true 112 | storage: true 113 | -------------------------------------------------------------------------------- /deploy/helm/templates/argo-controller-cm.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: workflow-controller-configmap 5 | namespace: {{ .Values.argoController.namespace }} 6 | data: 7 | containerRuntimeExecutor: {{ .Values.argoController.containerRuntimeExecutor }} 8 | -------------------------------------------------------------------------------- /deploy/helm/templates/argo-controller.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Namespace 4 | metadata: 5 | name: {{ .Values.argoController.namespace }} 6 | --- 7 | apiVersion: v1 8 | kind: ServiceAccount 9 | metadata: 10 | name: argo 11 | namespace: {{ .Values.argoController.namespace }} 12 | --- 13 | apiVersion: rbac.authorization.k8s.io/v1 14 | kind: ClusterRole 15 | metadata: 16 | name: argo-role 17 | rules: 18 | - apiGroups: 19 | - "" 20 | resources: 21 | - pods 22 | - pods/exec 23 | verbs: 24 | - create 25 | - get 26 | - list 27 | - watch 28 | - update 29 | - patch 30 | - delete 31 | - apiGroups: 32 | - "" 33 | resources: 34 | - configmaps 35 | verbs: 36 | - get 37 | - watch 38 | - list 39 | - apiGroups: 40 | - "" 41 | resources: 42 | - persistentvolumeclaims 43 | verbs: 44 | - create 45 | - delete 46 | - apiGroups: 47 | - argoproj.io 48 | resources: 49 | - workflows 50 | - workflows/finalizers 51 | verbs: 52 | - get 53 | - list 54 | - watch 55 | - update 56 | - patch 57 | - delete 58 | - create 59 | - apiGroups: 60 | - argoproj.io 61 | resources: 62 | - workflowtemplates 63 | - workflowtemplates/finalizers 64 | verbs: 65 | - get 66 | - list 67 | - watch 68 | - apiGroups: 69 | - "" 70 | resources: 71 | - serviceaccounts 72 | verbs: 73 | - get 74 | - list 75 | - apiGroups: 76 | - "" 77 | resources: 78 | - secrets 79 | verbs: 80 | - get 81 | - apiGroups: 82 | - argoproj.io 83 | resources: 84 | - cronworkflows 85 | - cronworkflows/finalizers 86 | verbs: 87 | - get 88 | - list 89 | - watch 90 | - update 91 | - patch 92 | - delete 93 | - apiGroups: 94 | - "" 95 | resources: 96 | - events 97 | verbs: 98 | - create 99 | - patch 100 | - apiGroups: 101 | - policy 102 | resources: 103 | - poddisruptionbudgets 104 | verbs: 105 | - create 106 | - get 107 | - delete 108 | --- 109 | apiVersion: rbac.authorization.k8s.io/v1 110 | kind: RoleBinding 111 | metadata: 112 | name: argo-binding 113 | namespace: {{ .Values.argoController.namespace }} 114 | roleRef: 115 | apiGroup: rbac.authorization.k8s.io 116 | kind: ClusterRole 117 | name: argo-role 118 | subjects: 119 | - kind: ServiceAccount 120 | name: argo 121 | namespace: {{ .Values.argoController.namespace }} 122 | --- 123 | apiVersion: v1 124 | kind: Service 125 | metadata: 126 | name: workflow-controller-metrics 127 | namespace: {{ .Values.argoController.namespace }} 128 | spec: 129 | ports: 130 | - name: metrics 131 | port: 9090 132 | protocol: TCP 133 | targetPort: 9090 134 | selector: 135 | app: workflow-controller 136 | --- 137 | apiVersion: apps/v1 138 | kind: Deployment 139 | metadata: 140 | name: workflow-controller 141 | namespace: {{ .Values.argoController.namespace }} 142 | spec: 143 | selector: 144 | matchLabels: 145 | app: workflow-controller 146 | template: 147 | metadata: 148 | labels: 149 | app: workflow-controller 150 | annotations: 151 | checksum/config: {{ include (print $.Template.BasePath "/argo-controller-cm.yaml") . | sha256sum }} 152 | spec: 153 | containers: 154 | - args: 155 | - --configmap 156 | - workflow-controller-configmap 157 | - --executor-image 158 | - {{ .Values.argoController.executorImage }} 159 | - --namespaced 160 | - --managed-namespace 161 | - {{ .Values.runs.namespace }} 162 | command: 163 | - workflow-controller 164 | image: {{ .Values.argoController.image }} 165 | name: workflow-controller 166 | nodeSelector: 167 | kubernetes.io/os: linux 168 | serviceAccountName: argo 169 | -------------------------------------------------------------------------------- /deploy/helm/templates/argo-server.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.argoServer.enabled -}} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: argo-server 6 | namespace: {{ .Values.argoController.namespace }} 7 | --- 8 | apiVersion: rbac.authorization.k8s.io/v1 9 | kind: ClusterRole 10 | metadata: 11 | name: argo-server-role 12 | rules: 13 | - apiGroups: 14 | - "" 15 | resources: 16 | - configmaps 17 | verbs: 18 | - get 19 | - watch 20 | - list 21 | - apiGroups: 22 | - "" 23 | resources: 24 | - secrets 25 | verbs: 26 | - get 27 | - apiGroups: 28 | - "" 29 | resources: 30 | - pods 31 | - pods/exec 32 | - pods/log 33 | verbs: 34 | - get 35 | - list 36 | - watch 37 | - delete 38 | - apiGroups: 39 | - "" 40 | resources: 41 | - secrets 42 | verbs: 43 | - get 44 | - apiGroups: 45 | - argoproj.io 46 | resources: 47 | - workflows 48 | - workflowtemplates 49 | - cronworkflows 50 | verbs: 51 | - create 52 | - get 53 | - list 54 | - watch 55 | - update 56 | - patch 57 | - delete 58 | --- 59 | apiVersion: rbac.authorization.k8s.io/v1 60 | kind: RoleBinding 61 | metadata: 62 | name: argo-server-binding 63 | namespace: {{ .Values.argoController.namespace }} 64 | roleRef: 65 | apiGroup: rbac.authorization.k8s.io 66 | kind: ClusterRole 67 | name: argo-server-role 68 | subjects: 69 | - kind: ServiceAccount 70 | name: argo-server 71 | namespace: {{ .Values.argoController.namespace }} 72 | --- 73 | apiVersion: v1 74 | kind: Service 75 | metadata: 76 | name: argo-server 77 | namespace: {{ .Values.argoController.namespace }} 78 | spec: 79 | ports: 80 | - name: web 81 | port: 2746 82 | targetPort: 2746 83 | selector: 84 | app: argo-server 85 | --- 86 | apiVersion: apps/v1 87 | kind: Deployment 88 | metadata: 89 | name: argo-server 90 | namespace: {{ .Values.argoController.namespace }} 91 | spec: 92 | selector: 93 | matchLabels: 94 | app: argo-server 95 | template: 96 | metadata: 97 | labels: 98 | app: argo-server 99 | spec: 100 | containers: 101 | - args: 102 | - server 103 | - --namespaced 104 | - --managed-namespace 105 | - {{ .Values.runs.namespace }} 106 | image: {{ .Values.argoServer.image }} 107 | name: argo-server 108 | ports: 109 | - containerPort: 2746 110 | name: web 111 | readinessProbe: 112 | httpGet: 113 | path: / 114 | port: 2746 115 | scheme: HTTP 116 | initialDelaySeconds: 10 117 | periodSeconds: 20 118 | nodeSelector: 119 | kubernetes.io/os: linux 120 | serviceAccountName: argo-server 121 | {{- end }} 122 | -------------------------------------------------------------------------------- /deploy/helm/templates/runs-namespace.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: {{ .Values.runs.namespace }} 5 | --- 6 | apiVersion: rbac.authorization.k8s.io/v1 7 | kind: Role 8 | metadata: 9 | name: workflow-creator 10 | namespace: {{ .Values.runs.namespace }} 11 | rules: 12 | - apiGroups: 13 | - argoproj.io 14 | resources: 15 | - workflows 16 | verbs: 17 | - create 18 | --- 19 | apiVersion: rbac.authorization.k8s.io/v1 20 | kind: Role 21 | metadata: 22 | name: argo-workflow 23 | namespace: {{ .Values.runs.namespace }} 24 | rules: 25 | - apiGroups: 26 | - "" 27 | resources: 28 | - pods 29 | verbs: 30 | - get 31 | - watch 32 | - patch 33 | - apiGroups: 34 | - "" 35 | resources: 36 | - pods/log 37 | verbs: 38 | - get 39 | - watch 40 | --- 41 | apiVersion: rbac.authorization.k8s.io/v1 42 | kind: RoleBinding 43 | metadata: 44 | name: argo-binding 45 | namespace: {{ .Values.runs.namespace }} 46 | roleRef: 47 | apiGroup: rbac.authorization.k8s.io 48 | kind: ClusterRole 49 | name: argo-role 50 | subjects: 51 | - kind: ServiceAccount 52 | name: argo 53 | namespace: {{ .Values.argoController.namespace }} 54 | --- 55 | apiVersion: rbac.authorization.k8s.io/v1 56 | kind: RoleBinding 57 | metadata: 58 | name: argo-server-binding 59 | namespace: {{ .Values.runs.namespace }} 60 | roleRef: 61 | apiGroup: rbac.authorization.k8s.io 62 | kind: ClusterRole 63 | name: argo-server-role 64 | subjects: 65 | - kind: ServiceAccount 66 | name: argo-server 67 | namespace: {{ .Values.argoController.namespace }} 68 | --- 69 | apiVersion: rbac.authorization.k8s.io/v1 70 | kind: RoleBinding 71 | metadata: 72 | name: runs-default-workflow 73 | namespace: {{ .Values.runs.namespace }} 74 | roleRef: 75 | apiGroup: rbac.authorization.k8s.io 76 | kind: Role 77 | name: argo-workflow 78 | subjects: 79 | - kind: ServiceAccount 80 | name: default 81 | namespace: {{ .Values.runs.namespace }} 82 | --- 83 | apiVersion: rbac.authorization.k8s.io/v1 84 | kind: RoleBinding 85 | metadata: 86 | name: worker-binding 87 | namespace: {{ .Values.runs.namespace }} 88 | subjects: 89 | - kind: ServiceAccount 90 | name: worker 91 | namespace: {{ .Values.worker.namespace }} 92 | roleRef: 93 | kind: Role 94 | name: workflow-creator 95 | apiGroup: rbac.authorization.k8s.io 96 | --- 97 | {{ if .Values.identities.benchmark.resourceId }} 98 | apiVersion: aadpodidentity.k8s.io/v1 99 | kind: AzureIdentityBinding 100 | metadata: 101 | name: benchmark 102 | namespace: {{ .Values.runs.namespace }} 103 | spec: 104 | azureIdentity: benchmark 105 | selector: benchmark 106 | --- 107 | apiVersion: aadpodidentity.k8s.io/v1 108 | kind: AzureIdentity 109 | metadata: 110 | name: benchmark 111 | namespace: {{ .Values.runs.namespace }} 112 | spec: 113 | type: 0 114 | resourceID: {{ .Values.identities.benchmark.resourceId }} 115 | clientID: {{ .Values.identities.benchmark.clientId }} 116 | --- 117 | {{- end }} 118 | 119 | {{- if .Values.identities.candidate.resourceId }} 120 | apiVersion: aadpodidentity.k8s.io/v1 121 | kind: AzureIdentityBinding 122 | metadata: 123 | name: candidate 124 | namespace: {{ .Values.runs.namespace }} 125 | spec: 126 | azureIdentity: candidate 127 | selector: candidate 128 | --- 129 | apiVersion: aadpodidentity.k8s.io/v1 130 | kind: AzureIdentity 131 | metadata: 132 | name: candidate 133 | namespace: {{ .Values.runs.namespace }} 134 | spec: 135 | type: 0 136 | resourceID: {{ .Values.identities.candidate.resourceId }} 137 | clientID: {{ .Values.identities.candidate.clientId }} 138 | --- 139 | {{- end }} 140 | -------------------------------------------------------------------------------- /deploy/helm/templates/sds-worker.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: {{ .Values.worker.namespace }} 5 | --- 6 | apiVersion: v1 7 | kind: ServiceAccount 8 | metadata: 9 | name: worker 10 | namespace: {{ .Values.worker.namespace }} 11 | --- 12 | apiVersion: apps/v1 13 | kind: Deployment 14 | metadata: 15 | name: worker 16 | namespace: {{ .Values.worker.namespace }} 17 | labels: 18 | app: worker 19 | spec: 20 | replicas: 1 21 | selector: 22 | matchLabels: 23 | app: worker 24 | template: 25 | metadata: 26 | labels: 27 | app: worker 28 | aadpodidbinding: worker 29 | spec: 30 | serviceAccountName: worker 31 | containers: 32 | - name: worker 33 | image: {{ .Values.worker.image }} 34 | imagePullPolicy: Always 35 | resources: 36 | limits: 37 | cpu: "1" 38 | memory: 1Gi 39 | env: 40 | - name: QUEUE_MODE 41 | value: {{ .Values.worker.queueMode }} 42 | - name: QUEUE_ENDPOINT 43 | value: {{ .Values.worker.queueEndpoint }} 44 | - name: AZURE_CLIENT_ID 45 | value: {{ .Values.identities.worker.clientId }} 46 | {{- if .Values.instrumentationKey }} 47 | - name: APPINSIGHTS_INSTRUMENTATIONKEY 48 | value: {{ .Values.instrumentationKey }} 49 | {{- end }} 50 | {{- if and .Values.identities.worker.tenantId .Values.identities.worker.clientSecret }} 51 | - name: AZURE_TENANT_ID 52 | value: {{ .Values.identities.worker.tenantId }} 53 | - name: AZURE_CLIENT_SECRET 54 | valueFrom: 55 | secretKeyRef: 56 | name: worker-identity-sp 57 | key: clientSecret 58 | {{- end }} 59 | {{- if .Values.worker.storage.storageClassName }} 60 | - name: STORAGE_CLASS_NAME 61 | value: {{ .Values.worker.storage.storageClassName }} 62 | {{- end }} 63 | --- 64 | {{- if and .Values.identities.worker.tenantId .Values.identities.worker.clientSecret }} 65 | apiVersion: v1 66 | kind: Secret 67 | metadata: 68 | name: worker-identity-sp 69 | namespace: worker 70 | data: 71 | clientSecret: {{ .Values.identities.worker.clientSecret | b64enc }} 72 | --- 73 | {{ else if .Values.identities.worker.resourceId }} 74 | apiVersion: aadpodidentity.k8s.io/v1 75 | kind: AzureIdentityBinding 76 | metadata: 77 | name: worker 78 | namespace: {{ .Values.worker.namespace }} 79 | spec: 80 | azureIdentity: worker 81 | selector: worker 82 | --- 83 | apiVersion: aadpodidentity.k8s.io/v1 84 | kind: AzureIdentity 85 | metadata: 86 | name: worker 87 | namespace: {{ .Values.worker.namespace }} 88 | spec: 89 | type: 0 90 | resourceID: {{ .Values.identities.worker.resourceId }} 91 | clientID: {{ .Values.identities.worker.clientId }} 92 | --- 93 | {{- end }} 94 | {{- if .Values.worker.storage.createStorageClass }} 95 | apiVersion: storage.k8s.io/v1 96 | kind: StorageClass 97 | metadata: 98 | name: {{ .Values.worker.storage.storageClassName }} 99 | provisioner: file.csi.azure.com 100 | parameters: 101 | resourceGroup: {{ .Values.worker.storage.resourceGroup }} 102 | storageAccount: {{ .Values.worker.storage.storageAccount }} 103 | reclaimPolicy: Delete 104 | volumeBindingMode: Immediate 105 | --- 106 | {{- end }} 107 | -------------------------------------------------------------------------------- /deploy/helm/values.dev.yaml: -------------------------------------------------------------------------------- 1 | # Default values for sds-worker development. 2 | 3 | instrumentationKey: 00000000-0000-0000-0000-000000000000 4 | 5 | argoController: 6 | containerRuntimeExecutor: pns 7 | executorImage: argoproj/argoexec:v2.11.0 8 | image: argoproj/workflow-controller:v2.11.0 9 | namespace: argo 10 | 11 | # Deploy argo-server 12 | argoServer: 13 | enabled: true 14 | image: argoproj/argocli:v2.11.0 15 | 16 | runs: 17 | namespace: runs 18 | 19 | worker: 20 | namespace: worker 21 | image: acanthamoeba/sds-worker:dev 22 | queueMode: azure 23 | storage: 24 | storageClassName: null 25 | createStorageClass: false 26 | 27 | # Use Service Principals instead of Managed Identity 28 | identities: 29 | worker: 30 | tenantId: 00000000-0000-0000-0000-000000000000 31 | clientId: 00000000-0000-0000-0000-000000000000 32 | clientSecret: 00000000-0000-0000-0000-000000000000 33 | benchmark: 34 | tenantId: 00000000-0000-0000-0000-000000000000 35 | clientId: 00000000-0000-0000-0000-000000000000 36 | clientSecret: 00000000-0000-0000-0000-000000000000 37 | candidate: 38 | tenantId: 00000000-0000-0000-0000-000000000000 39 | clientId: 00000000-0000-0000-0000-000000000000 40 | clientSecret: 00000000-0000-0000-0000-000000000000 41 | -------------------------------------------------------------------------------- /deploy/helm/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for sds-worker. 2 | 3 | # instrumentationKey: 4 | 5 | argoController: 6 | # Set `containerRuntimeExecutor: pns` when developing on kind 7 | containerRuntimeExecutor: docker 8 | executorImage: argoproj/argoexec:v2.11.0 9 | image: argoproj/workflow-controller:v2.11.0 10 | namespace: argo 11 | 12 | argoServer: 13 | # Determines whether to deploy the optional argo-server UX 14 | enabled: false 15 | image: argoproj/argocli:v2.11.0 16 | 17 | runs: 18 | namespace: runs 19 | 20 | worker: 21 | namespace: worker 22 | image: acanthamoeba/sds-worker 23 | queueMode: azure 24 | # Set the queue endpoint 25 | # queueEndpoint: https://.queue.core.windows.net/runs 26 | # Transient storage for runs. The AKS kubelet identity will need Storage Account Contributor over this account 27 | storage: 28 | createStorageClass: true 29 | storageClassName: runs-transient 30 | # resourceGroup: 31 | # storageAccount: 32 | 33 | identities: 34 | worker: {} 35 | # (required) Set the clientId associated with the identity 36 | # clientId: 00000000-0000-0000-0000-000000000000 37 | 38 | # Set the following to use a Managed Identity on AKS 39 | # resourceId: /subscriptions//resourceGroups//providers/Microsoft.ManagedIdentity/userAssignedIdentities/worker 40 | 41 | # Set the following to use a Service Principal during development 42 | # tenantId: 00000000-0000-0000-0000-000000000000 43 | # clientSecret: 00000000-0000-0000-0000-000000000000 44 | 45 | benchmark: {} 46 | # (required) Set the clientId associated with the identity 47 | # clientId: 00000000-0000-0000-0000-000000000000 48 | 49 | # Set the following to use a Managed Identity on AKS 50 | # resourceId: /subscriptions//resourceGroups//providers/Microsoft.ManagedIdentity/userAssignedIdentities/worker 51 | 52 | # Set the following to use a Service Principal during development 53 | # tenantId: 00000000-0000-0000-0000-000000000000 54 | # clientSecret: 00000000-0000-0000-0000-000000000000 55 | 56 | candidate: {} 57 | # (required) Set the clientId associated with the identity 58 | # clientId: 00000000-0000-0000-0000-000000000000 59 | 60 | # Set the following to use a Managed Identity on AKS 61 | # resourceId: /subscriptions//resourceGroups//providers/Microsoft.ManagedIdentity/userAssignedIdentities/worker 62 | 63 | # Set the following to use a Service Principal during development 64 | # tenantId: 00000000-0000-0000-0000-000000000000 65 | # clientSecret: 00000000-0000-0000-0000-000000000000 66 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.4" 2 | services: 3 | laboratory: 4 | build: 5 | context: . 6 | target: laboratory 7 | 8 | cli: 9 | build: 10 | context: . 11 | target: cli 12 | command: /bin/sh -c 'sleep 5 && sds-cli connect http://laboratory:3000 && sds-cli demo' 13 | depends_on: 14 | - laboratory 15 | -------------------------------------------------------------------------------- /docs/images/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/secure-data-sandbox/6b6df0781ebab237dface4597986d72293ff26e4/docs/images/.gitkeep -------------------------------------------------------------------------------- /lerna.json: -------------------------------------------------------------------------------- 1 | { 2 | "packages": [ 3 | "packages/sds", 4 | "packages/cli", 5 | "packages/worker", 6 | "packages/laboratory" 7 | ], 8 | "version": "0.0.0" 9 | } 10 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "root", 3 | "private": true, 4 | "scripts": { 5 | "catdetection": "./scripts/demo-catdetection.sh", 6 | "check": "lerna run check", 7 | "cli": "npm run start --silent --prefix packages/cli", 8 | "compile": "lerna run compile", 9 | "deploy:acr": "./deploy/deploy-acr.sh", 10 | "deploy:aks": "./deploy/deploy-aks.sh", 11 | "deploy:arm": "./deploy/deploy-arm.sh -g", 12 | "predev:azure": "serve -l 9001 deploy >/dev/null &", 13 | "dev:azure": "ngrok start -config .ngrok.yml arm", 14 | "dev:deploy:arm": "./deploy/deploy-arm.sh --dev -g", 15 | "dev:updateWorker": "./scripts/dev-worker.sh", 16 | "postdev:azure": "kill $(lsof -t -i:9001)", 17 | "fix": "lerna run fix", 18 | "laboratory": "dotenv -- npm run start --silent --prefix packages/laboratory", 19 | "pack": "lerna bootstrap --ignore-scripts && lerna exec npm pack", 20 | "pack:laboratory:appservice": "rimraf ./dist/laboratory && mkdir -p ./dist/laboratory && npm run pack && cd ./dist/laboratory && npm init -y && npm install --only=prod ../../packages/sds/*.tgz ../../packages/laboratory/*.tgz && zip -r sds-laboratory.zip package*.json node_modules", 21 | "postinstall": "lerna bootstrap", 22 | "test": "lerna run test" 23 | }, 24 | "devDependencies": { 25 | "@istanbuljs/nyc-config-typescript": "^1.0.1", 26 | "dotenv-cli": "^4.0.0", 27 | "gts": "^2.0.2", 28 | "lerna": "^3.22.1", 29 | "ngrok": "^3.3.0", 30 | "nyc": "^15.1.0", 31 | "rimraf": "^3.0.2", 32 | "serve": "^11.3.2", 33 | "source-map-support": "^0.5.19", 34 | "ts-node": "^9.0.0", 35 | "typescript": "^4.0.3" 36 | }, 37 | "dependencies": {} 38 | } 39 | -------------------------------------------------------------------------------- /packages/cli/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../.eslintrc.json" 3 | } 4 | -------------------------------------------------------------------------------- /packages/cli/.nycrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../.nycrc" 3 | } 4 | -------------------------------------------------------------------------------- /packages/cli/.prettierignore: -------------------------------------------------------------------------------- 1 | dist 2 | node_modules 3 | -------------------------------------------------------------------------------- /packages/cli/.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | ...require('../../.prettierrc.js'), 3 | } 4 | -------------------------------------------------------------------------------- /packages/cli/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@microsoft/sds-cli", 3 | "version": "0.1.0", 4 | "repository": { 5 | "url": "https://github.com/microsoft/secure-data-sandbox" 6 | }, 7 | "description": "A toolkit for conducting machine learning trials against confidential data", 8 | "license": "MIT", 9 | "main": "dist/index", 10 | "typings": "dist/index", 11 | "files": [ 12 | "dist" 13 | ], 14 | "engines": { 15 | "node": ">=12" 16 | }, 17 | "bin": { 18 | "sds-cli": "./dist/index.js" 19 | }, 20 | "scripts": { 21 | "check": "gts check", 22 | "clean": "gts clean", 23 | "compile": "tsc -p tsconfig.build.json", 24 | "fix": "gts fix", 25 | "prepare": "npm run compile", 26 | "posttest": "npm run check", 27 | "start": "ts-node src/index.ts", 28 | "test": "nyc ts-mocha" 29 | }, 30 | "devDependencies": { 31 | "@types/chai": "^4.2.12", 32 | "@types/chai-as-promised": "^7.1.3", 33 | "@types/js-yaml": "^3.12.5", 34 | "@types/luxon": "^1.25.0", 35 | "@types/mocha": "^8.0.3", 36 | "@types/node": "^13.13.21", 37 | "@types/uuid": "^8.3.0", 38 | "chai": "^4.2.0", 39 | "chai-as-promised": "^7.1.1", 40 | "eslint": "^7.10.0", 41 | "gts": "^2.0.2", 42 | "mocha": "^8.1.3", 43 | "nyc": "^15.1.0", 44 | "source-map-support": "^0.5.19", 45 | "ts-mocha": "^7.0.0", 46 | "ts-node": "^9.0.0", 47 | "typescript": "^4.0.3", 48 | "uuid": "^8.3.0" 49 | }, 50 | "dependencies": { 51 | "@azure/msal-node": "^1.0.0-alpha.9", 52 | "@microsoft/sds": "*", 53 | "applicationinsights": "^1.8.7", 54 | "axios": "^0.19.2", 55 | "commander": "^6.1.0", 56 | "fp-ts": "^2.8.2", 57 | "io-ts": "^2.2.10", 58 | "js-yaml": "^3.14.0", 59 | "luxon": "^1.25.0" 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /packages/cli/src/conn.ts: -------------------------------------------------------------------------------- 1 | import * as msal from '@azure/msal-node'; 2 | import { promises as fs } from 'fs'; 3 | import * as path from 'path'; 4 | import * as os from 'os'; 5 | import * as yaml from 'js-yaml'; 6 | import * as t from 'io-ts'; 7 | import * as url from 'url'; 8 | 9 | import { 10 | IllegalOperationError, 11 | LaboratoryClient, 12 | validate, 13 | ClientConnectionInfoType, 14 | } from '@microsoft/sds'; 15 | 16 | const defaultConnFilePath = 'sds.yaml'; 17 | 18 | const ConnectConfigurationType = t.intersection([ 19 | t.type({ 20 | endpoint: t.string, 21 | }), 22 | ClientConnectionInfoType, 23 | t.partial({ 24 | // from msal-common/AccountInfo 25 | account: t.type({ 26 | homeAccountId: t.string, 27 | environment: t.string, 28 | tenantId: t.string, 29 | username: t.string, 30 | }), 31 | }), 32 | ]); 33 | type IConnectConfiguration = t.TypeOf; 34 | 35 | export class LaboratoryConnection { 36 | static configDir: string = path.join(os.homedir(), '.sds'); 37 | 38 | private client: LaboratoryClient | undefined; 39 | 40 | connFilePath: string; 41 | tokenCachePath: string; 42 | 43 | constructor(connFilePath: string = defaultConnFilePath) { 44 | this.connFilePath = connFilePath; 45 | this.tokenCachePath = `${path.basename(connFilePath)}-accessTokens.json`; 46 | } 47 | 48 | async init(host: string) { 49 | const labUrl = new url.URL(host); 50 | const endpoint = labUrl.href; 51 | 52 | const connectionInfo = await new LaboratoryClient( 53 | endpoint 54 | ).negotiateConnection(); 55 | const config: IConnectConfiguration = { 56 | endpoint, 57 | ...connectionInfo, 58 | }; 59 | await this.writeConfig(yaml.safeDump(config)); 60 | const newClient = this.buildClient(config); 61 | await newClient.validateConnection(); 62 | this.client = newClient; 63 | } 64 | 65 | async getClient(): Promise { 66 | try { 67 | if (this.client) { 68 | return this.client; 69 | } 70 | 71 | const text = await this.readConfig(); 72 | const config = validate(ConnectConfigurationType, yaml.safeLoad(text)); 73 | this.client = this.buildClient(config); 74 | return this.client; 75 | } catch { 76 | throw new IllegalOperationError( 77 | 'No laboratory connection. Use the "connect" command to specify a laboratory.' 78 | ); 79 | } 80 | } 81 | 82 | private buildClient(config: IConnectConfiguration): LaboratoryClient { 83 | const tokenRetriever = 84 | config.type === 'aad' ? this.acquireAADAccessToken(config) : undefined; 85 | return new LaboratoryClient(config.endpoint, tokenRetriever); 86 | } 87 | 88 | private async readConfig(): Promise { 89 | const fullPath = path.join( 90 | LaboratoryConnection.configDir, 91 | this.connFilePath 92 | ); 93 | return await fs.readFile(fullPath, 'utf8'); 94 | } 95 | 96 | private async writeConfig(data: string): Promise { 97 | const fullPath = path.join( 98 | LaboratoryConnection.configDir, 99 | this.connFilePath 100 | ); 101 | await fs.mkdir(LaboratoryConnection.configDir, { recursive: true }); 102 | await fs.writeFile(fullPath, data); 103 | } 104 | 105 | private acquireAADAccessToken(config: IConnectConfiguration) { 106 | if (config.type !== 'aad') { 107 | throw new Error( 108 | 'Cannot retrieve an AAD access token for a non-AAD connection' 109 | ); 110 | } 111 | 112 | return async () => { 113 | const tokenCachePath = this.tokenCachePath; 114 | const pca = new msal.PublicClientApplication({ 115 | auth: { 116 | clientId: config.clientId, 117 | authority: config.authority, 118 | }, 119 | cache: { 120 | cachePlugin: { 121 | async readFromStorage() { 122 | const fullPath = path.join( 123 | LaboratoryConnection.configDir, 124 | tokenCachePath 125 | ); 126 | try { 127 | return await fs.readFile(fullPath, 'utf8'); 128 | } catch (err) { 129 | if (err && err.code === 'ENOENT') { 130 | return ''; 131 | } 132 | throw err; 133 | } 134 | }, 135 | async writeToStorage(getMergedState: (oldState: string) => string) { 136 | let oldFile = ''; 137 | try { 138 | oldFile = await this.readFromStorage(); 139 | } finally { 140 | const mergedState = getMergedState(oldFile); 141 | const fullPath = path.join( 142 | LaboratoryConnection.configDir, 143 | tokenCachePath 144 | ); 145 | await fs.mkdir(LaboratoryConnection.configDir, { 146 | recursive: true, 147 | }); 148 | await fs.writeFile(fullPath, mergedState); 149 | } 150 | }, 151 | }, 152 | }, 153 | }); 154 | const cache = pca.getTokenCache(); 155 | 156 | try { 157 | await cache.readFromPersistence(); 158 | const silentResult = await pca.acquireTokenSilent({ 159 | account: config.account!, 160 | scopes: config.scopes, 161 | }); 162 | cache.writeToPersistence(); 163 | return silentResult.accessToken; 164 | } catch (e) { 165 | const deviceCodeResult = await pca.acquireTokenByDeviceCode({ 166 | deviceCodeCallback: response => console.log(response.message), 167 | scopes: config.scopes, 168 | }); 169 | config.account = deviceCodeResult.account; 170 | await this.writeConfig(yaml.safeDump(config)); 171 | cache.writeToPersistence(); 172 | return deviceCodeResult.accessToken; 173 | } 174 | }; 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /packages/cli/src/decode_error.ts: -------------------------------------------------------------------------------- 1 | import { AxiosError } from 'axios'; 2 | import { URL } from 'url'; 3 | 4 | export function decodeError(e: NodeJS.ErrnoException | AxiosError): string { 5 | const axiosError = e as AxiosError; 6 | if (axiosError.isAxiosError) { 7 | const response = axiosError.response; 8 | if (response) { 9 | if (response.data && response.data.error) { 10 | // This case can be triggered by attempting to access a non-existent 11 | // benchmark, candidate, run, or suite. 12 | return response.status + ': ' + response.data.error.message; 13 | } else { 14 | // This case can be triggered by getting from a website like microsoft.com. 15 | const method = axiosError.config.method; 16 | const url = axiosError.config.url; 17 | return `${response.status}: cannot ${method} ${url}`; 18 | } 19 | } else { 20 | // We didn't even get a response. 21 | if (e.code === 'ENOTFOUND') { 22 | const method = axiosError.config.method; 23 | const url = axiosError.config.url; 24 | return `cannot ${method} ${url}`; 25 | } else if (e.code === 'ECONNREFUSED') { 26 | const url = new URL(axiosError.config.url!); 27 | const host = url.host; 28 | return `cannot connect to ${host}`; 29 | } 30 | // Fall through 31 | } 32 | } 33 | 34 | if (e.code === 'ENOENT') { 35 | // Most likely file not found. 36 | const message = `cannot open file "${(e as NodeJS.ErrnoException).path}".`; 37 | return message; 38 | } 39 | 40 | // Some other type of error. 41 | return e.message; 42 | } 43 | -------------------------------------------------------------------------------- /packages/cli/src/demo.ts: -------------------------------------------------------------------------------- 1 | import * as yaml from 'js-yaml'; 2 | 3 | import { 4 | IBenchmark, 5 | ICandidate, 6 | ISuite, 7 | LaboratoryClient, 8 | RunStatus, 9 | BenchmarkStageKind, 10 | } from '@microsoft/sds'; 11 | 12 | const benchmark1: IBenchmark = { 13 | name: 'benchmark1', 14 | author: 'author1', 15 | apiVersion: 'v1alpha1', 16 | stages: [ 17 | { 18 | // Candidate 19 | name: 'candidate', 20 | kind: BenchmarkStageKind.CANDIDATE, 21 | volumes: [ 22 | { 23 | name: 'training', 24 | path: '/input', 25 | readonly: true, 26 | }, 27 | ], 28 | }, 29 | { 30 | // Benchmark 31 | name: 'scoring', 32 | image: 'benchmark-image', 33 | kind: BenchmarkStageKind.CONTAINER, 34 | volumes: [ 35 | { 36 | name: 'reference', 37 | path: '/reference', 38 | readonly: true, 39 | }, 40 | ], 41 | }, 42 | ], 43 | }; 44 | 45 | const candidate1: ICandidate = { 46 | name: 'candidate1', 47 | author: 'author1', 48 | apiVersion: 'v1alpha1', 49 | benchmark: 'benchmark1', 50 | image: 'candidate1-image', 51 | }; 52 | 53 | const suite1: ISuite = { 54 | name: 'suite1', 55 | author: 'author1', 56 | apiVersion: 'v1alpha1', 57 | benchmark: 'benchmark1', 58 | volumes: [ 59 | { 60 | name: 'training', 61 | type: 'AzureBlob', 62 | target: 'https://sample.blob.core.windows.net/training', 63 | }, 64 | { 65 | name: 'reference', 66 | type: 'AzureBlob', 67 | target: 'https://sample.blob.core.windows.net/reference', 68 | }, 69 | ], 70 | }; 71 | 72 | export async function configureDemo(lab: LaboratoryClient) { 73 | await lab.upsertBenchmark(benchmark1); 74 | await lab.upsertCandidate(candidate1); 75 | await lab.upsertSuite(suite1); 76 | 77 | const run1 = await lab.createRunRequest({ 78 | candidate: candidate1.name, 79 | suite: suite1.name, 80 | }); 81 | await lab.updateRunStatus(run1.name, RunStatus.COMPLETED); 82 | await lab.reportRunResults(run1.name, { passed: 5, failed: 6 }); 83 | 84 | const run2 = await lab.createRunRequest({ 85 | candidate: candidate1.name, 86 | suite: suite1.name, 87 | }); 88 | await lab.updateRunStatus(run2.name, RunStatus.COMPLETED); 89 | await lab.reportRunResults(run2.name, { passed: 3, skipped: 7 }); 90 | 91 | console.log(); 92 | console.log('=== Sample benchmark ==='); 93 | console.log(yaml.safeDump(benchmark1)); 94 | 95 | console.log(); 96 | console.log('=== Sample candidate ==='); 97 | console.log(yaml.safeDump(candidate1)); 98 | 99 | console.log(); 100 | console.log('=== Sample suite ==='); 101 | console.log(yaml.safeDump(suite1)); 102 | 103 | console.log(); 104 | console.log(`Initiated run ${run1.name}`); 105 | console.log(`Initiated run ${run2.name}`); 106 | } 107 | -------------------------------------------------------------------------------- /packages/cli/src/formatting.ts: -------------------------------------------------------------------------------- 1 | export enum Alignment { 2 | LEFT = 'left', 3 | RIGHT = 'right', 4 | } 5 | 6 | export function* formatTable( 7 | alignments: Alignment[], 8 | rows: string[][] 9 | ): IterableIterator { 10 | const widths = new Array(alignments.length).fill(0); 11 | for (const row of rows) { 12 | for (let i = 0; i < row.length; ++i) { 13 | widths[i] = Math.max(widths[i], row[i].length); 14 | } 15 | } 16 | for (const row of rows) { 17 | const fields = row.map((column, i) => { 18 | switch (alignments[i]) { 19 | case Alignment.LEFT: 20 | return leftJustify(row[i], widths[i]); 21 | case Alignment.RIGHT: 22 | return rightJustify(row[i], widths[i]); 23 | } 24 | }); 25 | 26 | yield fields.join(' '); 27 | } 28 | } 29 | 30 | export function leftJustify(text: string, width: number) { 31 | if (text.length >= width) { 32 | return text; 33 | } else { 34 | const paddingWidth = width - text.length; 35 | const padding = new Array(paddingWidth + 1).join(' '); 36 | return text + padding; 37 | } 38 | } 39 | 40 | export function rightJustify(text: string, width: number) { 41 | if (text.length >= width) { 42 | return text; 43 | } else { 44 | const paddingWidth = width - text.length; 45 | const padding = new Array(paddingWidth + 1).join(' '); 46 | return padding + text; 47 | } 48 | } 49 | 50 | export function formatChoices(choices: string[]) { 51 | if (choices.length === 0) { 52 | throw new TypeError('internal error'); 53 | } else if (choices.length === 1) { 54 | return choices[0]; 55 | } else { 56 | return ( 57 | choices.slice(0, -1).join(', ') + ', or ' + choices[choices.length - 1] 58 | ); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /packages/cli/src/shims.d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-empty-interface */ 2 | 3 | // superagent 4 | // https://github.com/DefinitelyTyped/DefinitelyTyped/issues/12044 5 | declare interface XMLHttpRequest {} 6 | -------------------------------------------------------------------------------- /packages/cli/test/.mocharc.yml: -------------------------------------------------------------------------------- 1 | recursive: true 2 | require: 'source-map-support/register' 3 | spec: '**/*.ts' 4 | -------------------------------------------------------------------------------- /packages/cli/test/conn.test.ts: -------------------------------------------------------------------------------- 1 | import * as chai from 'chai'; 2 | import chaiAsPromised = require('chai-as-promised'); 3 | import 'mocha'; 4 | import { assert } from 'chai'; 5 | 6 | import * as fs from 'fs'; 7 | import { Server } from 'http'; 8 | import * as path from 'path'; 9 | import { v1 } from 'uuid'; 10 | 11 | import { LaboratoryConnection } from '../src/conn'; 12 | import { initTestEnvironment } from '../../laboratory/test/sequelize_laboratory/shared'; 13 | import { createApp } from '../../laboratory/src'; 14 | 15 | chai.use(chaiAsPromised); 16 | 17 | describe('LaboratoryConnection', () => { 18 | const connFilePath = `test-${v1()}.yaml`; 19 | let server: Server; 20 | 21 | before(async () => { 22 | const lab = await initTestEnvironment(); 23 | const app = await createApp(lab); 24 | server = app.listen(3001); 25 | }); 26 | 27 | describe('getClient', () => { 28 | it('fails when no connection is present', () => { 29 | const conn = new LaboratoryConnection(connFilePath); 30 | assert.isRejected(conn.getClient()); 31 | }); 32 | }); 33 | 34 | describe('init', () => { 35 | it('fails when laboratory is not available', async () => { 36 | const conn = new LaboratoryConnection(connFilePath); 37 | assert.isRejected(conn.init('http://localhost:65535')); 38 | }); 39 | 40 | it('succeeds against an unauthenticated laboratory', async () => { 41 | const conn = new LaboratoryConnection(connFilePath); 42 | await conn.init('http://localhost:3001'); 43 | }); 44 | }); 45 | 46 | describe('getLabClient', () => { 47 | it('gets a client', async () => { 48 | const conn = new LaboratoryConnection(connFilePath); 49 | const client = await conn.getClient(); 50 | assert.isNotNull(client); 51 | }); 52 | 53 | it('returns a cached client', async () => { 54 | const conn = new LaboratoryConnection(connFilePath); 55 | const client = await conn.getClient(); 56 | const client2 = await conn.getClient(); 57 | assert.equal(client2, client); 58 | }); 59 | }); 60 | 61 | after(() => { 62 | if (server) { 63 | server.close(); 64 | } 65 | const connFileAbsPath = path.join( 66 | LaboratoryConnection.configDir, 67 | connFilePath 68 | ); 69 | if (fs.existsSync(connFileAbsPath)) { 70 | fs.unlinkSync(connFileAbsPath); 71 | } 72 | }); 73 | }); 74 | -------------------------------------------------------------------------------- /packages/cli/test/formatting.test.ts: -------------------------------------------------------------------------------- 1 | import { assert, expect } from 'chai'; 2 | import 'mocha'; 3 | 4 | import { 5 | leftJustify, 6 | rightJustify, 7 | formatChoices, 8 | formatTable, 9 | Alignment, 10 | } from '../src/formatting'; 11 | 12 | describe('formatting', () => { 13 | describe('leftJustify', () => { 14 | it('returns the input for long strings', () => { 15 | const text = 'thisisaverylongstring'; 16 | const result = leftJustify(text, 5); 17 | assert.equal(result, text); 18 | }); 19 | 20 | it('pads the right side of the text', () => { 21 | const text = 'input'; 22 | const result = leftJustify(text, 20); 23 | 24 | const expected = 'input '; 25 | assert.equal(result, expected); 26 | assert.equal(result.length, 20); 27 | }); 28 | }); 29 | 30 | describe('rightJustify', () => { 31 | it('returns the input for long strings', () => { 32 | const text = 'thisisaverylongstring'; 33 | const result = rightJustify(text, 5); 34 | assert.equal(result, text); 35 | }); 36 | 37 | it('pads the left side of the text', () => { 38 | const text = 'input'; 39 | const result = rightJustify(text, 20); 40 | 41 | const expected = ' input'; 42 | assert.equal(result, expected); 43 | assert.equal(result.length, 20); 44 | }); 45 | }); 46 | 47 | describe('formatChoices', () => { 48 | it('throws an error on no choices', () => { 49 | const choices: string[] = []; 50 | expect(() => formatChoices(choices)).to.throw('internal error'); 51 | }); 52 | 53 | it('handles a single choice', () => { 54 | const choices = ['highlander']; 55 | const result = formatChoices(choices); 56 | assert.equal(result, 'highlander'); 57 | }); 58 | 59 | it('handles many choices', () => { 60 | const choices = ['one', 'two', 'three']; 61 | const result = formatChoices(choices); 62 | assert.equal(result, 'one, two, or three'); 63 | }); 64 | }); 65 | 66 | describe('formatTable', () => { 67 | it('renders a table', () => { 68 | const alignments = [Alignment.LEFT, Alignment.LEFT, Alignment.RIGHT]; 69 | const rows = [ 70 | ['candidate1', 'suite1', '100'], 71 | ['candidate1', 'suite2', '200'], 72 | ['some-other-candidate', 'some-other-suite', '9001'], 73 | ]; 74 | const result = [...formatTable(alignments, rows)].join('\n'); 75 | 76 | const expected = `candidate1 suite1 100 77 | candidate1 suite2 200 78 | some-other-candidate some-other-suite 9001`; 79 | assert.equal(result, expected); 80 | }); 81 | }); 82 | }); 83 | -------------------------------------------------------------------------------- /packages/cli/test/index.ts: -------------------------------------------------------------------------------- 1 | require('mocha'); 2 | import * as appInsights from 'applicationinsights'; 3 | 4 | before(() => { 5 | appInsights.setup('00000000-0000-0000-0000-000000000000'); 6 | appInsights.defaultClient.config.disableAppInsights = true; 7 | }); 8 | -------------------------------------------------------------------------------- /packages/cli/test/sds.test.ts: -------------------------------------------------------------------------------- 1 | import { assert } from 'chai'; 2 | import 'mocha'; 3 | 4 | describe('sds', () => { 5 | it('placeholder', () => { 6 | assert.isTrue(true); 7 | }); 8 | }); 9 | -------------------------------------------------------------------------------- /packages/cli/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.build.json", 3 | "compilerOptions": { 4 | "outDir": "./dist", 5 | }, 6 | "include": [ 7 | "src/**/*" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /packages/cli/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json" 3 | } 4 | -------------------------------------------------------------------------------- /packages/laboratory/.env.template: -------------------------------------------------------------------------------- 1 | # To use in bash: 2 | # set -o allexport; source .env; set +o allexport 3 | 4 | # Laboratory server configuration 5 | AZURE_CLIENT_ID=00000000-0000-0000-0000-000000000000 6 | QUEUE_MODE=azure 7 | QUEUE_ENDPOINT=https://mystorage.queue.core.windows.net/myqueue 8 | SQL_MODE=azuresql 9 | SQL_HOST=mydatabase.database.windows.net 10 | SQL_DB=laboratory 11 | 12 | # Laboratory AAD auth 13 | # If AUTH_MODE is not set, auth is disabled 14 | AUTH_MODE=aad 15 | AUTH_TENANT_ID=00000000-0000-0000-0000-000000000000 16 | AUTH_LABORATORY_CLIENT_ID=00000000-0000-0000-0000-000000000000 17 | AUTH_CLI_CLIENT_ID=00000000-0000-0000-0000-000000000000 18 | 19 | # Optional space-separated list of application clientIds that have full access to the laboratory as a fallback option for deployments that can't obtain global AAD admin consent 20 | AUTH_ALLOWED_APPLICATION_CLIENT_IDS=00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 00000000-0000-0000-0000-000000000000 21 | -------------------------------------------------------------------------------- /packages/laboratory/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../.eslintrc.json" 3 | } 4 | -------------------------------------------------------------------------------- /packages/laboratory/.nycrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../.nycrc" 3 | } 4 | -------------------------------------------------------------------------------- /packages/laboratory/.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | ...require('../../.prettierrc.js'), 3 | } 4 | -------------------------------------------------------------------------------- /packages/laboratory/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@microsoft/sds-laboratory", 3 | "version": "0.1.0", 4 | "repository": { 5 | "url": "https://github.com/microsoft/secure-data-sandbox" 6 | }, 7 | "description": "A toolkit for conducting machine learning trials against confidential data", 8 | "license": "MIT", 9 | "main": "dist/index", 10 | "typings": "dist/index", 11 | "files": [ 12 | "dist" 13 | ], 14 | "engines": { 15 | "node": ">=12" 16 | }, 17 | "bin": { 18 | "sds-laboratory": "./dist/main.js" 19 | }, 20 | "scripts": { 21 | "check": "gts check", 22 | "clean": "gts clean", 23 | "compile": "tsc -p tsconfig.build.json", 24 | "fix": "gts fix", 25 | "prepare": "npm run compile", 26 | "posttest": "npm run check", 27 | "start": "ts-node src/main.ts", 28 | "test": "nyc ts-mocha" 29 | }, 30 | "devDependencies": { 31 | "@types/bluebird": "^3.5.32", 32 | "@types/chai": "^4.2.12", 33 | "@types/chai-as-promised": "^7.1.3", 34 | "@types/chai-http": "^4.2.0", 35 | "@types/express": "^4.17.8", 36 | "@types/js-yaml": "^3.12.5", 37 | "@types/mocha": "^8.0.3", 38 | "@types/mustache": "^4.0.1", 39 | "@types/node": "^13.13.21", 40 | "@types/passport-azure-ad": "^4.0.7", 41 | "@types/tedious": "^4.0.1", 42 | "@types/uuid": "^8.3.0", 43 | "@types/validator": "^13.1.0", 44 | "chai": "^4.2.0", 45 | "chai-as-promised": "^7.1.1", 46 | "chai-exclude": "^2.0.2", 47 | "chai-http": "^4.3.0", 48 | "eslint": "^7.10.0", 49 | "gts": "^2.0.2", 50 | "js-yaml": "^3.14.0", 51 | "mocha": "^8.1.3", 52 | "nyc": "^15.1.0", 53 | "source-map-support": "^0.5.19", 54 | "ts-mocha": "^7.0.0", 55 | "typescript": "^4.0.3" 56 | }, 57 | "dependencies": { 58 | "@azure/identity": "^1.2.0", 59 | "@microsoft/sds": "*", 60 | "applicationinsights": "^1.8.7", 61 | "env-var": "^6.3.0", 62 | "express": "^4.17.1", 63 | "express-async-errors": "^3.1.1", 64 | "mustache": "^4.0.1", 65 | "passport": "^0.4.1", 66 | "passport-azure-ad": "^4.3.0", 67 | "reflect-metadata": "^0.1.13", 68 | "sequelize": "^5.22.3", 69 | "sequelize-typescript": "^1.1.0", 70 | "sqlite3": "^5.0.0", 71 | "strong-error-handler": "^3.5.0", 72 | "tedious": "^9.2.1", 73 | "uuid": "^8.3.0" 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /packages/laboratory/src/app.ts: -------------------------------------------------------------------------------- 1 | require('express-async-errors'); 2 | import * as express from 'express'; 3 | import * as errorhandler from 'strong-error-handler'; 4 | import * as passport from 'passport'; 5 | import { BearerStrategy, IBearerStrategyOption } from 'passport-azure-ad'; 6 | 7 | import { IClientConnectionInfo, ILaboratory } from '@microsoft/sds'; 8 | import { setErrorStatus } from './errors'; 9 | import { 10 | createBenchmarkRouter, 11 | createCandidateRouter, 12 | createRunRouter, 13 | createSuiteRouter, 14 | } from './routes'; 15 | import { requireRole, Role } from './auth'; 16 | 17 | import { 18 | AuthConfiguration, 19 | AADConfiguration, 20 | AuthMode, 21 | NoAuthConfiguration, 22 | } from './configuration'; 23 | 24 | function configureAADAuth(app: express.Express, config: AADConfiguration) { 25 | passport.use( 26 | new BearerStrategy( 27 | { 28 | identityMetadata: `https://login.microsoftonline.com/${config.tenantId}/v2.0/.well-known/openid-configuration`, 29 | clientID: config.laboratoryClientId, 30 | ignoreExpiration: config.ignoreExpiration, 31 | } as IBearerStrategyOption, 32 | (token, done) => { 33 | return done(null, token); 34 | } 35 | ) 36 | ); 37 | 38 | // unauthenticated endpoint for clients to retrieve connection info 39 | app 40 | .get('/connect', (req, res) => { 41 | const connectionInfo: IClientConnectionInfo = { 42 | type: 'aad', 43 | clientId: config.cliClientId, 44 | authority: `https://login.microsoftonline.com/${config.tenantId}`, 45 | scopes: config.scopes, 46 | }; 47 | res.json(connectionInfo); 48 | }) 49 | 50 | // require all endpoints to be authenticated with User role 51 | .use( 52 | passport.initialize(), 53 | passport.authenticate('oauth-bearer', { session: false }), 54 | requireRole(Role.User, config) 55 | ); 56 | } 57 | 58 | export async function createApp( 59 | lab: ILaboratory, 60 | auth: AuthConfiguration = NoAuthConfiguration 61 | ): Promise { 62 | const app = express().use(express.json()); 63 | 64 | // configure authorization 65 | switch (auth.mode) { 66 | case AuthMode.AAD: 67 | configureAADAuth(app, auth as AADConfiguration); 68 | break; 69 | case AuthMode.None: 70 | default: 71 | app.get('/connect', (req, res) => { 72 | const connectionInfo: IClientConnectionInfo = { 73 | type: 'unauthenticated', 74 | }; 75 | res.json(connectionInfo); 76 | }); 77 | break; 78 | } 79 | 80 | // Set up application routes 81 | app 82 | .get('/connect/validate', (req, res) => { 83 | res.status(200).end(); 84 | }) 85 | .use(createBenchmarkRouter(lab, auth)) 86 | .use(createCandidateRouter(lab)) 87 | .use(createRunRouter(lab, auth)) 88 | .use(createSuiteRouter(lab, auth)) 89 | 90 | // Handle known errors 91 | .use(setErrorStatus) 92 | 93 | // Hide details in error messages 94 | .use( 95 | errorhandler({ 96 | debug: process.env.NODE_ENV === 'development', 97 | log: process.env.NODE_ENV !== 'test', 98 | negotiateContentType: false, 99 | }) 100 | ); 101 | 102 | return app; 103 | } 104 | -------------------------------------------------------------------------------- /packages/laboratory/src/auth.ts: -------------------------------------------------------------------------------- 1 | import { ForbiddenError } from '@microsoft/sds'; 2 | import { Request, Response, NextFunction, RequestHandler } from 'express'; 3 | import { ITokenPayload } from 'passport-azure-ad'; 4 | 5 | import { AuthConfiguration, AuthMode, AADConfiguration } from './configuration'; 6 | 7 | export enum Role { 8 | User = 'user', 9 | Admin = 'admin', 10 | Benchmark = 'benchmark', 11 | } 12 | 13 | export function requireRole( 14 | role: Role, 15 | config: AuthConfiguration 16 | ): RequestHandler { 17 | return (req: Request, res: Response, next: NextFunction) => { 18 | // Allow request if auth is not enabled 19 | if (config.mode === AuthMode.None) { 20 | next(); 21 | } else if (config.mode === AuthMode.AAD) { 22 | const user = req.user as ITokenPayload; 23 | const aadConfig = config as AADConfiguration; 24 | 25 | let authorized = false; 26 | 27 | // Allow request if the application is on the explicit application allow list 28 | if ( 29 | user.azp && 30 | aadConfig.allowedApplicationClientIds.includes(user.azp) 31 | ) { 32 | authorized = true; 33 | } 34 | 35 | if ( 36 | user.roles?.includes(Role.Admin) || 37 | user.roles?.includes(Role.Benchmark) || 38 | user.roles?.includes(role) 39 | ) { 40 | authorized = true; 41 | } 42 | 43 | if (!authorized) { 44 | throw new ForbiddenError( 45 | `Caller does not have the required role: ${role}` 46 | ); 47 | } 48 | 49 | next(); 50 | } else { 51 | throw new Error(`Auth mode ${config.mode} not implemented`); 52 | } 53 | }; 54 | } 55 | -------------------------------------------------------------------------------- /packages/laboratory/src/configuration.ts: -------------------------------------------------------------------------------- 1 | import * as os from 'os'; 2 | import { 3 | QueueConfiguration, 4 | AzureCredential, 5 | ParseQueueConfiguration, 6 | } from '@microsoft/sds'; 7 | import { 8 | AzureSqlDatabaseConfiguration, 9 | DatabaseConfiguration, 10 | DatabaseMode, 11 | } from './database'; 12 | 13 | import * as env from 'env-var'; 14 | 15 | export enum AuthMode { 16 | AAD = 'AAD', 17 | None = 'none', 18 | } 19 | 20 | export interface AuthConfiguration { 21 | mode: AuthMode; 22 | } 23 | 24 | export interface AADConfiguration extends AuthConfiguration { 25 | mode: AuthMode.AAD; 26 | tenantId: string; 27 | laboratoryClientId: string; 28 | cliClientId: string; 29 | scopes: string[]; 30 | allowedApplicationClientIds: string[]; 31 | ignoreExpiration: boolean; 32 | } 33 | 34 | export const NoAuthConfiguration = { 35 | mode: AuthMode.None, 36 | }; 37 | 38 | export interface LaboratoryConfiguration { 39 | endpointBaseUrl: string; 40 | port: number; 41 | queue: QueueConfiguration; 42 | database: DatabaseConfiguration; 43 | auth: AuthConfiguration; 44 | } 45 | 46 | /** 47 | * Retrieve a DatabaseConfiguration from the current execution environment. 48 | */ 49 | export async function ParseDatabaseConfiguration(): Promise< 50 | DatabaseConfiguration 51 | > { 52 | const mode = env 53 | .get('SQL_MODE') 54 | .default(DatabaseMode.InMemory) 55 | .asEnum(Object.values(DatabaseMode)) as DatabaseMode; 56 | 57 | const host = env 58 | .get('SQL_HOST') 59 | .required(mode !== DatabaseMode.InMemory) 60 | .asString(); 61 | 62 | switch (mode) { 63 | case DatabaseMode.AzureSql: 64 | return { 65 | mode, 66 | host, 67 | database: env.get('SQL_DB').required().asString(), 68 | credential: await AzureCredential.getInstance(), 69 | } as AzureSqlDatabaseConfiguration; 70 | case DatabaseMode.InMemory: 71 | return { 72 | mode, 73 | host: 'localhost', 74 | }; 75 | } 76 | } 77 | 78 | function ParseAuthConfiguration(): AuthConfiguration { 79 | const authMode = env.get('AUTH_MODE').asString(); 80 | if (authMode === 'aad') { 81 | const tenantId = env.get('AUTH_TENANT_ID').required().asString(); 82 | const laboratoryClientId = env 83 | .get('AUTH_LABORATORY_CLIENT_ID') 84 | .required() 85 | .asString(); 86 | const cliClientId = env.get('AUTH_CLI_CLIENT_ID').required().asString(); 87 | const scopes = env 88 | .get('AUTH_SCOPES') 89 | .default('.default') 90 | .asArray(' ') 91 | .map(s => `api://${laboratoryClientId}/${s}`); 92 | const allowedApplicationClientIds = env 93 | .get('AUTH_ALLOWED_APPLICATION_CLIENT_IDS') 94 | .default('') 95 | .asArray(' '); 96 | 97 | // offline_access is required to use refresh tokens 98 | scopes.push('offline_access'); 99 | 100 | const config: AADConfiguration = { 101 | mode: AuthMode.AAD, 102 | tenantId, 103 | laboratoryClientId, 104 | cliClientId, 105 | scopes, 106 | allowedApplicationClientIds, 107 | ignoreExpiration: false, 108 | }; 109 | return config; 110 | } else { 111 | return NoAuthConfiguration; 112 | } 113 | } 114 | 115 | export async function ParseLaboratoryConfiguration(): Promise< 116 | LaboratoryConfiguration 117 | > { 118 | const port = env.get('WEBSITES_PORT').default(3000).asPortNumber(); 119 | 120 | let endpointBaseUrl = env.get('LABORATORY_ENDPOINT').asUrlString(); 121 | 122 | // if endpoint is not explicitly specified, check for WEBSITE_HOSTNAME and assume HTTPS over 443 123 | // this variable gets autowired by Azure App Service 124 | if (!endpointBaseUrl) { 125 | const hostname = env.get('WEBSITE_HOSTNAME').asString(); 126 | 127 | // if not found, fallback to machine hostname 128 | endpointBaseUrl = hostname 129 | ? `https://${hostname}` 130 | : `http://${os.hostname()}:${port}`; 131 | } 132 | 133 | return { 134 | endpointBaseUrl, 135 | port, 136 | queue: await ParseQueueConfiguration(), 137 | database: await ParseDatabaseConfiguration(), 138 | auth: ParseAuthConfiguration(), 139 | }; 140 | } 141 | -------------------------------------------------------------------------------- /packages/laboratory/src/database.ts: -------------------------------------------------------------------------------- 1 | import { SequelizeOptions } from 'sequelize-typescript'; 2 | import { ConnectionConfig } from 'tedious'; 3 | import { TokenCredential } from '@azure/identity'; 4 | 5 | export enum DatabaseMode { 6 | AzureSql = 'azuresql', 7 | InMemory = 'inmemory', 8 | } 9 | 10 | export interface DatabaseConfiguration { 11 | mode: DatabaseMode; 12 | host: string; 13 | } 14 | 15 | export interface AzureSqlDatabaseConfiguration extends DatabaseConfiguration { 16 | mode: DatabaseMode.AzureSql; 17 | database: string; 18 | credential: TokenCredential; 19 | } 20 | 21 | export function GetSequelizeOptions( 22 | config: DatabaseConfiguration 23 | ): SequelizeOptions { 24 | switch (config.mode) { 25 | case DatabaseMode.AzureSql: 26 | // eslint-disable-next-line no-case-declarations 27 | const azureConfig = config as AzureSqlDatabaseConfiguration; 28 | return { 29 | database: azureConfig.database, 30 | dialect: 'mssql', 31 | dialectOptions: { 32 | options: { 33 | encrypt: true, 34 | }, 35 | }, 36 | hooks: { 37 | beforeConnect: async config => { 38 | const tediousConfig = config.dialectOptions as ConnectionConfig; 39 | const accessToken = await azureConfig.credential.getToken( 40 | 'https://database.windows.net//.default' 41 | ); 42 | Object.assign(tediousConfig, { 43 | authentication: { 44 | type: 'azure-active-directory-access-token', 45 | options: { 46 | token: accessToken?.token, 47 | }, 48 | }, 49 | }); 50 | }, 51 | }, 52 | host: azureConfig.host, 53 | retry: { 54 | max: 3, 55 | }, 56 | }; 57 | case DatabaseMode.InMemory: 58 | return { 59 | dialect: 'sqlite', 60 | storage: ':memory:', 61 | logging: false, 62 | }; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /packages/laboratory/src/errors.ts: -------------------------------------------------------------------------------- 1 | import * as express from 'express'; 2 | 3 | import { 4 | EntityNotFoundError, 5 | ForbiddenError, 6 | IllegalOperationError, 7 | ValidationError, 8 | } from '@microsoft/sds'; 9 | 10 | export function setErrorStatus( 11 | err: Error, 12 | req: express.Request, 13 | res: express.Response, 14 | next: express.NextFunction 15 | ) { 16 | if (err instanceof EntityNotFoundError) { 17 | res.statusCode = 404; 18 | } else if ( 19 | err instanceof IllegalOperationError || 20 | err instanceof ValidationError 21 | ) { 22 | res.statusCode = 400; 23 | } else if (err instanceof ForbiddenError) { 24 | res.statusCode = 403; 25 | } 26 | next(err); 27 | } 28 | -------------------------------------------------------------------------------- /packages/laboratory/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './app'; 2 | -------------------------------------------------------------------------------- /packages/laboratory/src/main.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import { PipelineRun, GetQueue, InitTelemetry } from '@microsoft/sds'; 3 | import { ParseLaboratoryConfiguration } from './configuration'; 4 | import { GetSequelizeOptions } from './database'; 5 | import { 6 | initializeSequelize, 7 | SequelizeLaboratory, 8 | } from './sequelize_laboratory'; 9 | import { defaultClient as telemetryClient } from 'applicationinsights'; 10 | import { Events } from './telemetry'; 11 | 12 | import { createApp } from './app'; 13 | 14 | async function main() { 15 | InitTelemetry(); 16 | 17 | const config = await ParseLaboratoryConfiguration(); 18 | const queue = GetQueue(config.queue); 19 | 20 | // initializeSequelize binds Sequelize to the models, effectively becoming a singleton / service locator 21 | const sequelizeOptions = GetSequelizeOptions(config.database); 22 | await initializeSequelize(sequelizeOptions); 23 | 24 | const lab = new SequelizeLaboratory(config.endpointBaseUrl, queue); 25 | 26 | const app = await createApp(lab, config.auth); 27 | app.listen(config.port, () => { 28 | console.log('Starting SDS laboratory service.'); 29 | console.log(`Service url is ${config.endpointBaseUrl}.`); 30 | console.info(`Laboratory service listening on port ${config.port}.`); 31 | 32 | telemetryClient.trackEvent({ 33 | name: Events.LaboratoryStarted, 34 | }); 35 | }); 36 | } 37 | 38 | main().then(); 39 | -------------------------------------------------------------------------------- /packages/laboratory/src/routes/benchmarks.ts: -------------------------------------------------------------------------------- 1 | import { Router } from 'express'; 2 | 3 | import { BenchmarkType, ILaboratory, validate } from '@microsoft/sds'; 4 | import { requireRole, Role } from '../auth'; 5 | import { AuthConfiguration } from '../configuration'; 6 | 7 | export function createBenchmarkRouter( 8 | lab: ILaboratory, 9 | config: AuthConfiguration 10 | ): Router { 11 | const router = Router(); 12 | 13 | router.get('/benchmarks', async (req, res) => { 14 | res.json(await lab.allBenchmarks()); 15 | }); 16 | 17 | router 18 | .route('/benchmarks/:name') 19 | .get(async (req, res) => { 20 | res.json(await lab.oneBenchmark(req.params['name'])); 21 | }) 22 | .put(requireRole(Role.Admin, config), async (req, res) => { 23 | const benchmark = validate(BenchmarkType, req.body); 24 | await lab.upsertBenchmark(benchmark); 25 | res.json(await lab.oneBenchmark(benchmark.name)); 26 | }); 27 | 28 | return router; 29 | } 30 | -------------------------------------------------------------------------------- /packages/laboratory/src/routes/candidates.ts: -------------------------------------------------------------------------------- 1 | import { Router } from 'express'; 2 | 3 | import { CandidateType, ILaboratory, validate } from '@microsoft/sds'; 4 | 5 | export function createCandidateRouter(lab: ILaboratory): Router { 6 | const router = Router(); 7 | 8 | router.get('/candidates', async (req, res) => { 9 | res.json(await lab.allCandidates()); 10 | }); 11 | 12 | router 13 | .route('/candidates/:name') 14 | .get(async (req, res) => { 15 | res.json(await lab.oneCandidate(req.params['name'])); 16 | }) 17 | .put(async (req, res) => { 18 | const candidate = validate(CandidateType, req.body); 19 | await lab.upsertCandidate(candidate); 20 | res.json(await lab.oneCandidate(candidate.name)); 21 | }); 22 | 23 | return router; 24 | } 25 | -------------------------------------------------------------------------------- /packages/laboratory/src/routes/index.ts: -------------------------------------------------------------------------------- 1 | export * from './benchmarks'; 2 | export * from './candidates'; 3 | export * from './runs'; 4 | export * from './suites'; 5 | -------------------------------------------------------------------------------- /packages/laboratory/src/routes/runs.ts: -------------------------------------------------------------------------------- 1 | import { Router } from 'express'; 2 | import { Contracts } from 'applicationinsights'; 3 | import { defaultClient as telemetryClient } from 'applicationinsights'; 4 | import { ITokenPayload } from 'passport-azure-ad'; 5 | 6 | import { 7 | ILaboratory, 8 | ReportRunResultsType, 9 | RunRequestType, 10 | UpdateRunStatusType, 11 | validate, 12 | ValidationError, 13 | } from '@microsoft/sds'; 14 | import { requireRole, Role } from '../auth'; 15 | import { AuthConfiguration } from '../configuration'; 16 | 17 | export function createRunRouter( 18 | lab: ILaboratory, 19 | authConfig: AuthConfiguration 20 | ): Router { 21 | const router = Router(); 22 | 23 | router 24 | .route('/runs') 25 | .get(async (req, res) => { 26 | res.json(await lab.allRuns()); 27 | }) 28 | .post(async (req, res) => { 29 | const runRequest = validate(RunRequestType, req.body); 30 | const run = await lab.createRunRequest(runRequest); 31 | 32 | // log who initiates the run in appinsight 33 | const user = req.user as ITokenPayload; 34 | telemetryClient.trackTrace({ 35 | message: `user sub:'${ 36 | user ? user.sub : undefined 37 | }' initiated the run '${run.name}' using candidate '${ 38 | req.body.candidate 39 | }' and suite '${req.body.suite}'`, 40 | severity: Contracts.SeverityLevel.Information, 41 | }); 42 | res.status(202); 43 | res.json(run); 44 | }); 45 | 46 | router.get('/runs/results', async (req, res) => { 47 | if ( 48 | typeof req.query.benchmark !== 'string' || 49 | typeof req.query.suite !== 'string' 50 | ) { 51 | throw new ValidationError( 52 | 'Query parameters for `benchmark` and `suite` must be provided' 53 | ); 54 | } 55 | 56 | res.json( 57 | await lab.allRunResults(req.query['benchmark'], req.query['suite']) 58 | ); 59 | }); 60 | 61 | router 62 | .route('/runs/:name') 63 | .get(async (req, res) => { 64 | res.json(await lab.oneRun(req.params['name'])); 65 | }) 66 | .patch(requireRole(Role.Benchmark, authConfig), async (req, res) => { 67 | const { status } = validate(UpdateRunStatusType, req.body); 68 | await lab.updateRunStatus(req.params['name'], status); 69 | 70 | // log run status into appInsights 71 | telemetryClient.trackTrace({ 72 | message: `Run: the run status of '${req.params['name']}' is '${status}'`, 73 | severity: Contracts.SeverityLevel.Information, 74 | }); 75 | res.status(204); 76 | res.end(); 77 | }); 78 | 79 | router.post( 80 | '/runs/:name/results', 81 | requireRole(Role.Benchmark, authConfig), 82 | async (req, res) => { 83 | const { measures } = validate(ReportRunResultsType, req.body); 84 | await lab.reportRunResults(req.params['name'], measures); 85 | res.status(204); 86 | res.end(); 87 | } 88 | ); 89 | 90 | return router; 91 | } 92 | -------------------------------------------------------------------------------- /packages/laboratory/src/routes/suites.ts: -------------------------------------------------------------------------------- 1 | import { Router } from 'express'; 2 | 3 | import { ILaboratory, SuiteType, validate } from '@microsoft/sds'; 4 | import { requireRole, Role } from '../auth'; 5 | import { AuthConfiguration } from '../configuration'; 6 | 7 | export function createSuiteRouter( 8 | lab: ILaboratory, 9 | authConfig: AuthConfiguration 10 | ): Router { 11 | const router = Router(); 12 | 13 | router.get('/suites', async (req, res) => { 14 | res.json(await lab.allSuites()); 15 | }); 16 | 17 | router 18 | .route('/suites/:name') 19 | .get(async (req, res) => { 20 | res.json(await lab.oneSuite(req.params['name'])); 21 | }) 22 | .put(requireRole(Role.Admin, authConfig), async (req, res) => { 23 | const suite = validate(SuiteType, req.body); 24 | await lab.upsertSuite(suite); 25 | res.json(await lab.oneSuite(suite.name)); 26 | }); 27 | 28 | return router; 29 | } 30 | -------------------------------------------------------------------------------- /packages/laboratory/src/sequelize_laboratory/benchmark.ts: -------------------------------------------------------------------------------- 1 | import { Benchmark } from './models'; 2 | import { normalizeName, IBenchmark } from '@microsoft/sds'; 3 | 4 | export function normalizeBenchmark(benchmark: IBenchmark): IBenchmark { 5 | return { 6 | ...benchmark, 7 | name: normalizeName(benchmark.name), 8 | }; 9 | } 10 | 11 | export async function processBenchmark(benchmark: IBenchmark) { 12 | // Upsert Benchmark 13 | await Benchmark.upsert(benchmark); 14 | } 15 | -------------------------------------------------------------------------------- /packages/laboratory/src/sequelize_laboratory/candidate.ts: -------------------------------------------------------------------------------- 1 | import { Benchmark, Candidate } from './models'; 2 | import { 3 | normalizeName, 4 | ICandidate, 5 | IllegalOperationError, 6 | } from '@microsoft/sds'; 7 | 8 | export function normalizeCandidate(candidate: ICandidate): ICandidate { 9 | return { 10 | ...candidate, 11 | name: normalizeName(candidate.name), 12 | benchmark: normalizeName(candidate.benchmark), 13 | }; 14 | } 15 | 16 | export async function processCandidate(candidate: ICandidate): Promise { 17 | // Verify that referenced benchmark exists. 18 | const benchmark = await Benchmark.findOne({ 19 | where: { name: candidate.benchmark }, 20 | }); 21 | if (!benchmark) { 22 | const message = `Candidate references unknown benchmark ${candidate.benchmark}`; 23 | throw new IllegalOperationError(message); 24 | } 25 | 26 | await Candidate.upsert(candidate); 27 | } 28 | -------------------------------------------------------------------------------- /packages/laboratory/src/sequelize_laboratory/index.ts: -------------------------------------------------------------------------------- 1 | export * from './laboratory'; 2 | export * from './sequelize'; 3 | // models are private to sequelize_laboratory 4 | -------------------------------------------------------------------------------- /packages/laboratory/src/sequelize_laboratory/laboratory.ts: -------------------------------------------------------------------------------- 1 | import { 2 | EntityNotFoundError, 3 | IBenchmark, 4 | ICandidate, 5 | ILaboratory, 6 | IllegalOperationError, 7 | IResult, 8 | IRun, 9 | IRunRequest, 10 | ISuite, 11 | Measures, 12 | RunStatus, 13 | PipelineRun, 14 | normalizeName, 15 | normalizeRunName, 16 | IQueue, 17 | } from '@microsoft/sds'; 18 | 19 | import { normalizeBenchmark, processBenchmark } from './benchmark'; 20 | import { normalizeCandidate, processCandidate } from './candidate'; 21 | import { Benchmark, Candidate, Run, Suite, Result } from './models'; 22 | 23 | import { 24 | normalizeRunRequest, 25 | processRunResults, 26 | processRunRequest, 27 | processRunStatus, 28 | } from './run'; 29 | 30 | import { normalizeSuite, processSuite } from './suite'; 31 | 32 | export class SequelizeLaboratory implements ILaboratory { 33 | private readonly server: string; 34 | private readonly queue: IQueue; 35 | 36 | constructor(server: string, queue: IQueue) { 37 | this.queue = queue; 38 | this.server = server; 39 | } 40 | 41 | ///////////////////////////////////////////////////////////////////////////// 42 | // 43 | // Benchmarks 44 | // 45 | ///////////////////////////////////////////////////////////////////////////// 46 | allBenchmarks(): Promise { 47 | return Benchmark.findAll(); 48 | } 49 | 50 | async oneBenchmark(rawName: string): Promise { 51 | const name = normalizeName(rawName); 52 | const b = await Benchmark.findOne({ where: { name } }); 53 | 54 | if (b === null) { 55 | const message = `Benchmark "${name}" not found.`; 56 | throw new EntityNotFoundError(message); 57 | } 58 | 59 | return b; 60 | } 61 | 62 | async upsertBenchmark(b: IBenchmark, rawName?: string): Promise { 63 | const benchmark = normalizeBenchmark(b); 64 | 65 | // If optional rawName parameter is supplied, verify that its normalized 66 | // form is the same as the benchmark's normalized name. 67 | if (rawName !== undefined) { 68 | const name = normalizeName(rawName); 69 | if (name !== benchmark.name) { 70 | const message = `Benchmark name mismatch: "${benchmark.name}" != "${name}"`; 71 | throw new IllegalOperationError(message); 72 | } 73 | } 74 | 75 | await processBenchmark(benchmark); 76 | } 77 | 78 | ///////////////////////////////////////////////////////////////////////////// 79 | // 80 | // Candidates 81 | // 82 | ///////////////////////////////////////////////////////////////////////////// 83 | allCandidates(): Promise { 84 | return Candidate.findAll(); 85 | } 86 | 87 | async oneCandidate(rawName: string): Promise { 88 | const name = normalizeName(rawName); 89 | const candidate = await Candidate.findOne({ where: { name } }); 90 | 91 | if (candidate === null) { 92 | const message = `Candidate "${name}" not found.`; 93 | throw new EntityNotFoundError(message); 94 | } 95 | 96 | return candidate; 97 | } 98 | 99 | async upsertCandidate(c: ICandidate, rawName?: string): Promise { 100 | const candidate = normalizeCandidate(c); 101 | 102 | // If optional rawName parameter is supplied, verify that its normalized 103 | // form is the same as the candidate's normalized name. 104 | if (rawName !== undefined) { 105 | const name = normalizeName(rawName); 106 | if (name !== candidate.name) { 107 | const message = `Candidate name mismatch: "${candidate.name}" != "${name}"`; 108 | throw new IllegalOperationError(message); 109 | } 110 | } 111 | 112 | await processCandidate(candidate); 113 | } 114 | 115 | ///////////////////////////////////////////////////////////////////////////// 116 | // 117 | // Suites 118 | // 119 | ///////////////////////////////////////////////////////////////////////////// 120 | allSuites(): Promise { 121 | return Suite.findAll(); 122 | } 123 | 124 | async oneSuite(rawName: string): Promise { 125 | const name = normalizeName(rawName); 126 | const suite = await Suite.findOne({ where: { name } }); 127 | 128 | if (suite === null) { 129 | const message = `Suite "${name}" not found.`; 130 | throw new EntityNotFoundError(message); 131 | } 132 | 133 | return suite; 134 | } 135 | 136 | async upsertSuite(s: ISuite, rawName?: string): Promise { 137 | const suite = normalizeSuite(s); 138 | 139 | // If optional rawName parameter is supplied, verify that its normalized 140 | // form is the same as the candidate's normalized name. 141 | if (rawName !== undefined) { 142 | const name = normalizeName(rawName); 143 | if (name !== suite.name) { 144 | const message = `Suite name mismatch: "${suite.name}" != "${name}"`; 145 | throw new IllegalOperationError(message); 146 | } 147 | } 148 | 149 | await processSuite(suite); 150 | } 151 | 152 | ///////////////////////////////////////////////////////////////////////////// 153 | // 154 | // Runs and RunRequests 155 | // 156 | ///////////////////////////////////////////////////////////////////////////// 157 | allRuns(): Promise { 158 | return Run.findAll(); 159 | } 160 | 161 | async oneRun(rawName: string): Promise { 162 | const name = normalizeRunName(rawName); 163 | const run = await Run.findOne({ where: { name } }); 164 | 165 | if (run === null) { 166 | const message = `Run "${name}" not found.`; 167 | throw new EntityNotFoundError(message); 168 | } 169 | 170 | return run; 171 | } 172 | 173 | async createRunRequest(r: IRunRequest): Promise { 174 | const runRequest = normalizeRunRequest(r); 175 | return processRunRequest(this.server, runRequest, this.queue); 176 | } 177 | 178 | async updateRunStatus(rawName: string, status: RunStatus): Promise { 179 | const name = normalizeRunName(rawName); 180 | await processRunStatus(name, status); 181 | } 182 | 183 | ///////////////////////////////////////////////////////////////////////////// 184 | // 185 | // Results 186 | // 187 | ///////////////////////////////////////////////////////////////////////////// 188 | async reportRunResults(rawName: string, measures: Measures): Promise { 189 | const name = normalizeRunName(rawName); 190 | await processRunResults(name, measures); 191 | } 192 | 193 | async allRunResults(benchmark: string, suite: string): Promise { 194 | return Result.findAll({ where: { benchmark, suite } }); 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /packages/laboratory/src/sequelize_laboratory/models/benchmark.ts: -------------------------------------------------------------------------------- 1 | import { DataType, Column, Model, Table } from 'sequelize-typescript'; 2 | 3 | import { IBenchmark, PipelineStage } from '@microsoft/sds'; 4 | 5 | import { jsonColumn } from './decorators'; 6 | 7 | @Table 8 | export class Benchmark extends Model implements IBenchmark { 9 | @Column({ 10 | type: DataType.STRING, 11 | primaryKey: true, 12 | }) 13 | name!: string; 14 | 15 | @Column(DataType.STRING) 16 | author!: string; 17 | 18 | @Column(DataType.STRING) 19 | apiVersion!: string; 20 | 21 | @Column(jsonColumn('stages')) 22 | stages!: PipelineStage[]; 23 | } 24 | -------------------------------------------------------------------------------- /packages/laboratory/src/sequelize_laboratory/models/candidate.ts: -------------------------------------------------------------------------------- 1 | import { DataType, Column, Model, Table } from 'sequelize-typescript'; 2 | import { jsonColumn } from './decorators'; 3 | 4 | import { ICandidate } from '@microsoft/sds'; 5 | 6 | @Table 7 | export class Candidate extends Model implements ICandidate { 8 | @Column({ 9 | type: DataType.STRING, 10 | primaryKey: true, 11 | }) 12 | name!: string; 13 | 14 | @Column(DataType.STRING) 15 | author!: string; 16 | 17 | @Column(DataType.STRING) 18 | apiVersion!: string; 19 | 20 | @Column(DataType.STRING) 21 | benchmark!: string; 22 | 23 | @Column(DataType.STRING) 24 | image!: string; 25 | 26 | @Column(jsonColumn('cmd')) 27 | cmd!: string[]; 28 | 29 | @Column(jsonColumn<{ [x: string]: string }>('env')) 30 | env!: { [x: string]: string }; 31 | } 32 | -------------------------------------------------------------------------------- /packages/laboratory/src/sequelize_laboratory/models/decorators.ts: -------------------------------------------------------------------------------- 1 | import { DataType } from 'sequelize-typescript'; 2 | // 3 | // Helper function provides a column decorator for JSON string columns 4 | // that are represented as POJOs of type T. 5 | // 6 | export function jsonColumn(name: string) { 7 | return { 8 | // DataType.TEXT translates to VARCHAR(MAX) for MSSQL 9 | // https://github.com/sequelize/sequelize/blob/042cd693635ffba83ff7a2079974692af6f710a7/src/dialects/mssql/data-types.js#L91 10 | type: DataType.TEXT, 11 | get(): T | undefined { 12 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 13 | const value = (this as any).getDataValue(name) as string; 14 | // TODO: validate schema here. 15 | // Will likely need to pass in a schema parameter or 16 | // use some sort of global registry of schemas for types. 17 | 18 | // DESIGN NOTE: sequelize will attempt to get all columns, whether their 19 | // values are undefined or not. (e.g. in the case of an Update). Need to 20 | // handle undefined here. 21 | if (value) { 22 | return JSON.parse(value) as T; 23 | } else { 24 | return undefined; 25 | } 26 | }, 27 | set(value: T) { 28 | const text = JSON.stringify(value); 29 | 30 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 31 | (this as any).setDataValue(name, text); 32 | }, 33 | }; 34 | } 35 | -------------------------------------------------------------------------------- /packages/laboratory/src/sequelize_laboratory/models/index.ts: -------------------------------------------------------------------------------- 1 | export * from './benchmark'; 2 | export * from './candidate'; 3 | export * from './result'; 4 | export * from './run'; 5 | export * from './suite'; 6 | -------------------------------------------------------------------------------- /packages/laboratory/src/sequelize_laboratory/models/result.ts: -------------------------------------------------------------------------------- 1 | import { DataType, Column, Model, Table } from 'sequelize-typescript'; 2 | 3 | import { IResult, Measures } from '@microsoft/sds'; 4 | 5 | import { jsonColumn } from './decorators'; 6 | 7 | @Table 8 | export class Result extends Model implements IResult { 9 | @Column({ 10 | type: DataType.STRING, 11 | primaryKey: true, 12 | }) 13 | name!: string; 14 | 15 | @Column(DataType.STRING) 16 | author!: string; 17 | 18 | @Column(DataType.STRING) 19 | apiVersion!: string; 20 | 21 | @Column(DataType.STRING) 22 | benchmark!: string; 23 | 24 | @Column(DataType.STRING) 25 | suite!: string; 26 | 27 | @Column(DataType.STRING) 28 | candidate!: string; 29 | 30 | @Column(jsonColumn('measures')) 31 | measures!: Measures; 32 | } 33 | -------------------------------------------------------------------------------- /packages/laboratory/src/sequelize_laboratory/models/run.ts: -------------------------------------------------------------------------------- 1 | import { DataType, Column, Model, Table } from 'sequelize-typescript'; 2 | 3 | import { 4 | IBenchmark, 5 | ICandidate, 6 | IRun, 7 | ISuite, 8 | RunStatus, 9 | } from '@microsoft/sds'; 10 | 11 | import { jsonColumn } from './decorators'; 12 | 13 | @Table 14 | export class Run extends Model implements IRun { 15 | @Column({ 16 | type: DataType.STRING, 17 | primaryKey: true, 18 | }) 19 | name!: string; 20 | 21 | @Column(DataType.STRING) 22 | author!: string; 23 | 24 | @Column(DataType.STRING) 25 | apiVersion!: string; 26 | 27 | @Column(DataType.STRING) 28 | status!: RunStatus; 29 | 30 | @Column(jsonColumn('benchmark')) 31 | benchmark!: IBenchmark; 32 | 33 | @Column(jsonColumn('candidate')) 34 | candidate!: ICandidate; 35 | 36 | @Column(jsonColumn('suite')) 37 | suite!: ISuite; 38 | } 39 | -------------------------------------------------------------------------------- /packages/laboratory/src/sequelize_laboratory/models/suite.ts: -------------------------------------------------------------------------------- 1 | import { DataType, Column, Model, Table } from 'sequelize-typescript'; 2 | 3 | import { ISuite, SuiteVolume } from '@microsoft/sds'; 4 | 5 | import { jsonColumn } from './decorators'; 6 | 7 | @Table 8 | export class Suite extends Model implements ISuite { 9 | @Column({ 10 | type: DataType.STRING, 11 | primaryKey: true, 12 | }) 13 | name!: string; 14 | 15 | @Column(DataType.STRING) 16 | author!: string; 17 | 18 | @Column(DataType.STRING) 19 | apiVersion!: string; 20 | 21 | @Column(DataType.STRING) 22 | benchmark!: string; 23 | 24 | @Column(jsonColumn>('properties')) 25 | properties!: Record; 26 | 27 | @Column(jsonColumn('volumes')) 28 | volumes!: SuiteVolume[]; 29 | } 30 | -------------------------------------------------------------------------------- /packages/laboratory/src/sequelize_laboratory/run.ts: -------------------------------------------------------------------------------- 1 | import { v1 } from 'uuid'; 2 | import mustache = require('mustache'); 3 | 4 | // Don't HTML escape output 5 | mustache.escape = text => text; 6 | // Use single braces 7 | mustache.tags = ['{', '}']; 8 | 9 | import { 10 | EntityNotFoundError, 11 | IllegalOperationError, 12 | IResult, 13 | IRun, 14 | IRunRequest, 15 | Measures, 16 | RunStatus, 17 | BenchmarkStageKind, 18 | PipelineRun, 19 | PipelineRunStage, 20 | normalizeName, 21 | IQueue, 22 | PipelineRunStageVolume, 23 | } from '@microsoft/sds'; 24 | 25 | import { Benchmark, Candidate, Suite, Run, Result } from './models'; 26 | 27 | export function normalizeRunRequest(runRequest: IRunRequest): IRunRequest { 28 | return { 29 | ...runRequest, 30 | candidate: normalizeName(runRequest.candidate), 31 | suite: normalizeName(runRequest.suite), 32 | }; 33 | } 34 | 35 | export async function processRunRequest( 36 | server: string, 37 | runRequest: IRunRequest, 38 | queue: IQueue 39 | ): Promise { 40 | // Verify that referenced candidate exists. 41 | const candidate = await Candidate.findOne({ 42 | where: { name: runRequest.candidate }, 43 | }); 44 | if (!candidate) { 45 | const message = `Run request references unknown candidate ${runRequest.candidate}`; 46 | throw new IllegalOperationError(message); 47 | } 48 | 49 | // Verify that referenced suite exists. 50 | const suite = await Suite.findOne({ where: { name: runRequest.suite } }); 51 | if (!suite) { 52 | const message = `Run request references unknown suite ${runRequest.suite}`; 53 | throw new IllegalOperationError(message); 54 | } 55 | 56 | // Verify that candidate and suite reference same benchmark. 57 | if (candidate.benchmark !== suite.benchmark) { 58 | const message = `Candidate benchmark "${candidate.benchmark}" doesn't match suite benchmark "${suite.benchmark}"`; 59 | throw new IllegalOperationError(message); 60 | } 61 | 62 | // Verify that referenced benchmark exists. 63 | const benchmark = await Benchmark.findOne({ 64 | where: { name: candidate.benchmark }, 65 | }); 66 | if (!benchmark) { 67 | const message = `Candidate references unknown benchmark ${candidate.benchmark}`; 68 | throw new IllegalOperationError(message); 69 | } 70 | 71 | // 72 | // All ok. Create the run. 73 | // 74 | 75 | // TODO: consider moving name generation to normalize.ts. 76 | const name = v1(); 77 | 78 | const run: IRun = { 79 | name, 80 | author: 'unknown', // TODO: fill in name 81 | apiVersion: 'v1alpha1', 82 | status: RunStatus.CREATED, 83 | benchmark, 84 | candidate, 85 | suite, 86 | }; 87 | 88 | // Create the run record in the database. 89 | const result = await Run.create(run); 90 | 91 | // Queue the run request. 92 | const message = createMessage(server, name, result); 93 | await queue.enqueue(message); 94 | 95 | return result; 96 | } 97 | 98 | export async function processRunStatus( 99 | name: string, 100 | status: RunStatus 101 | ): Promise { 102 | // Verify that named run exists in the database. 103 | // DESIGN NOTE: this is a friendly, convenience check that warns the user 104 | // of a referential integrity problem at the time of the check. It makes 105 | // no attempt to guard against race conditions. 106 | // TODO: REVIEW: why not just rely on update() failing? 107 | const run = await Run.findOne({ 108 | where: { name }, 109 | }); 110 | if (!run) { 111 | const message = `Unknown run id ${name}`; 112 | throw new EntityNotFoundError(message); 113 | } 114 | 115 | // Update its status field 116 | await Run.update({ status }, { where: { name } }); 117 | } 118 | 119 | export async function processRunResults( 120 | name: string, 121 | measures: Measures 122 | ): Promise { 123 | // Find run in the database. 124 | // TODO: consider using transaction here to protect against race condition 125 | // where run is updated between the return of findOne() and the upsert(). 126 | const run = await Run.findOne({ 127 | where: { name }, 128 | }); 129 | if (!run) { 130 | const message = `Unknown run id ${name}`; 131 | throw new EntityNotFoundError(message); 132 | } 133 | 134 | // Upsert to Results table. 135 | // TODO: REVIEW: is it ok that the upsert can update all of the run-related 136 | // fields on subsequent calls to processRunResults? This could bring in new 137 | // values for these fields, if run were to change in the interim period. 138 | // Perhaps this should do a create(), instead. 139 | const results: IResult = { 140 | name: run.name, 141 | author: run.author, 142 | apiVersion: 'v1alpha1', 143 | benchmark: run.benchmark.name, 144 | suite: run.suite.name, 145 | candidate: run.candidate.name, 146 | measures, 147 | }; 148 | 149 | await Result.upsert(results); 150 | } 151 | 152 | function createMessage(server: string, name: string, run: IRun): PipelineRun { 153 | const benchmark = run.benchmark; 154 | const candidate = run.candidate; 155 | const suite = run.suite; 156 | 157 | const templateValues = { 158 | laboratoryEndpoint: server, 159 | run, 160 | benchmark, 161 | candidate, 162 | suite, 163 | }; 164 | 165 | const stages = benchmark.stages.map(stage => { 166 | const image = 167 | stage.kind === BenchmarkStageKind.CANDIDATE 168 | ? candidate.image 169 | : stage.image; 170 | 171 | const volumes = stage.volumes?.reduce( 172 | (result: PipelineRunStageVolume[], v) => { 173 | const sourceVolume = suite.volumes?.filter(sv => sv.name === v.name)[0]; 174 | if (sourceVolume) { 175 | result.push({ 176 | type: sourceVolume.type, 177 | target: v.path, 178 | source: sourceVolume.target, 179 | readonly: v.readonly, 180 | name: v.name, 181 | }); 182 | } 183 | 184 | return result; 185 | }, 186 | [] 187 | ); 188 | 189 | const runStage: PipelineRunStage = { 190 | name: stage.name, 191 | kind: stage.kind, 192 | image, 193 | volumes, 194 | }; 195 | 196 | // Add container arguments 197 | if (stage.cmd) { 198 | runStage.cmd = stage.cmd.map(s => mustache.render(s, templateValues)); 199 | } 200 | 201 | // Add environment variables specified by the benchmark 202 | if (stage.env) { 203 | runStage.env = stage.env; 204 | } 205 | 206 | // Add any self-configured environment vars that the candidate has specified 207 | if (stage.kind === BenchmarkStageKind.CANDIDATE && candidate.env) { 208 | runStage.env = { ...runStage.env, ...candidate.env }; 209 | } 210 | 211 | // Apply transforms 212 | for (const env in runStage.env) { 213 | runStage.env[env] = mustache.render(runStage.env[env], templateValues); 214 | } 215 | 216 | return runStage; 217 | }); 218 | 219 | const message: PipelineRun = { 220 | name, 221 | laboratoryEndpoint: server, 222 | stages, 223 | }; 224 | return message; 225 | } 226 | -------------------------------------------------------------------------------- /packages/laboratory/src/sequelize_laboratory/sequelize.ts: -------------------------------------------------------------------------------- 1 | import { Sequelize, SequelizeOptions } from 'sequelize-typescript'; 2 | 3 | import { Benchmark, Candidate, Result, Run, Suite } from './models'; 4 | 5 | export async function initializeSequelize( 6 | options: SequelizeOptions 7 | ): Promise { 8 | const sequelize = new Sequelize(options); 9 | sequelize.addModels([Benchmark, Candidate, Result, Run, Suite]); 10 | 11 | await sequelize.sync(); 12 | 13 | return sequelize; 14 | } 15 | -------------------------------------------------------------------------------- /packages/laboratory/src/sequelize_laboratory/suite.ts: -------------------------------------------------------------------------------- 1 | import { Benchmark, Suite } from './models'; 2 | import { normalizeName, IllegalOperationError, ISuite } from '@microsoft/sds'; 3 | 4 | export function normalizeSuite(suite: ISuite): ISuite { 5 | return { 6 | ...suite, 7 | name: normalizeName(suite.name), 8 | benchmark: normalizeName(suite.benchmark), 9 | }; 10 | } 11 | 12 | export async function processSuite(suite: ISuite): Promise { 13 | // Verify that referenced benchmark exists. 14 | const benchmark = await Benchmark.findOne({ 15 | where: { name: suite.benchmark }, 16 | }); 17 | if (!benchmark) { 18 | const message = `Suite references unknown benchmark ${suite.benchmark}`; 19 | throw new IllegalOperationError(message); 20 | } 21 | 22 | await Suite.upsert(suite); 23 | } 24 | -------------------------------------------------------------------------------- /packages/laboratory/src/shims.d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-empty-interface */ 2 | 3 | // superagent 4 | // https://github.com/DefinitelyTyped/DefinitelyTyped/issues/12044 5 | declare interface XMLHttpRequest {} 6 | -------------------------------------------------------------------------------- /packages/laboratory/src/telemetry.ts: -------------------------------------------------------------------------------- 1 | export enum Events { 2 | LaboratoryStarted = 'laboratoryStarted', 3 | } 4 | -------------------------------------------------------------------------------- /packages/laboratory/test/.mocharc.yml: -------------------------------------------------------------------------------- 1 | recursive: true 2 | require: 'source-map-support/register' 3 | spec: '**/*.ts' 4 | -------------------------------------------------------------------------------- /packages/laboratory/test/index.ts: -------------------------------------------------------------------------------- 1 | require('mocha'); 2 | import * as appInsights from 'applicationinsights'; 3 | 4 | before(() => { 5 | appInsights.setup('00000000-0000-0000-0000-000000000000'); 6 | appInsights.defaultClient.config.disableAppInsights = true; 7 | process.env.NODE_ENV = 'test'; 8 | }); 9 | -------------------------------------------------------------------------------- /packages/laboratory/test/samples.test.ts: -------------------------------------------------------------------------------- 1 | // Verify that the samples are valid against the laboratory 2 | import { 3 | IBenchmark, 4 | ICandidate, 5 | InMemoryQueue, 6 | IRunRequest, 7 | ISuite, 8 | PipelineRun, 9 | } from '@microsoft/sds'; 10 | import * as chai from 'chai'; 11 | import { assert } from 'chai'; 12 | import chaiHttp = require('chai-http'); 13 | chai.use(chaiHttp); 14 | 15 | import * as fs from 'fs'; 16 | import * as yaml from 'js-yaml'; 17 | 18 | import { createApp } from '../src'; 19 | import { initTestEnvironment } from './sequelize_laboratory/shared'; 20 | 21 | describe('laboratory/samples', () => { 22 | it('runs catdetection', async () => { 23 | const queue = new InMemoryQueue(); 24 | const lab = await initTestEnvironment(queue); 25 | const app = await createApp(lab); 26 | 27 | const benchmark = yaml.safeLoad( 28 | fs.readFileSync('../../samples/catdetection/benchmark.yml', 'utf8') 29 | ) as IBenchmark; 30 | let res = await chai 31 | .request(app) 32 | .put(`/benchmarks/${benchmark.name}`) 33 | .send(benchmark); 34 | assert.equal(res.status, 200); 35 | 36 | const suite = yaml.safeLoad( 37 | fs.readFileSync('../../samples/catdetection/suite.yml', 'utf8') 38 | ) as ISuite; 39 | res = await chai.request(app).put(`/suites/${suite.name}`).send(suite); 40 | assert.equal(res.status, 200); 41 | 42 | const candidate = yaml.safeLoad( 43 | fs.readFileSync('../../samples/catdetection/candidate.yml', 'utf8') 44 | ) as ICandidate; 45 | res = await chai 46 | .request(app) 47 | .put(`/candidates/${candidate.name}`) 48 | .send(candidate); 49 | assert.equal(res.status, 200); 50 | 51 | const runRequest: IRunRequest = { 52 | candidate: candidate.name, 53 | suite: suite.name, 54 | }; 55 | res = await chai.request(app).post('/runs').send(runRequest); 56 | assert.equal(res.status, 202); 57 | 58 | const runs = await queue.dequeue(1); 59 | const run = runs[0].value; 60 | 61 | const expected: PipelineRun = { 62 | laboratoryEndpoint: 'http://localhost:3000', 63 | name: res.body.name, 64 | stages: [ 65 | { 66 | name: 'prep', 67 | kind: 'container', 68 | image: 'acanthamoeba/sds-prep', 69 | env: { 70 | IMAGE_URL: 'https://placekitten.com/1024/768', 71 | BENCHMARK_AUTHOR: 'acanthamoeba', 72 | }, 73 | volumes: [ 74 | { 75 | name: 'images', 76 | type: 'ephemeral', 77 | source: undefined, 78 | target: '/out', 79 | readonly: false, 80 | }, 81 | ], 82 | }, 83 | { 84 | name: 'candidate', 85 | kind: 'candidate', 86 | image: 'acanthamoeba/sds-candidate', 87 | env: { 88 | API_KEY: '', 89 | SERVICE_ENDPOINT: 90 | 'https://.cognitiveservices.azure.com/', 91 | }, 92 | volumes: [ 93 | { 94 | name: 'images', 95 | type: 'ephemeral', 96 | source: undefined, 97 | target: '/in', 98 | readonly: true, 99 | }, 100 | { 101 | name: 'predictions', 102 | type: 'ephemeral', 103 | source: undefined, 104 | target: '/out', 105 | readonly: false, 106 | }, 107 | ], 108 | }, 109 | { 110 | name: 'eval', 111 | kind: 'container', 112 | image: 'acanthamoeba/sds-eval', 113 | env: { 114 | EXPECTED_ANIMAL: 'cat', 115 | CANDIDATE_IMAGE: 'acanthamoeba/sds-candidate', 116 | }, 117 | cmd: [run.name, 'http://localhost:3000'], 118 | volumes: [ 119 | { 120 | name: 'predictions', 121 | type: 'ephemeral', 122 | source: undefined, 123 | target: '/results', 124 | readonly: true, 125 | }, 126 | { 127 | name: 'scores', 128 | type: 'ephemeral', 129 | source: undefined, 130 | target: '/scores', 131 | readonly: false, 132 | }, 133 | ], 134 | }, 135 | ], 136 | }; 137 | 138 | assert.deepEqual(run, expected); 139 | }); 140 | }); 141 | -------------------------------------------------------------------------------- /packages/laboratory/test/sequelize_laboratory/benchmarks.test.ts: -------------------------------------------------------------------------------- 1 | import { ILaboratory } from '@microsoft/sds'; 2 | import * as chai from 'chai'; 3 | import { assert } from 'chai'; 4 | import chaiAsPromised = require('chai-as-promised'); 5 | import chaiExclude from 'chai-exclude'; 6 | 7 | import { 8 | benchmark1, 9 | benchmark2, 10 | benchmark3, 11 | } from '../../../sds/test/laboratory/data'; 12 | 13 | import { assertDeepEqual, initTestEnvironment } from './shared'; 14 | 15 | chai.use(chaiExclude); 16 | chai.use(chaiAsPromised); 17 | 18 | describe('laboratory/benchmarks', () => { 19 | let lab: ILaboratory; 20 | 21 | beforeEach(async () => { 22 | lab = await initTestEnvironment(); 23 | }); 24 | 25 | it('allBenchmarks()', async () => { 26 | const empty = await lab.allBenchmarks(); 27 | assert.deepEqual(empty, []); 28 | 29 | await lab.upsertBenchmark(benchmark1); 30 | const results1 = await lab.allBenchmarks(); 31 | assertDeepEqual(results1, [benchmark1]); 32 | 33 | await lab.upsertBenchmark(benchmark2); 34 | const results2 = await lab.allBenchmarks(); 35 | assertDeepEqual(results2, [benchmark1, benchmark2]); 36 | }); 37 | 38 | it('oneBenchmark()', async () => { 39 | await lab.upsertBenchmark(benchmark1); 40 | await lab.upsertBenchmark(benchmark2); 41 | 42 | const result1 = await lab.oneBenchmark('benchmark1'); 43 | assertDeepEqual(result1, benchmark1); 44 | 45 | const result2 = await lab.oneBenchmark('benchmark2'); 46 | assertDeepEqual(result2, benchmark2); 47 | 48 | // Throws for unknown name. 49 | await assert.isRejected(lab.oneBenchmark('unknown')); 50 | }); 51 | 52 | it('upsertBenchmark()', async () => { 53 | await lab.upsertBenchmark(benchmark1); 54 | const results1 = await lab.allBenchmarks(); 55 | assertDeepEqual(results1, [benchmark1]); 56 | 57 | await lab.upsertBenchmark(benchmark2); 58 | const results2 = await lab.allBenchmarks(); 59 | assertDeepEqual(results2, [benchmark1, benchmark2]); 60 | 61 | const benchmark3 = { 62 | ...benchmark1, 63 | }; 64 | await lab.upsertBenchmark(benchmark3); 65 | const results3 = await lab.allBenchmarks(); 66 | assertDeepEqual(results3, [benchmark3, benchmark2]); 67 | }); 68 | 69 | it('upsertBenchmark() - express route mismatch', async () => { 70 | await assert.isRejected(lab.upsertBenchmark(benchmark1, 'benchmark2')); 71 | }); 72 | 73 | it('upsertBenchmark() - normalization', async () => { 74 | // Throws for invalid name 75 | const b1 = { 76 | ...benchmark3, 77 | name: '123_invalid_name', 78 | }; 79 | await assert.isRejected(lab.upsertBenchmark(b1)); 80 | 81 | // Lowercases name 82 | const b2 = { 83 | ...benchmark3, 84 | name: benchmark3.name.toUpperCase(), 85 | }; 86 | await lab.upsertBenchmark(b2); 87 | 88 | const result = await lab.oneBenchmark(benchmark3.name); 89 | assertDeepEqual(result, benchmark3); 90 | }); 91 | }); 92 | -------------------------------------------------------------------------------- /packages/laboratory/test/sequelize_laboratory/candidates.test.ts: -------------------------------------------------------------------------------- 1 | import { ILaboratory } from '@microsoft/sds'; 2 | import * as chai from 'chai'; 3 | import { assert } from 'chai'; 4 | import chaiAsPromised = require('chai-as-promised'); 5 | import chaiExclude from 'chai-exclude'; 6 | 7 | import { 8 | benchmark1, 9 | candidate1, 10 | candidate2, 11 | candidate3, 12 | } from '../../../sds/test/laboratory/data'; 13 | 14 | import { assertDeepEqual, initTestEnvironment } from './shared'; 15 | 16 | chai.use(chaiExclude); 17 | chai.use(chaiAsPromised); 18 | 19 | describe('laboratory/candidates', () => { 20 | let lab: ILaboratory; 21 | 22 | beforeEach(async () => { 23 | lab = await initTestEnvironment(); 24 | }); 25 | 26 | it('allCandidates()', async () => { 27 | // First add benchmark referenced by candidate1 and candidate2. 28 | await lab.upsertBenchmark(benchmark1); 29 | 30 | const empty = await lab.allCandidates(); 31 | assert.deepEqual(empty, []); 32 | 33 | await lab.upsertCandidate(candidate1); 34 | const results1 = await lab.allCandidates(); 35 | assertDeepEqual(results1, [candidate1]); 36 | 37 | await lab.upsertCandidate(candidate2); 38 | const results2 = await lab.allCandidates(); 39 | assertDeepEqual(results2, [candidate1, candidate2]); 40 | }); 41 | 42 | it('oneCandidate()', async () => { 43 | // First add benchmark referenced by candidate1 and candidate2. 44 | await lab.upsertBenchmark(benchmark1); 45 | await lab.upsertCandidate(candidate1); 46 | await lab.upsertCandidate(candidate2); 47 | 48 | const result1 = await lab.oneCandidate('candidate1'); 49 | assertDeepEqual(result1, candidate1); 50 | 51 | const result2 = await lab.oneCandidate('candidate2'); 52 | assertDeepEqual(result2, candidate2); 53 | 54 | // Throws for unknown name. 55 | await assert.isRejected(lab.oneCandidate('unknown')); 56 | }); 57 | 58 | it('upsertCandidate()', async () => { 59 | // First add benchmark referenced by candidate1 and candidate2. 60 | await lab.upsertBenchmark(benchmark1); 61 | 62 | await lab.upsertCandidate(candidate1); 63 | const results1 = await lab.allCandidates(); 64 | assertDeepEqual(results1, [candidate1]); 65 | 66 | await lab.upsertCandidate(candidate2); 67 | const results2 = await lab.allCandidates(); 68 | assertDeepEqual(results2, [candidate1, candidate2]); 69 | 70 | const candidate3 = { 71 | ...candidate1, 72 | apiVersion: candidate1.apiVersion + 'x', 73 | }; 74 | await lab.upsertCandidate(candidate3); 75 | const results3 = await lab.allCandidates(); 76 | assertDeepEqual(results3, [candidate3, candidate2]); 77 | }); 78 | 79 | it('upsertCandidate() - express route mismatch', async () => { 80 | // First add benchmark referenced by candidate1 and candidate2. 81 | await lab.upsertBenchmark(benchmark1); 82 | 83 | await assert.isRejected(lab.upsertCandidate(candidate1, 'candidate2')); 84 | }); 85 | 86 | it('upsertCandidate() - normalization', async () => { 87 | // First add benchmark referenced by candidate1 and candidate2. 88 | await lab.upsertBenchmark(benchmark1); 89 | 90 | // Throws for invalid name 91 | const c1 = { 92 | ...candidate3, 93 | name: '123_invalid_name', 94 | }; 95 | await assert.isRejected(lab.upsertCandidate(c1)); 96 | 97 | // Throws for invalid benchmark name 98 | const c2 = { 99 | ...candidate3, 100 | benchmark: '123_invalid_name', 101 | }; 102 | await assert.isRejected(lab.upsertCandidate(c2)); 103 | 104 | // Lowercases name, benchmark 105 | const c4 = { 106 | ...candidate3, 107 | name: candidate3.name.toUpperCase(), 108 | benchmark: candidate3.benchmark.toUpperCase(), 109 | }; 110 | await lab.upsertCandidate(c4); 111 | 112 | const result = await lab.oneCandidate(candidate3.name); 113 | assertDeepEqual(result, candidate3); 114 | 115 | // Throws on non-existent benchmark 116 | const c5 = { 117 | ...candidate3, 118 | name: candidate3.name.toUpperCase(), 119 | benchmark: 'unknown', 120 | }; 121 | await assert.isRejected(lab.upsertCandidate(c5)); 122 | }); 123 | }); 124 | -------------------------------------------------------------------------------- /packages/laboratory/test/sequelize_laboratory/models/models.test.ts: -------------------------------------------------------------------------------- 1 | import { assert } from 'chai'; 2 | 3 | import { 4 | Benchmark, 5 | Candidate, 6 | Run, 7 | Suite, 8 | } from '../../../src/sequelize_laboratory/models'; 9 | 10 | import { 11 | benchmark1, 12 | candidate1, 13 | suite1, 14 | run1, 15 | } from '../../../../sds/test/laboratory/data'; 16 | 17 | import { assertDeepEqual } from '../shared'; 18 | import { initializeSequelize } from '../../../src/sequelize_laboratory'; 19 | 20 | describe('sequelize', () => { 21 | before(async () => { 22 | await initializeSequelize({ 23 | dialect: 'sqlite', 24 | storage: ':memory:', 25 | logging: false, 26 | }); 27 | }); 28 | 29 | // TODO: jsonColumn roundtrip 30 | 31 | describe('models', () => { 32 | it('benchmark roundtrip', async () => { 33 | // Create a benchmark and read it back. 34 | await Benchmark.create(benchmark1); 35 | const result = (await Benchmark.findOne({ 36 | where: { name: benchmark1.name }, 37 | }))!; 38 | assert.isDefined(result.createdAt); 39 | assert.isDefined(result.updatedAt); 40 | assertDeepEqual(result, benchmark1); 41 | 42 | // Update the benchmark and verify the updatedDate field changes. 43 | await Benchmark.update(result, { where: { name: benchmark1.name } }); 44 | const result2 = (await Benchmark.findOne({ 45 | where: { name: benchmark1.name }, 46 | }))!; 47 | assert.notEqual(result2.createdAt, result.createdAt); 48 | assert.notEqual(result2.updatedAt, result.updatedAt); 49 | }); 50 | 51 | it('candidate roundtrip', async () => { 52 | // Create a candidate and read it back. 53 | await Candidate.create(candidate1); 54 | const result = (await Candidate.findOne({ 55 | where: { name: candidate1.name }, 56 | }))!; 57 | assert.isDefined(result.createdAt); 58 | assert.isDefined(result.updatedAt); 59 | assertDeepEqual(result, candidate1); 60 | 61 | // Update the candidate and verify the updatedDate field changes. 62 | await Candidate.update(result, { where: { name: candidate1.name } }); 63 | const result2 = (await Candidate.findOne({ 64 | where: { name: candidate1.name }, 65 | }))!; 66 | assert.notEqual(result2.createdAt, result.createdAt); 67 | assert.notEqual(result2.updatedAt, result.updatedAt); 68 | }); 69 | 70 | it('suite roundtrip', async () => { 71 | // Create a suite and read it back. 72 | await Suite.create(suite1); 73 | const result = (await Suite.findOne({ where: { name: suite1.name } }))!; 74 | assert.isDefined(result.createdAt); 75 | assert.isDefined(result.updatedAt); 76 | assertDeepEqual(result, suite1); 77 | 78 | // Update the suite and verify the updatedDate field changes. 79 | await Suite.update(result, { where: { name: suite1.name } }); 80 | const result2 = (await Suite.findOne({ where: { name: suite1.name } }))!; 81 | assert.notEqual(result2.createdAt, result.createdAt); 82 | assert.notEqual(result2.updatedAt, result.updatedAt); 83 | }); 84 | 85 | it('run roundtrip', async () => { 86 | // Create a run and read it back. 87 | await Run.create(run1); 88 | const result = (await Run.findOne({ where: { name: run1.name } }))!; 89 | assert.isDefined(result.createdAt); 90 | assert.isDefined(result.updatedAt); 91 | assertDeepEqual(result, run1); 92 | 93 | // Update the run and verify the updatedDate field changes. 94 | await Run.update(result, { where: { name: run1.name } }); 95 | const result2 = (await Run.findOne({ where: { name: run1.name } }))!; 96 | assert.notEqual(result2.createdAt, result.createdAt); 97 | assert.notEqual(result2.updatedAt, result.updatedAt); 98 | }); 99 | }); 100 | }); 101 | -------------------------------------------------------------------------------- /packages/laboratory/test/sequelize_laboratory/shared.ts: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // 3 | // Utilities functions for tests in this directory. 4 | // 5 | /////////////////////////////////////////////////////////////////////////////// 6 | import * as chai from 'chai'; 7 | import { assert } from 'chai'; 8 | import chaiExclude from 'chai-exclude'; 9 | 10 | import { 11 | initializeSequelize, 12 | SequelizeLaboratory, 13 | } from '../../src/sequelize_laboratory'; 14 | import { PipelineRun, InMemoryQueue, IQueue } from '@microsoft/sds'; 15 | 16 | import { serviceURL } from '../../../sds/test/laboratory/data'; 17 | 18 | chai.use(chaiExclude); 19 | 20 | /////////////////////////////////////////////////////////////////////////////// 21 | // 22 | // Test environment setup 23 | // 24 | /////////////////////////////////////////////////////////////////////////////// 25 | 26 | export async function initTestEnvironment( 27 | queue: IQueue = new InMemoryQueue() 28 | ) { 29 | await initializeSequelize({ 30 | dialect: 'sqlite', 31 | storage: ':memory:', 32 | logging: false, 33 | }); 34 | return new SequelizeLaboratory(serviceURL, queue); 35 | } 36 | 37 | /////////////////////////////////////////////////////////////////////////////// 38 | // 39 | // Utility functions 40 | // 41 | /////////////////////////////////////////////////////////////////////////////// 42 | // Strip off most sequelize properties, by round-tripping through JSON. 43 | function toPOJO(x: T): T { 44 | return JSON.parse(JSON.stringify(x)) as T; 45 | } 46 | 47 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 48 | export function assertDeepEqual(observed: any, expected: any): void { 49 | assert.deepEqualExcludingEvery(toPOJO(observed), expected, [ 50 | 'createdAt', 51 | 'updatedAt', 52 | 'id', 53 | ]); 54 | } 55 | -------------------------------------------------------------------------------- /packages/laboratory/test/sequelize_laboratory/suites.test.ts: -------------------------------------------------------------------------------- 1 | import { ILaboratory } from '@microsoft/sds'; 2 | import * as chai from 'chai'; 3 | import { assert } from 'chai'; 4 | import chaiAsPromised = require('chai-as-promised'); 5 | import chaiExclude from 'chai-exclude'; 6 | 7 | import { 8 | benchmark1, 9 | suite1, 10 | suite2, 11 | suite3, 12 | } from '../../../sds/test/laboratory/data'; 13 | 14 | import { assertDeepEqual, initTestEnvironment } from './shared'; 15 | 16 | chai.use(chaiExclude); 17 | chai.use(chaiAsPromised); 18 | 19 | describe('laboratory/suites', () => { 20 | let lab: ILaboratory; 21 | 22 | beforeEach(async () => { 23 | lab = await initTestEnvironment(); 24 | }); 25 | 26 | it('allSuites()', async () => { 27 | // First add benchmark referenced by suite1 and suite2. 28 | await lab.upsertBenchmark(benchmark1); 29 | 30 | const empty = await lab.allSuites(); 31 | assert.deepEqual(empty, []); 32 | 33 | await lab.upsertSuite(suite1); 34 | const results1 = await lab.allSuites(); 35 | assertDeepEqual(results1, [suite1]); 36 | 37 | await lab.upsertSuite(suite2); 38 | const results2 = await lab.allSuites(); 39 | assertDeepEqual(results2, [suite1, suite2]); 40 | }); 41 | 42 | it('oneSuite()', async () => { 43 | // First add benchmark referenced by suite1 and suite2. 44 | await lab.upsertBenchmark(benchmark1); 45 | await lab.upsertSuite(suite1); 46 | await lab.upsertSuite(suite2); 47 | 48 | const result1 = await lab.oneSuite('suite1'); 49 | assertDeepEqual(result1, suite1); 50 | 51 | const result2 = await lab.oneSuite('suite2'); 52 | assertDeepEqual(result2, suite2); 53 | 54 | // Throws for unknown name. 55 | await assert.isRejected(lab.oneSuite('unknown')); 56 | }); 57 | 58 | it('upsertSuite()', async () => { 59 | // First add benchmark referenced by suite1 and suite2. 60 | await lab.upsertBenchmark(benchmark1); 61 | 62 | await lab.upsertSuite(suite1); 63 | const results1 = await lab.allSuites(); 64 | assertDeepEqual(results1, [suite1]); 65 | 66 | await lab.upsertSuite(suite2); 67 | const results2 = await lab.allSuites(); 68 | assertDeepEqual(results2, [suite1, suite2]); 69 | 70 | const suite3 = { 71 | ...suite1, 72 | apiVersion: suite1.apiVersion = 'x', 73 | }; 74 | await lab.upsertSuite(suite3); 75 | const results3 = await lab.allSuites(); 76 | assertDeepEqual(results3, [suite3, suite2]); 77 | }); 78 | 79 | it('upsertSuite() - express route mismatch', async () => { 80 | // First add benchmark referenced by suite1 and suite2. 81 | await lab.upsertBenchmark(benchmark1); 82 | 83 | await assert.isRejected(lab.upsertSuite(suite1, 'suite2')); 84 | }); 85 | 86 | it('upsertSuite() - normalization', async () => { 87 | // First add benchmark referenced by suite1 and suite2. 88 | await lab.upsertBenchmark(benchmark1); 89 | 90 | // Throws for invalid name 91 | const c1 = { 92 | ...suite3, 93 | name: '123_invalid_name', 94 | }; 95 | await assert.isRejected(lab.upsertSuite(c1)); 96 | 97 | // Throws for invalid benchmark name 98 | const c2 = { 99 | ...suite3, 100 | benchmark: '123_invalid_name', 101 | }; 102 | await assert.isRejected(lab.upsertSuite(c2)); 103 | 104 | // Lowercases name, benchmark 105 | const c4 = { 106 | ...suite3, 107 | name: suite3.name.toUpperCase(), 108 | benchmark: suite3.benchmark.toUpperCase(), 109 | }; 110 | await lab.upsertSuite(c4); 111 | 112 | const result = await lab.oneSuite(suite3.name); 113 | assertDeepEqual(result, suite3); 114 | 115 | // Throws on non-existent benchmark 116 | const c5 = { 117 | ...suite3, 118 | name: suite3.name.toUpperCase(), 119 | benchmark: 'unknown', 120 | }; 121 | await assert.isRejected(lab.upsertSuite(c5)); 122 | }); 123 | }); 124 | -------------------------------------------------------------------------------- /packages/laboratory/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.build.json", 3 | "compilerOptions": { 4 | "experimentalDecorators": true, 5 | "outDir": "./dist", 6 | }, 7 | "include": [ 8 | "src/**/*" 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /packages/laboratory/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "compilerOptions": { 4 | "experimentalDecorators": true 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /packages/sds/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../.eslintrc.json" 3 | } 4 | -------------------------------------------------------------------------------- /packages/sds/.nycrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../.nycrc" 3 | } 4 | -------------------------------------------------------------------------------- /packages/sds/.prettierignore: -------------------------------------------------------------------------------- 1 | dist 2 | node_modules 3 | -------------------------------------------------------------------------------- /packages/sds/.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | ...require('../../.prettierrc.js'), 3 | } 4 | -------------------------------------------------------------------------------- /packages/sds/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@microsoft/sds", 3 | "version": "0.1.0", 4 | "repository": { 5 | "url": "https://github.com/microsoft/secure-data-sandbox" 6 | }, 7 | "description": "A toolkit for conducting machine learning trials against confidential data", 8 | "license": "MIT", 9 | "main": "dist/index", 10 | "typings": "dist/index", 11 | "files": [ 12 | "dist" 13 | ], 14 | "engines": { 15 | "node": ">=12" 16 | }, 17 | "scripts": { 18 | "check": "gts check", 19 | "clean": "gts clean", 20 | "compile": "tsc -p tsconfig.build.json", 21 | "fix": "gts fix", 22 | "prepare": "npm run compile", 23 | "posttest": "npm run check", 24 | "test": "nyc ts-mocha" 25 | }, 26 | "devDependencies": { 27 | "@sinonjs/fake-timers": "^6.0.1", 28 | "@types/chai": "^4.2.12", 29 | "@types/chai-as-promised": "^7.1.3", 30 | "@types/luxon": "^1.25.0", 31 | "@types/mocha": "^8.0.3", 32 | "@types/nock": "^11.1.0", 33 | "@types/node": "^13.13.21", 34 | "@types/sinonjs__fake-timers": "^6.0.1", 35 | "@types/uuid": "^8.3.0", 36 | "chai": "^4.2.0", 37 | "chai-as-promised": "^7.1.1", 38 | "dotenv": "^8.2.0", 39 | "eslint": "^7.10.0", 40 | "gts": "^2.0.2", 41 | "mocha": "^8.1.3", 42 | "nock": "^13.0.4", 43 | "nyc": "^15.1.0", 44 | "source-map-support": "^0.5.19", 45 | "ts-mocha": "^7.0.0", 46 | "typescript": "^4.0.3", 47 | "uuid": "^8.3.0" 48 | }, 49 | "dependencies": { 50 | "@azure/identity": "^1.2.0", 51 | "@azure/storage-queue": "^12.1.0", 52 | "applicationinsights": "^1.8.7", 53 | "axios": "^0.19.2", 54 | "env-var": "^6.3.0", 55 | "fp-ts": "^2.8.2", 56 | "io-ts": "^2.2.10", 57 | "luxon": "^1.25.0" 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /packages/sds/src/configuration.ts: -------------------------------------------------------------------------------- 1 | import { promisify } from 'util'; 2 | import * as env from 'env-var'; 3 | import { 4 | DefaultAzureCredential, 5 | ManagedIdentityCredential, 6 | TokenCredential, 7 | } from '@azure/identity'; 8 | 9 | import { 10 | QueueMode, 11 | QueueConfiguration, 12 | AzureStorageQueueConfiguration, 13 | } from './queue'; 14 | 15 | const sleep = promisify(setTimeout); 16 | 17 | export class AzureCredential { 18 | private static tokenRetryIntervalMs = 5000; 19 | private static tokenRetryLimit = 12; 20 | 21 | private static instance: TokenCredential; 22 | 23 | private constructor() {} 24 | 25 | static async getInstance(): Promise { 26 | if (AzureCredential.instance) { 27 | return AzureCredential.instance; 28 | } 29 | 30 | const clientId = env.get('AZURE_CLIENT_ID').asString(); 31 | const errors = []; 32 | 33 | // Retry loop is a workaround for usage with aad-pod-identity, where the identity is not 34 | // mounted to the VMSS within 500ms, triggering a lockout that renders the credential 35 | // object unable to be used. We keep trying until we get a valid token or timeout 36 | for (let i = 0; i < AzureCredential.tokenRetryLimit; i++) { 37 | try { 38 | return await AzureCredential.initCredential(clientId); 39 | } catch (err) { 40 | errors.push(err); 41 | console.log( 42 | `[${i} of ${AzureCredential.tokenRetryLimit}] Unable to acquire AzureCredential. Retrying in ${AzureCredential.tokenRetryIntervalMs}ms...` 43 | ); 44 | await sleep(AzureCredential.tokenRetryIntervalMs); 45 | } 46 | } 47 | 48 | console.error( 49 | `Unable to acquire AzureCredential after ${AzureCredential.tokenRetryLimit} attempts` 50 | ); 51 | throw errors; 52 | } 53 | 54 | private static async initCredential(clientId?: string) { 55 | // DefaultAzureCredential currently fails when trying to get tokens for a User-Assigned Identity when deployed 56 | // to Azure App Service. https://github.com/Azure/azure-sdk-for-js/issues/11595 57 | // When this Issue is resolved, we should remove the try/catch and only use DefaultAzureCredential 58 | try { 59 | const cred = new DefaultAzureCredential({ 60 | managedIdentityClientId: clientId, 61 | }); 62 | await cred.getToken('https://management.azure.com/.default'); 63 | AzureCredential.instance = cred; 64 | return AzureCredential.instance; 65 | } catch (err) { 66 | // If DefaultAzureCredential has failed and there's no clientId - throw the original error 67 | if (!clientId) { 68 | throw err; 69 | } 70 | 71 | // Workaround for https://github.com/Azure/azure-sdk-for-js/issues/11595 72 | // Exceptions here should be allowed to throw 73 | const cred = new ManagedIdentityCredential(clientId); 74 | await cred.getToken('https://management.azure.com/.default'); 75 | AzureCredential.instance = cred; 76 | return AzureCredential.instance; 77 | } 78 | } 79 | } 80 | 81 | /** 82 | * Retrieve a QueueConfiguration from the current execution environment. 83 | */ 84 | export async function ParseQueueConfiguration(): Promise { 85 | const mode = env 86 | .get('QUEUE_MODE') 87 | .default(QueueMode.InMemory) 88 | .asEnum(Object.values(QueueMode)) as QueueMode; 89 | 90 | const endpoint = env 91 | .get('QUEUE_ENDPOINT') 92 | .required(mode !== QueueMode.InMemory) 93 | .asUrlString(); 94 | 95 | switch (mode) { 96 | case QueueMode.Azure: 97 | return { 98 | mode, 99 | endpoint, 100 | credential: await AzureCredential.getInstance(), 101 | shouldCreateQueue: false, 102 | } as AzureStorageQueueConfiguration; 103 | case QueueMode.InMemory: 104 | return { 105 | mode: QueueMode.InMemory, 106 | endpoint: 'http://localhost', 107 | }; 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /packages/sds/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './laboratory'; 2 | export * from './queue'; 3 | export * from './messages'; 4 | export * from './telemetry'; 5 | export * from './configuration'; 6 | -------------------------------------------------------------------------------- /packages/sds/src/laboratory/client/index.ts: -------------------------------------------------------------------------------- 1 | export * from './client'; 2 | -------------------------------------------------------------------------------- /packages/sds/src/laboratory/index.ts: -------------------------------------------------------------------------------- 1 | export * from './client'; 2 | export * from './interfaces'; 3 | export * from './normalize'; 4 | export * from './validate'; 5 | -------------------------------------------------------------------------------- /packages/sds/src/laboratory/normalize.ts: -------------------------------------------------------------------------------- 1 | import { IllegalOperationError } from './interfaces'; 2 | 3 | // Goals: 4 | // Suitable blob and file paths (eliminate most special characters) 5 | // Suitable for Azure table names (start with alpha, all lowercase) 6 | // Suiteable for bash command parameters (eliminate most special characters) 7 | // Eliminate risk of injection attack 8 | // Eliminate risk of aliasing attack 9 | // Alpha-numeric + [.-_] 10 | // Starts with alpha. 11 | // Length limited 12 | // Azure Tables: ^[A-Za-z][A-Za-z0-9]{2,62}$ 13 | export function normalizeName(name: string): string { 14 | const s = name.toLowerCase(); 15 | if (!s.match(/^[a-z][a-z0-9]{2,62}$/)) { 16 | const message = `Invalid name format "${name}". Names must be alpha numeric, starting with an alpha.`; 17 | throw new IllegalOperationError(message); 18 | } 19 | return s; 20 | } 21 | 22 | export function normalizeRunName(name: string): string { 23 | const n = name.toLowerCase(); 24 | 25 | // Verify that n is a guid. 26 | // https://stackoverflow.com/questions/7905929/how-to-test-valid-uuid-guid 27 | if ( 28 | !n.match( 29 | /^[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}$/i 30 | ) 31 | ) { 32 | const message = `Invalid run id "${name}". Run ids should be guids.`; 33 | throw new IllegalOperationError(message); 34 | } 35 | 36 | return n; 37 | } 38 | -------------------------------------------------------------------------------- /packages/sds/src/laboratory/validate.ts: -------------------------------------------------------------------------------- 1 | import { isLeft } from 'fp-ts/lib/Either'; 2 | import { Decoder } from 'io-ts'; 3 | 4 | import { ValidationError } from './interfaces'; 5 | 6 | export function validate(decoder: Decoder, data: I): A { 7 | const x = decoder.decode(data); 8 | if (isLeft(x)) { 9 | const message = `${decoder.name}: data does not conform to schema`; 10 | throw new ValidationError(message); 11 | } else { 12 | return x.right; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /packages/sds/src/messages.ts: -------------------------------------------------------------------------------- 1 | // Data contracts for queue messages 2 | 3 | export type PipelineRun = { 4 | name: string; 5 | laboratoryEndpoint: string; 6 | stages: PipelineRunStage[]; 7 | // TODO: consider elevating volumes to top-level property 8 | }; 9 | 10 | export interface PipelineRunStage { 11 | name: string; 12 | kind: string; 13 | image: string; 14 | cmd?: string[]; 15 | env?: Record; 16 | volumes?: PipelineRunStageVolume[]; 17 | } 18 | 19 | export interface PipelineRunStageVolume { 20 | type: string; 21 | target: string; 22 | name: string; 23 | // TODO: allow `source` as optional for type: 'ephemeral' 24 | source: string | undefined; 25 | readonly: boolean; 26 | } 27 | -------------------------------------------------------------------------------- /packages/sds/src/queue/azure.ts: -------------------------------------------------------------------------------- 1 | import { TokenCredential } from '@azure/identity'; 2 | import { DequeuedMessageItem, QueueClient } from '@azure/storage-queue'; 3 | import { IQueue, QueueMessage, QueueConfiguration, QueueMode } from '.'; 4 | import { TelemetryClient, defaultClient } from 'applicationinsights'; 5 | import { Events } from '../telemetry'; 6 | 7 | export interface AzureStorageQueueConfiguration extends QueueConfiguration { 8 | mode: QueueMode.Azure; 9 | credential: TokenCredential; 10 | shouldCreateQueue: boolean; 11 | } 12 | 13 | /** 14 | * Simple client to send/receive messages via Azure Storage Queue 15 | */ 16 | export class AzureStorageQueue implements IQueue { 17 | private queueCreated = false; 18 | 19 | private readonly client: QueueClient; 20 | private readonly shouldCreateQueue: boolean; 21 | private readonly telemetryClient: TelemetryClient; 22 | 23 | /** 24 | * Creates an instance of AzureStorageQueue. 25 | * @param url A URL string pointing to Azure Storage queue, such as "https://mystorage.queue.core.windows.net/myqueue". 26 | * @param credential A TokenCredential from the @azure/identity package to authenticate via Azure Active Directory. 27 | */ 28 | constructor( 29 | config: AzureStorageQueueConfiguration, 30 | telemetryClient: TelemetryClient = defaultClient 31 | ) { 32 | this.client = new QueueClient(config.endpoint, config.credential); 33 | this.shouldCreateQueue = config.shouldCreateQueue; 34 | this.telemetryClient = telemetryClient; 35 | } 36 | 37 | async enqueue(message: T): Promise { 38 | await this.ensureQueue(); 39 | 40 | const wireMessage = JSON.stringify(message); 41 | const response = await this.client.sendMessage(wireMessage); 42 | 43 | this.telemetryClient.trackEvent({ 44 | name: Events.QueueMessageCreated, 45 | properties: { 46 | storageAccount: this.client.accountName, 47 | name: this.client.name, 48 | messageId: response.messageId, 49 | }, 50 | }); 51 | } 52 | 53 | async dequeue(count = 1): Promise>> { 54 | await this.ensureQueue(); 55 | 56 | const response = await this.client.receiveMessages({ 57 | numberOfMessages: count, 58 | }); 59 | 60 | return response.receivedMessageItems.map(m => { 61 | this.telemetryClient.trackEvent({ 62 | name: Events.QueueMessageDequeued, 63 | properties: { 64 | storageAccount: this.client.accountName, 65 | name: this.client.name, 66 | messageId: m.messageId, 67 | }, 68 | }); 69 | 70 | return { 71 | value: JSON.parse(m.messageText) as T, 72 | dequeueCount: m.dequeueCount, 73 | complete: () => this.completeMessage(m), 74 | }; 75 | }); 76 | } 77 | 78 | private async completeMessage(m: DequeuedMessageItem): Promise { 79 | await this.client.deleteMessage(m.messageId, m.popReceipt); 80 | 81 | this.telemetryClient.trackEvent({ 82 | name: Events.QueueMessageCompleted, 83 | properties: { 84 | storageAccount: this.client.accountName, 85 | name: this.client.name, 86 | messageId: m.messageId, 87 | }, 88 | }); 89 | } 90 | 91 | private async ensureQueue(): Promise { 92 | if (this.shouldCreateQueue && !this.queueCreated) { 93 | await this.client.create(); 94 | this.queueCreated = true; 95 | 96 | this.telemetryClient.trackEvent({ 97 | name: Events.QueueCreated, 98 | properties: { 99 | storageAccount: this.client.accountName, 100 | name: this.client.name, 101 | }, 102 | }); 103 | } 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /packages/sds/src/queue/index.ts: -------------------------------------------------------------------------------- 1 | import { AzureStorageQueue, AzureStorageQueueConfiguration } from './azure'; 2 | import { InMemoryQueue } from './inmemory'; 3 | 4 | /** 5 | * Simple interface to send/receive messages from a queue. 6 | */ 7 | // eslint-disable-next-line @typescript-eslint/interface-name-prefix 8 | export interface IQueue { 9 | /** 10 | * Enqueue a single message, serialized as JSON. 11 | * @param message An object/message to place on the queue. 12 | */ 13 | enqueue(message: T): Promise; 14 | 15 | /** 16 | * Dequeue a batch of JSON-formatted messages. 17 | * You will need to call message.complete() on each message to remove it from the queue. 18 | */ 19 | dequeue(count: number): Promise>>; 20 | } 21 | 22 | /** 23 | * Message envelope that supports generic types and processing confirmation. 24 | */ 25 | export interface QueueMessage { 26 | value: T; 27 | dequeueCount: number; 28 | complete(): Promise; 29 | } 30 | 31 | /** 32 | * Represents a specific Queue implementation. 33 | */ 34 | export enum QueueMode { 35 | Azure = 'azure', 36 | InMemory = 'inmemory', 37 | } 38 | 39 | export interface QueueConfiguration { 40 | mode: QueueMode; 41 | endpoint: string; 42 | } 43 | 44 | /** 45 | * Factory to get a Queue. 46 | * @param mode The implementation to use. 47 | * @param endpoint The location of the queue. 48 | */ 49 | export function GetQueue(config: QueueConfiguration): IQueue { 50 | switch (config.mode) { 51 | case QueueMode.InMemory: 52 | return new InMemoryQueue(); 53 | case QueueMode.Azure: 54 | return new AzureStorageQueue(config as AzureStorageQueueConfiguration); 55 | } 56 | } 57 | 58 | // re-exports so 'queue' is usable at the top-level 59 | export * from './azure'; 60 | export * from './inmemory'; 61 | export * from './processor'; 62 | -------------------------------------------------------------------------------- /packages/sds/src/queue/inmemory.ts: -------------------------------------------------------------------------------- 1 | import { IQueue, QueueMessage } from '.'; 2 | 3 | export class InMemoryQueue implements IQueue { 4 | // Explicitly allowing 'any' for this test helper object 5 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 6 | readonly data: any[] = []; 7 | 8 | readonly visibilityTimeout: number; 9 | readonly dequeueCounts = new Map(); 10 | 11 | constructor(visibilityTimeout = 1000) { 12 | this.visibilityTimeout = visibilityTimeout; 13 | } 14 | 15 | async enqueue(message: T): Promise { 16 | this.data.push(message); 17 | } 18 | 19 | async dequeue(count: number): Promise>> { 20 | const items = new Array(); 21 | 22 | for (let i = 0; i < count; i++) { 23 | if (this.data.length > 0) { 24 | items.push(this.data.shift()); 25 | } 26 | } 27 | 28 | for (const item of items) { 29 | const json = JSON.stringify(item); 30 | let count = this.dequeueCounts.get(json) ?? 0; 31 | count++; 32 | this.dequeueCounts.set(json, count); 33 | } 34 | 35 | return items.map(m => { 36 | const itemTimeout = setTimeout(() => { 37 | this.data.unshift(m); 38 | }, this.visibilityTimeout); 39 | 40 | const json = JSON.stringify(m); 41 | 42 | return { 43 | value: m, 44 | dequeueCount: this.dequeueCounts.get(json)!, 45 | complete: async () => { 46 | clearTimeout(itemTimeout); 47 | this.dequeueCounts.delete(json); 48 | }, 49 | }; 50 | }); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /packages/sds/src/queue/processor.ts: -------------------------------------------------------------------------------- 1 | import { IQueue } from '.'; 2 | import { TelemetryClient, defaultClient } from 'applicationinsights'; 3 | import { Events } from '../telemetry'; 4 | 5 | export interface QueueProcessorOptions { 6 | receiveBatchSize: number; 7 | receiveIntervalMs: number; 8 | maxAttemptsPerMessage: number; 9 | } 10 | 11 | const defaultQueueProcessorOptions: QueueProcessorOptions = { 12 | receiveBatchSize: 1, 13 | receiveIntervalMs: 5000, 14 | maxAttemptsPerMessage: 3, 15 | }; 16 | 17 | /** 18 | * Process messages from a queue with a pluggable message handler. 19 | */ 20 | export class QueueProcessor { 21 | private readonly queue: IQueue; 22 | private readonly options: Readonly; 23 | private interval?: NodeJS.Timeout; 24 | private readonly telemetryClient: TelemetryClient; 25 | 26 | constructor( 27 | queue: IQueue, 28 | options?: Partial, 29 | telemetryClient: TelemetryClient = defaultClient 30 | ) { 31 | this.queue = queue; 32 | this.options = { ...defaultQueueProcessorOptions, ...options }; 33 | this.telemetryClient = telemetryClient; 34 | } 35 | 36 | /** 37 | * Start processing messages from the queue. 38 | * @param processor A function that processes a single message. 39 | */ 40 | start(processor: MessageProcessor) { 41 | if (!this.interval) { 42 | setImmediate(() => this.processBatch(processor)); 43 | this.interval = setInterval( 44 | () => this.processBatch(processor), 45 | this.options.receiveIntervalMs 46 | ); 47 | } 48 | } 49 | 50 | /** 51 | * Stop receiving new messages. Messages already received will be processed. 52 | */ 53 | stop() { 54 | if (this.interval) { 55 | clearInterval(this.interval); 56 | this.interval = undefined; 57 | } 58 | } 59 | 60 | private async processBatch(processMessage: MessageProcessor) { 61 | for (const message of await this.queue.dequeue( 62 | this.options.receiveBatchSize 63 | )) { 64 | try { 65 | if (message.dequeueCount > this.options.maxAttemptsPerMessage) { 66 | //TODO: dead letter 67 | console.warn(`could not process ${JSON.stringify(message)}`); 68 | await message.complete(); 69 | 70 | this.telemetryClient.trackEvent({ 71 | name: Events.QueueMessageUnprocessable, 72 | properties: { 73 | attempts: message.dequeueCount, 74 | }, 75 | }); 76 | continue; 77 | } 78 | 79 | await processMessage(message.value); 80 | await message.complete(); 81 | } catch (e) { 82 | //TODO: log errors 83 | console.error(e); 84 | } 85 | } 86 | } 87 | } 88 | 89 | type MessageProcessor = (message: T) => Promise; 90 | -------------------------------------------------------------------------------- /packages/sds/src/shims.d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-empty-interface */ 2 | 3 | // superagent 4 | // https://github.com/DefinitelyTyped/DefinitelyTyped/issues/12044 5 | declare interface XMLHttpRequest {} 6 | -------------------------------------------------------------------------------- /packages/sds/src/telemetry.ts: -------------------------------------------------------------------------------- 1 | import * as appInsights from 'applicationinsights'; 2 | 3 | export enum Events { 4 | QueueCreated = 'queueCreated', 5 | QueueMessageCompleted = 'queueMessageCompleted', 6 | QueueMessageCreated = 'queueMessageCreated', 7 | QueueMessageDequeued = 'queueMessageDequeued', 8 | QueueMessageUnprocessable = 'queueMessageUnprocessable', 9 | } 10 | 11 | export function InitTelemetry() { 12 | try { 13 | appInsights.setup().start(); 14 | } catch (err) { 15 | console.warn('Warning: Could not initialize telemetry'); 16 | appInsights.setup('00000000-0000-0000-0000-000000000000').start(); 17 | appInsights.defaultClient.config.disableAppInsights = true; 18 | if (err.message) { 19 | console.warn(`Warning: ${err.message}`); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /packages/sds/test/.mocharc.yml: -------------------------------------------------------------------------------- 1 | recursive: true 2 | require: 'source-map-support/register' 3 | spec: '**/*.ts' 4 | -------------------------------------------------------------------------------- /packages/sds/test/functional/.mocharc.yml: -------------------------------------------------------------------------------- 1 | spec: build/test/functional 2 | recursive: true 3 | require: 'dotenv/config' 4 | -------------------------------------------------------------------------------- /packages/sds/test/functional/configuration.ts: -------------------------------------------------------------------------------- 1 | import { DefaultAzureCredential } from '@azure/identity'; 2 | import { QueueClient } from '@azure/storage-queue'; 3 | import * as env from 'env-var'; 4 | import { v1 } from 'uuid'; 5 | import { 6 | AzureStorageQueueConfiguration, 7 | GetQueue, 8 | QueueMode, 9 | } from '../../src/queue'; 10 | 11 | export function getQueueConfiguration() { 12 | const serviceUrl = env.get('TEST_QUEUE_SERVICE_URL').required().asUrlString(); 13 | const credential = new DefaultAzureCredential(); 14 | const queueEndpoint = `${serviceUrl}${v1()}`; 15 | const config: AzureStorageQueueConfiguration = { 16 | mode: QueueMode.Azure, 17 | endpoint: queueEndpoint, 18 | credential, 19 | shouldCreateQueue: true, 20 | }; 21 | 22 | return { 23 | client: new QueueClient(queueEndpoint, credential), 24 | queue: GetQueue(config), 25 | }; 26 | } 27 | -------------------------------------------------------------------------------- /packages/sds/test/functional/queue/azure.test.ts: -------------------------------------------------------------------------------- 1 | import { QueueClient } from '@azure/storage-queue'; 2 | import { assert } from 'chai'; 3 | import { IQueue } from '../../../src/queue'; 4 | import { getQueueConfiguration } from '../configuration'; 5 | 6 | describe('functional.queue.azure', () => { 7 | interface TestMessage { 8 | name: string; 9 | children?: Array<{ 10 | id: number; 11 | childName: string; 12 | }>; 13 | } 14 | 15 | describe('withExistingQueue', () => { 16 | let client: QueueClient; 17 | let queue: IQueue; 18 | 19 | before(async () => { 20 | const config = getQueueConfiguration(); 21 | client = config.client; 22 | queue = config.queue; 23 | 24 | await client.create(); 25 | }); 26 | 27 | after(async () => { 28 | await client.delete(); 29 | }); 30 | 31 | it('enqueues', async () => { 32 | await queue.enqueue('test1'); 33 | await queue.enqueue('test2'); 34 | 35 | const response = await client.peekMessages({ numberOfMessages: 32 }); 36 | assert.lengthOf(response.peekedMessageItems, 2); 37 | }); 38 | 39 | it('dequeues', async () => { 40 | const batch1 = await queue.dequeue(1); 41 | const msg1 = batch1[0]; 42 | assert.equal(msg1.value, 'test1'); 43 | await msg1.complete(); 44 | 45 | const batch2 = await queue.dequeue(1); 46 | const msg2 = batch2[0]; 47 | assert.equal(msg2.value, 'test2'); 48 | await msg2.complete(); 49 | 50 | const response = await client.peekMessages({ numberOfMessages: 32 }); 51 | assert.lengthOf(response.peekedMessageItems, 0); 52 | }); 53 | 54 | it('handlesObjects', async () => { 55 | const input: TestMessage = { 56 | name: 'foo', 57 | }; 58 | 59 | const { queue } = getQueueConfiguration(); 60 | await queue.enqueue(input); 61 | 62 | const response = await queue.dequeue(1); 63 | const msg = response[0]; 64 | const output = msg.value; 65 | await msg.complete(); 66 | 67 | assert.deepStrictEqual(output, input); 68 | }); 69 | 70 | it('handlesObjectsWithChildren', async () => { 71 | const input: TestMessage = { 72 | name: 'foo', 73 | children: [ 74 | { 75 | id: 1, 76 | childName: 'child1', 77 | }, 78 | { 79 | id: 2, 80 | childName: 'child2', 81 | }, 82 | ], 83 | }; 84 | 85 | const { queue } = getQueueConfiguration(); 86 | await queue.enqueue(input); 87 | 88 | const response = await queue.dequeue(1); 89 | const msg = response[0]; 90 | const output = msg.value; 91 | await msg.complete(); 92 | 93 | assert.deepStrictEqual(output, input); 94 | }); 95 | }); 96 | 97 | describe('withNewQueue', () => { 98 | let client: QueueClient; 99 | let queue: IQueue; 100 | 101 | before(async () => { 102 | const config = getQueueConfiguration(); 103 | client = config.client; 104 | queue = config.queue; 105 | }); 106 | 107 | after(async () => { 108 | await client.delete(); 109 | }); 110 | 111 | it('enqueues', async () => { 112 | await queue.enqueue('test1'); 113 | await queue.enqueue('test2'); 114 | 115 | const response = await client.peekMessages({ numberOfMessages: 32 }); 116 | assert.lengthOf(response.peekedMessageItems, 2); 117 | }); 118 | }); 119 | }); 120 | -------------------------------------------------------------------------------- /packages/sds/test/index.ts: -------------------------------------------------------------------------------- 1 | require('mocha'); 2 | import * as appInsights from 'applicationinsights'; 3 | 4 | before(() => { 5 | appInsights.setup('00000000-0000-0000-0000-000000000000'); 6 | appInsights.defaultClient.config.disableAppInsights = true; 7 | }); 8 | -------------------------------------------------------------------------------- /packages/sds/test/queue/azure.test.ts: -------------------------------------------------------------------------------- 1 | import { DefaultAzureCredential } from '@azure/identity'; 2 | import { assert } from 'chai'; 3 | import { AzureStorageQueue } from '../../src/queue/azure'; 4 | import { QueueMode } from '../../src/queue'; 5 | 6 | const QUEUE_ENDPOINT = 'https://mystorage.queue.core.windows.net/myqueue'; 7 | 8 | describe('queue.azure', () => { 9 | it('initializes', () => { 10 | const client = new AzureStorageQueue({ 11 | mode: QueueMode.Azure, 12 | endpoint: QUEUE_ENDPOINT, 13 | credential: new DefaultAzureCredential(), 14 | shouldCreateQueue: false, 15 | }); 16 | assert.isNotNull(client); 17 | }); 18 | }); 19 | -------------------------------------------------------------------------------- /packages/sds/test/queue/index.ts: -------------------------------------------------------------------------------- 1 | import { IQueue, QueueMessage } from '../../src/queue'; 2 | 3 | export class FakeQueue implements IQueue { 4 | // Explicitly allowing 'any' for this test helper object 5 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 6 | readonly data: any[] = []; 7 | 8 | readonly visibilityTimeout: number; 9 | readonly dequeueCounts = new Map(); 10 | 11 | constructor(visibilityTimeout = 1000) { 12 | this.visibilityTimeout = visibilityTimeout; 13 | } 14 | 15 | async enqueue(message: T): Promise { 16 | this.data.push(message); 17 | } 18 | 19 | async dequeue(count: number): Promise>> { 20 | const items = new Array(); 21 | 22 | for (let i = 0; i < count; i++) { 23 | if (this.data.length > 0) { 24 | items.push(this.data.shift()); 25 | } 26 | } 27 | 28 | for (const item of items) { 29 | const json = JSON.stringify(item); 30 | let count = this.dequeueCounts.get(json) ?? 0; 31 | count++; 32 | this.dequeueCounts.set(json, count); 33 | } 34 | 35 | return items.map(m => { 36 | const itemTimeout = setTimeout(() => { 37 | this.data.unshift(m); 38 | }, this.visibilityTimeout); 39 | 40 | const json = JSON.stringify(m); 41 | 42 | return { 43 | value: m, 44 | dequeueCount: this.dequeueCounts.get(json)!, 45 | complete: async () => { 46 | clearTimeout(itemTimeout); 47 | this.dequeueCounts.delete(json); 48 | }, 49 | }; 50 | }); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /packages/sds/test/queue/processor.test.ts: -------------------------------------------------------------------------------- 1 | import { assert } from 'chai'; 2 | import * as FakeTimers from '@sinonjs/fake-timers'; 3 | import { QueueProcessor, InMemoryQueue } from '../../src/queue'; 4 | 5 | describe('processor', () => { 6 | let clock: FakeTimers.InstalledClock; 7 | 8 | before(() => { 9 | clock = FakeTimers.install(); 10 | }); 11 | 12 | after(() => { 13 | clock.uninstall(); 14 | }); 15 | 16 | it('initializesWithDefaults', () => { 17 | const queue = new InMemoryQueue(); 18 | const processor = new QueueProcessor(queue); 19 | 20 | assert.isNotNull(processor); 21 | }); 22 | 23 | it('starts', async () => { 24 | const queue = new InMemoryQueue(); 25 | const processor = new QueueProcessor(queue); 26 | 27 | await Promise.all([ 28 | queue.enqueue('msg 1'), 29 | queue.enqueue('msg 2'), 30 | queue.enqueue('msg 3'), 31 | ]); 32 | 33 | processor.start(async () => { 34 | // no-op 35 | }); 36 | 37 | assert.equal(queue.data.length, 3); 38 | await clock.tickAsync(15000); 39 | assert.equal(queue.data.length, 0); 40 | 41 | processor.stop(); 42 | }); 43 | 44 | it('stops', async () => { 45 | const queue = new InMemoryQueue(); 46 | const processor = new QueueProcessor(queue, { 47 | receiveBatchSize: 1, 48 | receiveIntervalMs: 5000, 49 | }); 50 | 51 | await Promise.all([ 52 | queue.enqueue('msg 1'), 53 | queue.enqueue('msg 2'), 54 | queue.enqueue('msg 3'), 55 | ]); 56 | 57 | processor.start(async () => { 58 | // no-op 59 | }); 60 | 61 | assert.equal(queue.data.length, 3); 62 | await clock.tickAsync(5000); 63 | assert.equal(queue.data.length, 1); 64 | 65 | processor.stop(); 66 | await clock.tickAsync(60000); 67 | assert.equal(queue.data.length, 1); 68 | }); 69 | 70 | it('usesMaxAttemptsPerMessage', async () => { 71 | const queue = new InMemoryQueue(1000); 72 | const processor = new QueueProcessor(queue, { 73 | maxAttemptsPerMessage: 3, 74 | receiveBatchSize: 1, 75 | receiveIntervalMs: 5000, 76 | }); 77 | 78 | await queue.enqueue('msg 1'); 79 | 80 | let attempts = 0; 81 | processor.start(async () => { 82 | attempts++; 83 | throw new Error('simulated failure'); 84 | }); 85 | 86 | assert.equal(queue.data.length, 1); 87 | await clock.tickAsync(15000); 88 | assert.equal(attempts, 3); 89 | assert.equal(queue.data.length, 0); 90 | 91 | processor.stop(); 92 | }); 93 | 94 | it('usesReceiveBatchSize', async () => { 95 | const queue = new InMemoryQueue(); 96 | const processor = new QueueProcessor(queue, { 97 | receiveBatchSize: 5, 98 | receiveIntervalMs: 5000, 99 | }); 100 | 101 | await Promise.all([ 102 | queue.enqueue('msg 1'), 103 | queue.enqueue('msg 2'), 104 | queue.enqueue('msg 3'), 105 | queue.enqueue('msg 4'), 106 | queue.enqueue('msg 5'), 107 | ]); 108 | 109 | let messageCount = 0; 110 | processor.start(async () => { 111 | messageCount++; 112 | }); 113 | 114 | assert.equal(queue.data.length, 5); 115 | await clock.tickAsync(5000); 116 | assert.equal(queue.data.length, 0); 117 | assert.equal(messageCount, 5); 118 | 119 | processor.stop(); 120 | }); 121 | 122 | it('usesReceiveIntervalMs', async () => { 123 | const queue = new InMemoryQueue(); 124 | const processor = new QueueProcessor(queue, { 125 | receiveIntervalMs: 10000, 126 | }); 127 | 128 | await Promise.all([ 129 | queue.enqueue('msg 1'), 130 | queue.enqueue('msg 2'), 131 | queue.enqueue('msg 3'), 132 | ]); 133 | 134 | processor.start(async () => { 135 | // no-op 136 | }); 137 | 138 | assert.equal(queue.data.length, 3); 139 | 140 | await clock.tickAsync(1); 141 | assert.equal(queue.data.length, 2); 142 | 143 | await clock.tickAsync(10000); 144 | assert.equal(queue.data.length, 1); 145 | 146 | await clock.tickAsync(10000); 147 | assert.equal(queue.data.length, 0); 148 | 149 | processor.stop(); 150 | }); 151 | }); 152 | -------------------------------------------------------------------------------- /packages/sds/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.build.json", 3 | "compilerOptions": { 4 | "outDir": "./dist" 5 | }, 6 | "include": [ 7 | "src/**/*" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /packages/sds/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json" 3 | } 4 | -------------------------------------------------------------------------------- /packages/worker/.env.template: -------------------------------------------------------------------------------- 1 | # To use in bash: 2 | # set -o allexport; source .env; set +o allexport 3 | 4 | # Worker configuration 5 | QUEUE_MODE=azure 6 | QUEUE_ENDPOINT=https://mystorage.queue.core.windows.net/myqueue 7 | -------------------------------------------------------------------------------- /packages/worker/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../.eslintrc.json" 3 | } 4 | -------------------------------------------------------------------------------- /packages/worker/.nycrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../.nycrc" 3 | } 4 | -------------------------------------------------------------------------------- /packages/worker/.prettierignore: -------------------------------------------------------------------------------- 1 | dist 2 | node_modules 3 | -------------------------------------------------------------------------------- /packages/worker/.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | ...require('../../.prettierrc.js'), 3 | } 4 | -------------------------------------------------------------------------------- /packages/worker/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@microsoft/sds-worker", 3 | "version": "0.1.0", 4 | "repository": { 5 | "url": "https://github.com/microsoft/secure-data-sandbox" 6 | }, 7 | "description": "A toolkit for conducting machine learning trials against confidential data", 8 | "license": "MIT", 9 | "main": "dist/index", 10 | "typings": "dist/index", 11 | "files": [ 12 | "dist" 13 | ], 14 | "engines": { 15 | "node": ">=12" 16 | }, 17 | "bin": { 18 | "sds-worker": "./dist/index.js" 19 | }, 20 | "scripts": { 21 | "check": "gts check", 22 | "clean": "gts clean", 23 | "compile": "tsc -p tsconfig.build.json", 24 | "fix": "gts fix", 25 | "prepare": "npm run compile", 26 | "posttest": "npm run check", 27 | "test": "nyc ts-mocha" 28 | }, 29 | "devDependencies": { 30 | "@types/chai": "^4.2.12", 31 | "@types/mocha": "^8.0.3", 32 | "chai": "^4.2.0", 33 | "eslint": "^7.10.0", 34 | "gts": "^2.0.2", 35 | "mocha": "^8.1.3", 36 | "nyc": "^15.1.0", 37 | "source-map-support": "^0.5.19", 38 | "ts-mocha": "^7.0.0", 39 | "typescript": "^4.0.3" 40 | }, 41 | "dependencies": { 42 | "@kubernetes/client-node": "^0.12.1", 43 | "@microsoft/sds": "*", 44 | "applicationinsights": "^1.8.7", 45 | "env-var": "^6.3.0", 46 | "stream-buffers": "^3.0.2" 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /packages/worker/src/argoWorker.ts: -------------------------------------------------------------------------------- 1 | import * as k8s from '@kubernetes/client-node'; 2 | import { 3 | IQueue, 4 | QueueProcessor, 5 | PipelineRun, 6 | BenchmarkStageKind, 7 | } from '@microsoft/sds'; 8 | import { defaultClient } from 'applicationinsights'; 9 | import { Workflow, Template, PersistentVolumeClaim } from './argo'; 10 | import { ArgoWorkerConfiguration } from './configuration'; 11 | 12 | // Executes Runs by creating an Argo workflow 13 | export class ArgoWorker { 14 | private readonly processor: QueueProcessor; 15 | private readonly crd: k8s.CustomObjectsApi; 16 | private readonly config: ArgoWorkerConfiguration; 17 | 18 | constructor( 19 | queue: IQueue, 20 | kc: k8s.KubeConfig, 21 | config: ArgoWorkerConfiguration 22 | ) { 23 | this.processor = new QueueProcessor(queue); 24 | this.crd = kc.makeApiClient(k8s.CustomObjectsApi); 25 | this.config = config; 26 | } 27 | 28 | start() { 29 | this.processor.start(run => this.processRun(run)); 30 | } 31 | 32 | stop() { 33 | this.processor.stop(); 34 | } 35 | 36 | private async processRun(run: PipelineRun) { 37 | console.log(`Processing run: ${run.name}`); 38 | 39 | const workflow = createWorkflow(run, this.config); 40 | await this.crd.createNamespacedCustomObject( 41 | 'argoproj.io', 42 | 'v1alpha1', 43 | 'runs', 44 | 'workflows', 45 | workflow 46 | ); 47 | } 48 | } 49 | 50 | export function createWorkflow( 51 | run: PipelineRun, 52 | config: ArgoWorkerConfiguration 53 | ): Workflow { 54 | const steps = run.stages.map(s => [ 55 | { 56 | name: s.name, 57 | template: s.name, 58 | }, 59 | ]); 60 | 61 | const templates = run.stages.map(s => { 62 | const template: Template = { 63 | name: s.name, 64 | metadata: { 65 | labels: { 66 | aadpodidbinding: getIdentity(s.kind), 67 | }, 68 | }, 69 | container: { 70 | image: s.image, 71 | }, 72 | }; 73 | 74 | const volumeMounts = s.volumes?.map(v => ({ 75 | name: v.name, 76 | mountPath: v.target, 77 | readOnly: v.readonly, 78 | })); 79 | if (volumeMounts) { 80 | template.container!.volumeMounts = volumeMounts; 81 | } 82 | 83 | if (s.cmd) { 84 | template.container!.args = s.cmd; 85 | } 86 | 87 | if (s.env) { 88 | template.container!.env = Object.entries(s.env || {})?.map(e => ({ 89 | name: e[0], 90 | value: e[1], 91 | })); 92 | } else { 93 | template.container!.env = []; 94 | } 95 | 96 | if (defaultClient.config.instrumentationKey) { 97 | template.container!.env!.push({ 98 | name: 'APPINSIGHTS_INSTRUMENTATIONKEY', 99 | value: defaultClient.config.instrumentationKey, 100 | }); 101 | } 102 | 103 | return template; 104 | }); 105 | 106 | const workflow: Workflow = { 107 | apiVersion: 'argoproj.io/v1alpha1', 108 | kind: 'Workflow', 109 | metadata: { 110 | generateName: run.name, 111 | }, 112 | spec: { 113 | entrypoint: 'run', 114 | templates: [ 115 | { 116 | name: 'run', 117 | steps: steps, 118 | }, 119 | ...templates, 120 | ], 121 | ttlStrategy: { 122 | secondsAfterSuccess: config.successfulRunGCSeconds, 123 | }, 124 | }, 125 | }; 126 | 127 | if (run.stages.some(s => s.volumes?.length || 0 > 0)) { 128 | const volumeClaimTemplates: PersistentVolumeClaim[] = []; 129 | 130 | for (const stage of run.stages) { 131 | for (const volume of stage.volumes || []) { 132 | if (!volumeClaimTemplates.some(c => c.metadata?.name === volume.name)) { 133 | const pvc: PersistentVolumeClaim = { 134 | metadata: { 135 | name: volume.name, 136 | }, 137 | spec: { 138 | accessModes: ['ReadWriteOnce'], 139 | resources: { 140 | requests: { 141 | storage: '1Gi', 142 | }, 143 | }, 144 | }, 145 | }; 146 | if (config.storageClassName) { 147 | pvc.spec!.storageClassName = config.storageClassName; 148 | } 149 | volumeClaimTemplates.push(pvc); 150 | } 151 | } 152 | } 153 | 154 | workflow.spec.volumeClaimTemplates = volumeClaimTemplates; 155 | } 156 | 157 | return workflow; 158 | } 159 | 160 | function getIdentity(kind: string) { 161 | switch (kind) { 162 | case BenchmarkStageKind.CANDIDATE: 163 | return 'candidate'; 164 | case BenchmarkStageKind.CONTAINER: 165 | return 'benchmark'; 166 | default: 167 | return 'none'; 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /packages/worker/src/configuration.ts: -------------------------------------------------------------------------------- 1 | import * as env from 'env-var'; 2 | 3 | export interface ArgoWorkerConfiguration { 4 | successfulRunGCSeconds: number; 5 | storageClassName?: string; 6 | } 7 | 8 | export function ParseArgoWorkerConfiguration(): ArgoWorkerConfiguration { 9 | return { 10 | successfulRunGCSeconds: env 11 | .get('SUCCESSFUL_RUN_GC_SECONDS') 12 | .default(300) 13 | .asInt(), 14 | storageClassName: env.get('STORAGE_CLASS_NAME').asString(), 15 | }; 16 | } 17 | -------------------------------------------------------------------------------- /packages/worker/src/index.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import * as k8s from '@kubernetes/client-node'; 3 | import { 4 | PipelineRun, 5 | GetQueue, 6 | InitTelemetry, 7 | ParseQueueConfiguration, 8 | } from '@microsoft/sds'; 9 | import { ArgoWorker } from './argoWorker'; 10 | import { defaultClient as telemetryClient } from 'applicationinsights'; 11 | import { Events } from './telemetry'; 12 | import { ParseArgoWorkerConfiguration } from './configuration'; 13 | 14 | async function main() { 15 | InitTelemetry(); 16 | 17 | const queueConfig = await ParseQueueConfiguration(); 18 | const queue = GetQueue(queueConfig); 19 | const workerConfig = ParseArgoWorkerConfiguration(); 20 | 21 | const kc = new k8s.KubeConfig(); 22 | kc.loadFromCluster(); 23 | 24 | // todo: capture SIGTERM & graceful shutdown 25 | const worker = new ArgoWorker(queue, kc, workerConfig); 26 | worker.start(); 27 | console.log('Worker started'); 28 | 29 | telemetryClient.trackEvent({ 30 | name: Events.WorkerStarted, 31 | }); 32 | } 33 | 34 | main().then(); 35 | -------------------------------------------------------------------------------- /packages/worker/src/shims.d.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-empty-interface */ 2 | 3 | // superagent 4 | // https://github.com/DefinitelyTyped/DefinitelyTyped/issues/12044 5 | declare interface XMLHttpRequest {} 6 | -------------------------------------------------------------------------------- /packages/worker/src/telemetry.ts: -------------------------------------------------------------------------------- 1 | export enum Events { 2 | WorkerStarted = 'workerStarted', 3 | } 4 | -------------------------------------------------------------------------------- /packages/worker/test/.mocharc.yml: -------------------------------------------------------------------------------- 1 | recursive: true 2 | require: 'source-map-support/register' 3 | spec: '**/*.ts' 4 | -------------------------------------------------------------------------------- /packages/worker/test/index.ts: -------------------------------------------------------------------------------- 1 | require('mocha'); 2 | import * as appInsights from 'applicationinsights'; 3 | 4 | before(() => { 5 | appInsights.setup('00000000-0000-0000-0000-000000000000'); 6 | appInsights.defaultClient.config.disableAppInsights = true; 7 | }); 8 | -------------------------------------------------------------------------------- /packages/worker/tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.build.json", 3 | "compilerOptions": { 4 | "outDir": "./dist", 5 | }, 6 | "include": [ 7 | "src/**/*" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /packages/worker/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json" 3 | } 4 | -------------------------------------------------------------------------------- /sample-data/benchmark1.yaml: -------------------------------------------------------------------------------- 1 | name: benchmark1 2 | author: author1 3 | apiVersion: v1alpha1 4 | stages: 5 | - name: candidate 6 | kind: candidate 7 | volumes: 8 | - name: training 9 | path: /input 10 | readonly: true 11 | - name: scoring 12 | kind: container 13 | image: benchmark-image 14 | volumes: 15 | - name: reference 16 | path: /reference 17 | readonly: true 18 | -------------------------------------------------------------------------------- /sample-data/candidate1.yaml: -------------------------------------------------------------------------------- 1 | name: candidate1 2 | author: author1 3 | apiVersion: v1alpha1 4 | benchmark: benchmark1 5 | image: candidate1-image 6 | -------------------------------------------------------------------------------- /sample-data/suite1.yaml: -------------------------------------------------------------------------------- 1 | name: suite1 2 | author: author1 3 | apiVersion: v1alpha1 4 | benchmark: benchmark1 5 | volumes: 6 | - name: training 7 | type: AzureBlob 8 | target: https://sample.blob.core.windows.net/training 9 | - name: reference 10 | type: AzureBlob 11 | target: https://sample.blob.core.windows.net/reference 12 | -------------------------------------------------------------------------------- /samples/catdetection/README.md: -------------------------------------------------------------------------------- 1 | # Cat Detection 2 | 3 | This sample demonstrates a three stage pipeline with the following behavior 4 | 5 | 1. Preparation 6 | * Downloads a cat picture from the Internet 7 | * Stores the image on an ephemeral volume 8 | 2. Candidate 9 | * Reads the source image 10 | * Makes a call to Azure Cognitive Services to get its category 11 | * Translates the raw API output into the desired format 12 | * Saves the result to an ephemeral volume 13 | 3. Evaluation 14 | * Reads the Candidate's results 15 | * Determines whether or not a cat was correctly identified 16 | 17 | ## Configuration 18 | 19 | Update the environment variables in `candidate.yml` to point to an Azure Cognitive Services account 20 | 21 | ```yaml 22 | env: 23 | API_KEY: 24 | SERVICE_ENDPOINT: https://.cognitiveservices.azure.com/ 25 | ``` 26 | 27 | ## Usage 28 | 29 | ```shell 30 | # Set up the Benchmark and Suite 31 | npm run cli create benchmark ./samples/catdetection/benchmark.yml 32 | npm run cli create suite ./samples/catdetection/suite.yml 33 | 34 | # Create a Candidate 35 | npm run cli create candidate ./samples/catdetection/candidate.yml 36 | 37 | # Run the Candidate 38 | npm run cli run cognitiveservices images 39 | 40 | ``` 41 | 42 | ## Explanation 43 | 44 | This sample demonstrates several aspects of SDS for Benchmarks/Candidates/Runs 45 | 46 | Downloading a cat photo is analagous to executing a query against an API or data lake to load data - which may or may not be preprocessed for optimal usage by the Candidate. Volumes are used to durably pass data between containers - but not used as long-term storage. 47 | 48 | The Candidate reads and has access only to the data that it has been provided. The expectation is that the Candidate adheres to the contract set forth by the Benchmark design. It also calls out to an external API for some, but not all of its ML-related work. 49 | 50 | And finally, the Evaluation container is aware of ground-truth, and scores the Candidate's results based on a criteria that has meaning to the business, versus other methods that may be purely mathematical. 51 | -------------------------------------------------------------------------------- /samples/catdetection/benchmark-eval/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | 3 | VOLUME /results 4 | 5 | COPY ./start.sh /start.sh 6 | ENTRYPOINT ["/bin/sh", "/start.sh"] 7 | -------------------------------------------------------------------------------- /samples/catdetection/benchmark-eval/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | # Verify that the candidate predicted our expected animal 5 | PREDICTION=$(cat /results/prediction.txt) 6 | 7 | SCORE=0 8 | if [ "$PREDICTION" == "$EXPECTED_ANIMAL" ]; then 9 | echo "Perfect match for run!" 10 | SCORE=1 11 | else 12 | echo "Prediction did not match expected value: $EXPECTED" 13 | fi 14 | 15 | echo "TODO: POST to $2: {run: $1, score: $SCORE}" 16 | -------------------------------------------------------------------------------- /samples/catdetection/benchmark-prep/Dockerfile: -------------------------------------------------------------------------------- 1 | # Pulls data from an external API to prepare it for processing 2 | FROM curlimages/curl:7.71.1 3 | 4 | # Bypass k8s volume ownership issues 5 | USER root 6 | RUN mkdir /out 7 | VOLUME /out 8 | 9 | COPY ./start.sh /start.sh 10 | ENTRYPOINT ["/bin/sh", "/start.sh"] 11 | -------------------------------------------------------------------------------- /samples/catdetection/benchmark-prep/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | # Download a picture 5 | curl -L $IMAGE_URL -o /out/image.jpg 6 | -------------------------------------------------------------------------------- /samples/catdetection/benchmark.yml: -------------------------------------------------------------------------------- 1 | name: catdetection 2 | author: acanthamoeba 3 | apiVersion: v1alpha1 4 | stages: 5 | - name: prep 6 | kind: container 7 | image: acanthamoeba/sds-prep 8 | env: 9 | IMAGE_URL: '{suite.properties.imageUrl}' 10 | BENCHMARK_AUTHOR: '{benchmark.author}' 11 | volumes: 12 | - name: images 13 | path: /out 14 | readonly: false 15 | 16 | - name: candidate 17 | kind: candidate 18 | volumes: 19 | - name: images 20 | path: /in 21 | readonly: true 22 | - name: predictions 23 | path: /out 24 | readonly: false 25 | 26 | - name: eval 27 | kind: container 28 | image: acanthamoeba/sds-eval 29 | env: 30 | EXPECTED_ANIMAL: cat 31 | CANDIDATE_IMAGE: '{candidate.image}' 32 | cmd: ['{run.name}', '{laboratoryEndpoint}'] 33 | volumes: 34 | - name: predictions 35 | path: /results 36 | readonly: true 37 | - name: scores 38 | path: /scores 39 | readonly: false 40 | 41 | -------------------------------------------------------------------------------- /samples/catdetection/candidate.yml: -------------------------------------------------------------------------------- 1 | name: cognitiveservices 2 | author: acanthamoeba 3 | apiVersion: v1alpha1 4 | benchmark: catdetection 5 | image: acanthamoeba/sds-candidate 6 | # TODO: move to secrets 7 | env: 8 | API_KEY: 9 | SERVICE_ENDPOINT: https://.cognitiveservices.azure.com/ 10 | -------------------------------------------------------------------------------- /samples/catdetection/candidate/Dockerfile: -------------------------------------------------------------------------------- 1 | # Performs image detection by making a call to Azure Cognitive Services 2 | FROM curlimages/curl:7.71.1 3 | 4 | # Bypass k8s volume ownership issues 5 | USER root 6 | RUN mkdir /out \ 7 | && apk add jq 8 | VOLUME /out 9 | 10 | ENV API_KEY='' 11 | ENV SERVICE_NAME='' 12 | 13 | COPY ./start.sh /start.sh 14 | CMD ["/bin/sh", "/start.sh"] 15 | -------------------------------------------------------------------------------- /samples/catdetection/candidate/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | if [[ -z "$API_KEY" || -z "$SERVICE_ENDPOINT" ]]; then 5 | echo "You must set the following environment variables: [API_KEY, SERVICE_ENDPOINT]" 6 | exit 1 7 | fi 8 | 9 | if [ ! -f "/in/image.jpg" ]; then 10 | echo "File not found: There must be an image located at: /in/image.jpg" 11 | exit 1 12 | fi 13 | 14 | # Capture the raw response 15 | curl -X POST \ 16 | -H "Ocp-Apim-Subscription-Key: $API_KEY" \ 17 | -H 'Content-Type: application/octet-stream' \ 18 | --data-binary @/in/image.jpg \ 19 | "${SERVICE_ENDPOINT}vision/v3.0/analyze?visualFeatures=Categories" > /out/raw.json 20 | 21 | # Parse the generated output to produce a description in the proper format 22 | cat /out/raw.json | jq -r '.categories[0].name | sub("animal_"; "")' > /out/prediction.txt 23 | -------------------------------------------------------------------------------- /samples/catdetection/suite.yml: -------------------------------------------------------------------------------- 1 | name: images 2 | author: acanthamoeba 3 | apiVersion: v1alpha1 4 | benchmark: catdetection 5 | properties: 6 | imageUrl: 'https://placekitten.com/1024/768' 7 | volumes: 8 | - name: images 9 | type: ephemeral 10 | - name: predictions 11 | type: ephemeral 12 | - name: scores 13 | type: ephemeral 14 | -------------------------------------------------------------------------------- /scripts/README.md: -------------------------------------------------------------------------------- 1 | # scripts 2 | 3 | Scripts used for the development of secure-data-sandbox. These aren't used or deployed at runtime. 4 | 5 | ## Usage 6 | 7 | * In the repo root, copy `.env.template` to `.env` and fill in your values 8 | * `npm run compile` 9 | * Run a script via `node -r dotenv/config build/scripts/.js` 10 | -------------------------------------------------------------------------------- /scripts/demo-catdetection.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | npm run cli create benchmark ./samples/catdetection/benchmark.yml 5 | npm run cli create suite ./samples/catdetection/suite.yml 6 | npm run cli create candidate ./samples/catdetection/candidate.yml 7 | 8 | npm run cli run cognitiveservices images 9 | -------------------------------------------------------------------------------- /scripts/demo-sample-data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | npm run cli create benchmark sample-data/benchmark1.yaml 5 | npm run cli create suite sample-data/suite1.yaml 6 | npm run cli create candidate sample-data/candidate1.yaml 7 | 8 | npm run cli run candidate1 suite1 9 | -------------------------------------------------------------------------------- /scripts/dev-npm-audit-fix.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | # Remove monorepo package references 5 | sed -i '/"@microsoft\/sds": "*"/d' packages/*/package.json 6 | 7 | # Update the monorepo root 8 | npm install --ignore-scripts 9 | npm audit fix 10 | 11 | # Update each package 12 | lerna exec -- npm install --ignore-scripts 13 | lerna exec npm audit fix 14 | 15 | # Readd monorepo package references 16 | sed -i 's/\("dependencies": {\)/\1\n "@microsoft\/sds": "*",/' packages/cli/package.json 17 | sed -i 's/\("dependencies": {\)/\1\n "@microsoft\/sds": "*",/' packages/laboratory/package.json 18 | sed -i 's/\("dependencies": {\)/\1\n "@microsoft\/sds": "*",/' packages/worker/package.json 19 | 20 | # Validate 21 | npm install 22 | lerna exec npm rebuild || true 23 | npm run test 24 | -------------------------------------------------------------------------------- /scripts/dev-npm-update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | # Remove monorepo package references 5 | sed -i '/"@microsoft\/sds": "*"/d' packages/*/package.json 6 | 7 | # Update the monorepo root 8 | npm install --ignore-scripts 9 | npm update 10 | 11 | # Update each package 12 | lerna exec -- npm install --ignore-scripts 13 | lerna exec npm update 14 | 15 | # Readd monorepo package references 16 | sed -i 's/\("dependencies": {\)/\1\n "@microsoft\/sds": "*",/' packages/cli/package.json 17 | sed -i 's/\("dependencies": {\)/\1\n "@microsoft\/sds": "*",/' packages/laboratory/package.json 18 | sed -i 's/\("dependencies": {\)/\1\n "@microsoft\/sds": "*",/' packages/worker/package.json 19 | 20 | # Validate 21 | npm install 22 | lerna exec npm rebuild || true 23 | npm run test 24 | -------------------------------------------------------------------------------- /scripts/dev-vm-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | # Useful tools for developing / debugging on a remote terminal connected to the detonation chamber 5 | 6 | # Azure CLI 7 | curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash 8 | az --version 9 | 10 | # kubectl 11 | sudo az aks install-cli 12 | 13 | # helm 14 | sudo snap install helm --classic 15 | 16 | # Docker 17 | sudo addgroup --system docker 18 | sudo adduser $USER docker 19 | newgrp docker 20 | 21 | sudo snap install docker 22 | sudo snap start docker 23 | 24 | # Node 25 | sudo snap install node --classic --channel=12 26 | 27 | # octant 28 | curl -LO https://github.com/vmware-tanzu/octant/releases/download/v0.15.0/octant_0.15.0_Linux-64bit.tar.gz 29 | tar xvf octant_0.15.0_Linux-64bit.tar.gz 30 | sudo mv ./octant_0.15.0_Linux-64bit/octant /usr/local/bin 31 | -------------------------------------------------------------------------------- /scripts/dev-worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | export DOCKER_BUILDKIT=1 5 | 6 | IMAGE=acanthamoeba/sds-worker:dev 7 | 8 | # Build and push the worker 9 | docker build -t $IMAGE --target worker . 10 | docker push $IMAGE 11 | 12 | # Scale to 0 replicas 13 | kubectl scale -n worker --replicas 0 deploy/worker 14 | 15 | # Scale back to 1 16 | kubectl scale -n worker --replicas 1 deploy/worker 17 | 18 | # To see details on run execution, run the following, then browse to http://localhost:2746 19 | # kubectl port-forward -n argo service/argo-server 2746 20 | -------------------------------------------------------------------------------- /scripts/laboratory.rest: -------------------------------------------------------------------------------- 1 | # List benchmarks 2 | GET {{laboratoryHost}}/benchmarks 3 | 4 | ### 5 | 6 | # Create a benchmark 7 | PUT {{laboratoryHost}}/benchmarks/benchmark1 8 | Content-Type: application/json 9 | 10 | { 11 | "name": "benchmark1", 12 | "author": "admin", 13 | "apiVersion": "v1alpha1", 14 | "stages": [ 15 | { 16 | "name": "candidate", 17 | "kind": "candidate", 18 | "volumes": [ 19 | { 20 | "volume": "training", 21 | "path": "/input" 22 | } 23 | ] 24 | }, 25 | { 26 | "name": "scoring", 27 | "kind": "container", 28 | "image": "{{laboratoryRegistry}}/alpine", 29 | "volumes": [ 30 | { 31 | "volume": "reference", 32 | "path": "/reference", 33 | "readonly": true 34 | } 35 | ] 36 | } 37 | ] 38 | } 39 | 40 | ### 41 | 42 | # Get one benchmark 43 | GET {{laboratoryHost}}/benchmarks/benchmark1 44 | 45 | ### 46 | 47 | # Get all suites 48 | GET {{laboratoryHost}}/suites 49 | 50 | ### 51 | 52 | # Create a suite 53 | PUT {{laboratoryHost}}/suites/suite1 54 | Content-Type: application/json 55 | 56 | { 57 | "name": "suite1", 58 | "author": "admin", 59 | "apiVersion": "v1alpha1", 60 | "benchmark": "benchmark1", 61 | "volumes": [ 62 | { 63 | "name": "training", 64 | "type": "AzureBlob", 65 | "target": "https://sample.blob.core.windows.net/training" 66 | }, 67 | { 68 | "name": "reference", 69 | "type": "AzureBlob", 70 | "target": "https://sample.blob.core.windows.net/training" 71 | } 72 | ] 73 | } 74 | 75 | ### 76 | 77 | # Get one suite 78 | GET {{laboratoryHost}}/suites/suite1 79 | 80 | ### 81 | 82 | # Get all candidates 83 | GET {{laboratoryHost}}/candidates 84 | 85 | ### 86 | 87 | # Create a candidate 88 | PUT {{laboratoryHost}}/candidates/candidate1 89 | Content-Type: application/json 90 | 91 | { 92 | "name": "candidate1", 93 | "author": "admin", 94 | "apiVersion": "v1alpha1", 95 | "benchmark": "benchmark1", 96 | "image": "{{laboratoryRegistry}}/alpine" 97 | } 98 | 99 | ### 100 | 101 | # Get one candidate 102 | GET {{laboratoryHost}}/candidates/candidate1 103 | 104 | ### 105 | 106 | # Get all runs 107 | GET {{laboratoryHost}}/runs 108 | 109 | ### 110 | 111 | # @name createRun 112 | # Create a run 113 | POST {{laboratoryHost}}/runs 114 | Content-Type: application/json 115 | 116 | { 117 | "candidate": "candidate1", 118 | "suite": "suite1" 119 | } 120 | 121 | ### 122 | 123 | @runName = {{createRun.response.body.$.name}} 124 | 125 | # Get one run 126 | GET {{laboratoryHost}}/runs/{{runName}} 127 | 128 | ### 129 | 130 | # Update run status 131 | PATCH {{laboratoryHost}}/runs/{{runName}} 132 | Content-Type: application/json 133 | 134 | { 135 | "status": "running" 136 | } 137 | 138 | ### 139 | 140 | # Report run results 141 | POST {{laboratoryHost}}/runs/{{runName}}/results 142 | Content-Type: application/json 143 | 144 | { 145 | "measures": { 146 | "passed": 5, 147 | "failed": 6 148 | } 149 | } 150 | 151 | ### 152 | 153 | # Get run results 154 | GET {{laboratoryHost}}/runs?benchmark=benchmark1&suite=suite1 155 | -------------------------------------------------------------------------------- /tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./node_modules/gts/tsconfig-google.json", 3 | "exclude": [ 4 | "node_modules", 5 | "dist" 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.build.json", 3 | "compilerOptions": { 4 | "baseUrl": ".", 5 | "paths": { 6 | "@microsoft/sds": ["packages/sds/src"] 7 | } 8 | } 9 | } 10 | --------------------------------------------------------------------------------