├── utils.js ├── result.js ├── config.json ├── auth.js ├── .npmrc ├── transcript.js ├── package.json ├── context.js ├── .gitignore ├── LICENSE ├── logger.js ├── resultsManager.js ├── SECURITY.md ├── suite.js ├── suiteData.js ├── app.js ├── testData.js ├── README.md ├── transcriptTransformationScript.js ├── .pipelines └── [HealthBot]_[FunctionalTestService]_[Master].yml ├── directlineclient.js └── test.js /utils.js: -------------------------------------------------------------------------------- 1 | 2 | var stringifySpace = 4; 3 | 4 | var Utils = function () { 5 | } 6 | 7 | Utils.prototype.stringify = function(obj, space) { 8 | space = space || stringifySpace; 9 | return JSON.stringify(obj, null, space); 10 | } 11 | 12 | module.exports = new Utils(); 13 | -------------------------------------------------------------------------------- /result.js: -------------------------------------------------------------------------------- 1 | class Result { 2 | constructor({ success, message, code, conversationId }) { 3 | this.success = success; 4 | this.message = message; 5 | this.code = code; 6 | this.conversationId = conversationId; 7 | } 8 | } 9 | 10 | module.exports = Result; -------------------------------------------------------------------------------- /config.json: -------------------------------------------------------------------------------- 1 | { 2 | "defaults": { 3 | "timeoutMilliseconds": 5000, 4 | "testSuiteResultsRetentionSeconds": 3600, 5 | "defaultBatchSize": 3, 6 | "failureTolerance": 1, 7 | "defaultRoleName": "botFunctionalTestsService" 8 | }, 9 | "testsDir": "tests" 10 | } 11 | -------------------------------------------------------------------------------- /auth.js: -------------------------------------------------------------------------------- 1 | function auth(token) { 2 | return function (req, res, next) { 3 | const currToken = req.query?.token || req.body?.token; 4 | 5 | if (currToken === token) { 6 | next(); 7 | return; 8 | } 9 | 10 | res.setHeader("content-type", "text/plain"); 11 | res.status(401).send("Unauthorized."); 12 | }; 13 | } 14 | 15 | module.exports = auth; 16 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | # Auto generated file from Gardener Plugin CentralFeedServiceAdoptionPlugin 2 | @coherence-design-system:registry=https://pkgs.dev.azure.com/mshealthil/_packaging/HealthILFeed/npm/registry/ 3 | @ms:registry=https://pkgs.dev.azure.com/mshealthil/_packaging/HealthILFeed/npm/registry/ 4 | @m365-admin:registry=https://pkgs.dev.azure.com/mshealthil/_packaging/HealthILFeed/npm/registry/ 5 | @healthil:registry=https://pkgs.dev.azure.com/mshealthil/_packaging/MSHealthILFeed/npm/registry/ 6 | 7 | registry=https://pkgs.dev.azure.com/mshealthil/HealthIL/_packaging/healthil_PublicPackages/npm/registry/ 8 | 9 | always-auth=true 10 | -------------------------------------------------------------------------------- /transcript.js: -------------------------------------------------------------------------------- 1 | var _ = require("underscore"); 2 | 3 | class Transcript { 4 | static getMessages(transcript) { 5 | var messages = (transcript && _.isArray(transcript)) ? _.filter(transcript, function (obj) { 6 | return obj.type === "message" || obj.type === "endOfConversation"; 7 | } 8 | ) : []; 9 | messages = _.map(messages, 10 | function(message) { 11 | return _.pick(message, ["type", "text", "attachments", "speak", "locale", "textFormat", "from", "recipient", "value"]); 12 | }); 13 | return messages; 14 | } 15 | } 16 | 17 | module.exports = Transcript; 18 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "healthbottestservice", 3 | "version": "1.0.0", 4 | "description": "Bot functional testing framework", 5 | "main": "app.js", 6 | "scripts": { 7 | "build": "echo no build defined", 8 | "test": "echo \"Error: no test specified\" && exit 1" 9 | }, 10 | "keywords": [ 11 | "healthbot", 12 | "testing", 13 | "functional" 14 | ], 15 | "author": "Arie Schwartzman + Gil Shacham", 16 | "license": "MIT", 17 | "dependencies": { 18 | "applicationinsights": "^2.5.1", 19 | "axios": "^1.6.8", 20 | "body-parser": "^1.20.2", 21 | "chai": "^4.3.4", 22 | "deep-object-diff": "^1.0.4", 23 | "dotenv": "^8.6.0", 24 | "express": "^4.19.2", 25 | "sanitize-filename": "^1.6.3", 26 | "underscore": "^1.13.1" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /context.js: -------------------------------------------------------------------------------- 1 | const logger = require("./logger.js"); 2 | 3 | class Context { 4 | 5 | constructor(req, response) { 6 | this.request = req; 7 | this.response = response; 8 | } 9 | 10 | done(status, body) { 11 | this.response.setHeader("content-type", "application/json"); 12 | this.response.status(status).send(body); 13 | } 14 | 15 | success({ message, conversationId }) { 16 | logger.log("success: " + message); 17 | this.done(200, this.request.query.includeConversationId ? { message, conversationId } : message); 18 | } 19 | 20 | failure(code, reason) { 21 | logger.log("failure: " + JSON.stringify(reason)); 22 | this.done(code, reason); 23 | } 24 | } 25 | 26 | module.exports = Context; 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (https://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # TypeScript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # next.js build output 61 | .next 62 | 63 | #WebStorm 64 | .idea/ 65 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /logger.js: -------------------------------------------------------------------------------- 1 | const { format } = require("util"); 2 | const { TelemetryClient } = require("applicationinsights"); 3 | const config = require("./config.json"); 4 | 5 | const keyEnvVarName = "ApplicationInsightsInstrumentationKey"; 6 | 7 | function telemetryClientLogger() { 8 | const telemetryClient = new TelemetryClient(process.env[keyEnvVarName]); 9 | telemetryClient.context.tags["ai.cloud.role"] = process.env["roleName"] || config.defaults.defaultRoleName; 10 | 11 | return { 12 | log(...args) { 13 | telemetryClient.trackTrace({ message: format(...args) }); 14 | telemetryClient.flush(); 15 | console.log(...args); 16 | }, 17 | event(name, properties) { 18 | telemetryClient.trackEvent({ name, properties }); 19 | } 20 | }; 21 | } 22 | 23 | function consoleLogger() { 24 | return { 25 | log(...args) { 26 | console.log(...args); 27 | }, 28 | event(name, properties) {} 29 | }; 30 | } 31 | 32 | function censorSecrets(obj, paths) { 33 | const copy = structuredClone(obj); 34 | if (!copy || !paths) { 35 | return copy; 36 | } 37 | 38 | for (const path of paths) { 39 | const pathParts = path.split('.'); 40 | const lastPathPart = pathParts.pop(); 41 | let current = copy; 42 | 43 | // Traverse to the parent object 44 | for (const pathPart of pathParts) { 45 | current = current?.[pathPart]; 46 | if (!current) break; 47 | } 48 | 49 | // Censor the final property if it exists 50 | if (current?.[lastPathPart] !== undefined) { 51 | current[lastPathPart] = '****'; 52 | } 53 | } 54 | return copy; 55 | } 56 | 57 | const logger = process.env[keyEnvVarName] ? telemetryClientLogger() : consoleLogger(); 58 | logger.censorSecrets = censorSecrets; 59 | 60 | module.exports = logger; -------------------------------------------------------------------------------- /resultsManager.js: -------------------------------------------------------------------------------- 1 | const crypto = require('crypto'); 2 | 3 | let activeRunIds = new Set(); // runId is an identifier for a suite run. 4 | let runIdToResults = {}; // runId --> [arrayOfResults, verdict] 5 | 6 | /** 7 | * Returns the set of activeRunIds 8 | * @return activeRunIds 9 | */ 10 | function getActiveRunIds() { 11 | return activeRunIds; 12 | } 13 | 14 | /** 15 | * Returns a random RunId that is currently not in use. This function also updates the set of active run ids. 16 | * @return string fresh Id 17 | */ 18 | function getFreshRunId() { 19 | let res = crypto.randomBytes(8).toString('hex'); 20 | while (activeRunIds.has(res)) { // Ensures that the runId is currently unique. 21 | res = crypto.randomBytes(8).toString('hex'); 22 | } 23 | activeRunIds.add(res); 24 | return res; 25 | } 26 | 27 | /** 28 | * Updates the results of a suite, given test id (runId), array of test results and a verdict ("success", "failure"). 29 | * @param runId 30 | * @param testResults 31 | * @param errorMessage 32 | * @param verdict 33 | * @return void 34 | * 35 | */ 36 | function updateSuiteResults (runId, testResults, errorMessage, verdict) { 37 | if (activeRunIds.has(runId)) { 38 | runIdToResults[runId] = {}; 39 | runIdToResults[runId]["results"] = testResults; 40 | runIdToResults[runId]["errorMessage"] = errorMessage; 41 | runIdToResults[runId]["verdict"] = verdict; 42 | } 43 | } 44 | 45 | /** 46 | * Deletes results of a suite given runId from resultsManagerInstance.runIds and from resultsManagerInstance.runIdToResults. 47 | * @param runId 48 | * @return void 49 | */ 50 | function deleteSuiteResult(runId) { 51 | if (activeRunIds.has(runId)) { 52 | activeRunIds.delete(runId); 53 | delete runIdToResults[runId]; 54 | } 55 | } 56 | 57 | /** 58 | * Returns the test results of a given runId. 59 | * @param runId 60 | * @return The array representing the tests results of the given runId. If test results is not ready, null is returned. 61 | */ 62 | function getSuiteResults(runId) { 63 | if (runIdToResults.hasOwnProperty(runId)) { // If test results are ready 64 | return runIdToResults[runId]; // Return them. 65 | } 66 | else { 67 | return null; // Else, null is returned. 68 | } 69 | } 70 | 71 | module.exports = {getActiveRunIds, getFreshRunId, updateSuiteResults, deleteSuiteResult, getSuiteResults}; 72 | 73 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /suite.js: -------------------------------------------------------------------------------- 1 | var _ = require("underscore"); 2 | 3 | var utils = require("./utils"); 4 | var config = require("./config.json"); 5 | const sleep = require("util").promisify(setTimeout); 6 | 7 | var Test = require("./test"); 8 | var Result = require("./result"); 9 | var ResultsManager = require("./resultsManager"); 10 | const logger = require("./logger"); 11 | 12 | class Suite { 13 | 14 | constructor(context, runId, suiteData) { 15 | this.results = []; 16 | this.context = context; 17 | this.runId = runId; 18 | this.suiteData = suiteData; 19 | } 20 | 21 | async runTest(testData) { 22 | try { 23 | let tolerance = parseInt(process.env["failureTolerance"]) ? parseInt(process.env["failureTolerance"]) : config.defaults.failureTolerance; 24 | let result; 25 | while (!(result && result.success === true) && tolerance > 0) { 26 | result = await Test.perform(this.context, testData); 27 | tolerance--; 28 | } 29 | return result; 30 | } 31 | catch (err) { 32 | return new Result({ success: false, message: err.message, code: 400 }); 33 | } 34 | } 35 | 36 | summarizeTestsResults() { 37 | const success = _.every(this.results, (result) => result && result.success); 38 | const details = this.context.request.query.includeConversationId || this.context.request.body.includeConversationId ? 39 | this.results : 40 | _.pluck(this.results, "message"); 41 | 42 | if (success) { 43 | logger.event("TestSuiteSucceeded", { suite: this.suiteData.name, details }); 44 | ResultsManager.updateSuiteResults(this.runId, details, "", "success"); 45 | } 46 | else { 47 | logger.event("TestSuiteFailed", { suite: this.suiteData.name, details }); 48 | ResultsManager.updateSuiteResults(this.runId, details, "", "failure"); 49 | } 50 | } 51 | 52 | async run() { 53 | logger.log("Suite.run started"); 54 | logger.log("suiteData: " + utils.stringify(this.suiteData)); 55 | // We will divide the tests into batches. Batch size is determined by env var "BatchSize" (default 3). 56 | const batchSize = parseInt(process.env["BatchSize"]) ? parseInt(process.env["BatchSize"]) : config.defaults.defaultBatchSize; 57 | let testPromises = []; 58 | try { 59 | for (let i=0; i path.extname(fileName) === '.transcript') 47 | .map(fileName => ({path: path.join(testsDir, fileName)})); 48 | } else { 49 | throw new Error("Request must contain a 'tests' array or directory name containing *.transcript files."); 50 | } 51 | } 52 | suiteData = new SuiteData(suiteDataObj, request.query); 53 | break; 54 | } 55 | if (suiteData) { 56 | await suiteData.init(); 57 | } 58 | return suiteData; 59 | } 60 | 61 | static async getSuiteData(query) { 62 | var suiteURL = query.url; 63 | if (suiteURL) { 64 | const { data } = await axios.get(suiteURL); // CodeQL [SM04580] this is a closed api that is only accessible to an internal testing service, so the ssrf risk is mitigated 65 | return new SuiteData(data, query); 66 | } 67 | else { 68 | throw new Error("A 'url' parameter should be included on the query string."); 69 | } 70 | } 71 | 72 | } 73 | 74 | async function createTestData(tests, defaults) { 75 | async function createData(test, index) { 76 | return new Promise(async function(resolve, reject) { 77 | try { 78 | resolve(await TestData.fromObject(test, defaults)); 79 | } 80 | catch (err) { 81 | reject(new Error(`tests[${index}]: ${err.message}`)); 82 | } 83 | }); 84 | } 85 | return Promise.all(tests.map(createData)); 86 | } 87 | 88 | module.exports = SuiteData; 89 | -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | require("dotenv").config(); 2 | 3 | const auth = require("./auth.js"); 4 | var Context = require("./context.js"); 5 | var TestData = require("./testData.js"); 6 | var Test = require("./test"); 7 | var SuiteData = require("./suiteData.js"); 8 | var Suite = require("./suite"); 9 | var ResultsManager = require("./resultsManager"); 10 | var config = require("./config.json"); 11 | 12 | const express = require("express"); 13 | const bodyParser = require("body-parser"); 14 | const logger = require("./logger.js"); 15 | 16 | logger.log("Server initialized"); 17 | const server = express(); 18 | 19 | server.use(bodyParser.json({limit: '5mb'})); 20 | 21 | const requiredAuthToken = process.env.REQUIRED_AUTH_TOKEN; 22 | 23 | if (requiredAuthToken) { 24 | server.use(auth(requiredAuthToken)); 25 | } 26 | 27 | server.get("/test", handleRunTest); 28 | server.post("/test", handleRunTest); 29 | server.get("/suite", handleRunSuite); 30 | server.post("/suite", handleRunSuite); 31 | server.get("/getResults/:runId", handleGetTestResults); 32 | 33 | const port = process.env.PORT || 3000; 34 | server.listen(port, function () { 35 | logger.log(`Express server listening on port ${port}`); 36 | }); 37 | 38 | async function handleRunTest(request, response, next) { 39 | const context = new Context(request, response); 40 | logger.log(`processing a test ${request.method} request.`); 41 | 42 | try { 43 | const testData = await TestData.fromRequest(request); 44 | Test.run(context, testData); 45 | } 46 | catch (err) { 47 | context.failure(400, err.message); 48 | } 49 | } 50 | 51 | async function handleRunSuite(request, response, next) { 52 | const context = new Context(request, response); 53 | logger.log(`processing a suite ${request.method} request.`); 54 | const runId = ResultsManager.getFreshRunId(); 55 | logger.log("Started suite run with runIn " + runId); 56 | // Get the suite data from the request. 57 | try { 58 | var suiteData = await SuiteData.fromRequest(request); 59 | logger.log("Successfully got all tests from the request for runId " + runId); 60 | } 61 | catch (err){ 62 | response.setHeader("content-type", "application/json"); 63 | response.status(400).send({results: [], errorMessage:"Could not get tests data from request", verdict:"error"}); 64 | ResultsManager.deleteSuiteResult(runId); 65 | logger.log("Could not get tests data from request for runId " + runId); 66 | logger.log(err); 67 | return; 68 | } 69 | // Send a response with status code 202 and location header based on runId, and start the tests. 70 | response.setHeader("content-type", "application/json"); 71 | response.setHeader("Location", "http://" + request.headers.host + "/getResults/" + runId); 72 | response.status(202).send("Tests are running."); 73 | let testSuite = new Suite(context, runId, suiteData); 74 | try { 75 | await testSuite.run(); 76 | logger.log("Finished suite run with runId " + runId); 77 | setTimeout(() => { 78 | ResultsManager.deleteSuiteResult(runId); 79 | logger.log("Deleted suite results for runId " + runId); 80 | }, config.defaults.testSuiteResultsRetentionSeconds*1000); // Delete suite results data after a constant time after tests end. 81 | } 82 | catch (err) { 83 | ResultsManager.updateSuiteResults(runId, [], "Error while running test suite", "error"); 84 | logger.log("Error occurred during suite run with runIn " + runId); 85 | } 86 | } 87 | 88 | async function handleGetTestResults(request, response, next) { 89 | const runId = request.params.runId; 90 | const activeRunIds = ResultsManager.getActiveRunIds(); 91 | if (!activeRunIds.has(runId)) { // If runId doesn't exist (either deleted or never existed) 92 | response.setHeader("content-type", "application/json"); 93 | response.status(404).send({results: [], errorMessage:"RunId does not exist.", verdict:"error"}); 94 | return; 95 | } 96 | // Else, runId exists. 97 | const resultsObject = ResultsManager.getSuiteResults(runId); 98 | if (!resultsObject) { // If results are not ready 99 | response.setHeader("content-type", "application/json"); 100 | response.setHeader("Location", "http://" + request.headers.host + "/getResults/" + runId); 101 | response.setHeader("Retry-After", 10); 102 | response.status(202).send("Tests are still running."); 103 | } 104 | else { // Results are ready 105 | response.setHeader("content-type", "application/json"); 106 | if (resultsObject["verdict"] === "success" || resultsObject["verdict"] === "failure") { // If tests finished without errors, send response with status code 200. 107 | response.status(200).send(resultsObject); 108 | } 109 | else if (resultsObject["verdict"] === "error") { // If there was an error while running the tests, send response with status code 500 110 | response.status(500).send(resultsObject); 111 | ResultsManager.deleteSuiteResult(runId); // In case of an error while running test suite, delete suite results once user knows about it. 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /testData.js: -------------------------------------------------------------------------------- 1 | var _ = require("underscore"); 2 | var Transcript = require("./transcript"); 3 | 4 | var config = require("./config.json"); 5 | const fs = require("fs"); 6 | const path = require('path'); 7 | const exists = require('util').promisify(fs.exists); 8 | 9 | const readFile = require('util').promisify(fs.readFile); 10 | 11 | const retry_amount = 3; 12 | const sleep = require('util').promisify(setTimeout); 13 | const axios = require('axios'); 14 | 15 | const executeWithRetries = async (func, ...args) => { 16 | for (let retry_count = 0; retry_count < retry_amount; retry_count++) { 17 | if (retry_count > 0) { 18 | await sleep(retry_count*1000); 19 | } 20 | try { 21 | const content = await func(...args); 22 | return content; 23 | } 24 | catch (err) { 25 | if (retry_count === retry_amount - 1) { 26 | console.error(err); 27 | } 28 | } 29 | } 30 | } 31 | 32 | 33 | class TestData { 34 | 35 | constructor(obj, query) { 36 | this.name = (query && query.name) || (obj && obj.name) || (query?.url?.split('/').pop()) || (query?.path?.split('/').pop()); 37 | this.version = (query && query.version) || (obj && obj.version); 38 | this.timeout = (query && query.timeout) || (obj && obj.timeout) || config.defaults.timeoutMilliseconds; 39 | this.bot = (query && query.bot) || (obj && obj.bot) || process.env["DefaultBot"]; 40 | if (!this.bot) { 41 | throw new Error("Configuration error: No bot name was given as a query parameter nor as a test property and no DefaultBot in application settings."); 42 | } 43 | this.secret = query?.botSecret || obj?.botSecret || this.getSecretFromEnvVar(); 44 | this.customDirectlineDomain = query?.customDirectlineDomain || obj?.customDirectlineDomain; 45 | if (!this.secret) { 46 | throw new Error(`Configuration error: BotSecret is missing for ${this.bot}.`); 47 | } 48 | this.userId = (query && (query.userId || query.userid)) || (obj && (obj.userId || obj.userid)); 49 | this.messages = (obj && obj.messages) || Transcript.getMessages(obj); 50 | if (!(this.messages && Array.isArray(this.messages) && this.messages.length > 0)) { 51 | throw new Error("A test must contain a non-empty 'messages' array or consist of a bot conversation transcript.") 52 | } 53 | } 54 | 55 | getSecretFromEnvVar() { 56 | var extractedSecret = null; 57 | try { 58 | extractedSecret = JSON.parse(process.env['SECRETS'])[this.bot]; 59 | } 60 | catch { 61 | throw new Error("Invalid format of bot secrets JSON"); 62 | } 63 | return extractedSecret; 64 | } 65 | 66 | static inheritedProperties() { 67 | return ["version", "timeout", "bot", "userId", "botSecret"]; 68 | } 69 | 70 | static async fromRequest(request) { 71 | var testData = null; 72 | switch (request.method) { 73 | case "GET": 74 | testData = await this.getTestData(request.query); 75 | break; 76 | case "POST": 77 | testData = new TestData(request.body, request.query); 78 | break; 79 | } 80 | return testData; 81 | } 82 | 83 | static async getTestData(query) { 84 | var testURL = query.url; 85 | if (testURL) { 86 | const { data } = await axios.get(testURL); // CodeQL [SM04580] this is a closed api that is only accessible to an internal testing service, so the ssrf risk is mitigated 87 | return new TestData(data, query); 88 | } else if (query.path) { 89 | const fullTestPath = path.join(config.testsDir, query.path).normalize(); 90 | if (!(await exists(fullTestPath)) || !fullTestPath.startsWith(config.testsDir)) { 91 | throw new Error("Test file invalid or not exists."); 92 | } 93 | const content = await executeWithRetries(readFile, fullTestPath); 94 | return new TestData(JSON.parse(content), query); 95 | } else { 96 | throw new Error("A 'url' or 'path' parameters should be included on the query string."); 97 | } 98 | } 99 | 100 | static async fromObject(obj, defaults) { 101 | var testData = null; 102 | if (obj.hasOwnProperty("url") && obj.url) { 103 | const { data } = await executeWithRetries(axios.get, obj.url); 104 | testData = new TestData(data, {...defaults, ...obj}); 105 | } else if (obj.hasOwnProperty("path") && obj.path) { 106 | const content = await executeWithRetries(readFile, obj.path); 107 | testData = new TestData(JSON.parse(content), {...defaults, ...obj}); 108 | } 109 | else { 110 | testData = new TestData(obj, {}); 111 | } 112 | var testDataProto = Object.getPrototypeOf(testData); 113 | testData = _.extend(_.pick(defaults, this.inheritedProperties()), testData); 114 | Object.setPrototypeOf(testData, testDataProto); 115 | return testData; 116 | } 117 | 118 | } 119 | 120 | module.exports = TestData; 121 | 122 | 123 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Bot Functional Testing Service 2 | 3 | A service to enable functional testing of a [Microsoft Bot Framework](https://dev.botframework.com/) bot. A call to this service programmatically simulates a user’s back-and-forth conversation with a bot, to test whether the bot behaves as expected. 4 | 5 | When calling the service, a _Test_ is given as input. A _Test_ is basically a “recording” of a user’s conversation with a bot. The _Test_ is run against a given bot to check whether the conversation occurs as expected. 6 | 7 | The service exposes a RESTful API for running _Tests_. The HTTP response code of an API call indicates whether the conversation had occurred as expected or not. If not, the response body contains information regarding the _Test_ failure. 8 | 9 | ## Creating a Test 10 | 11 | In order to create _Tests_, you should work with the [Bot Framework Emulator](https://docs.microsoft.com/en-us/azure/bot-service/bot-service-debug-emulator?view=azure-bot-service-4.0). 12 | 13 | **Note:** After going through [installation basics](https://docs.microsoft.com/en-us/azure/bot-service/bot-service-debug-emulator?view=azure-bot-service-4.0#prerequisites), make sure you configure [ngrok](https://ngrok.com/) as detailed [here](https://docs.microsoft.com/en-us/azure/bot-service/bot-service-debug-emulator?view=azure-bot-service-4.0#configure-ngrok). 14 | 15 | The simplest way to create a _Test_ is to have a conversation with your bot within the emulator, then save the _Transcript_ ([.transcript file](https://docs.microsoft.com/en-us/azure/bot-service/bot-builder-debug-transcript?view=azure-bot-service-4.0#the-bot-transcript-file)) as explained [here](https://docs.microsoft.com/en-us/azure/bot-service/bot-builder-debug-transcript?view=azure-bot-service-4.0#creatingstoring-a-bot-transcript-file). The _Transcript_ can be used by itself as a _Test_ for the service. The service will relate to the relevant information in the _Transcript_ (ignoring conversation-specific details like timestamps, for example) and attempt to conduct a similar conversation, sending the user utterances to the bot and expecting the same bot replies. 16 | 17 | ## Deployment 18 | 19 | The service is a [Node.js](https://nodejs.org) application. It can be installed using [npm](https://www.npmjs.com) (`npm install`). 20 | 21 | It can be easily deployed to Azure as an App Service: 22 | 23 | [![Deploy to Azure](https://azuredeploy.net/deploybutton.png)](https://azuredeploy.net/) 24 | 25 | ### Environment Variables 26 | 27 | The service communicates with a bot, therefore it needs to know the bot's [Web Chat](https://docs.microsoft.com/en-us/azure/bot-service/bot-service-channel-connect-webchat?view=azure-bot-service-4.0) [secret key](https://docs.microsoft.com/en-us/azure/bot-service/bot-service-channel-connect-webchat?view=azure-bot-service-4.0#step-1). 28 | 29 | The service may communicate with multiple bots, so each bot should be identified by a logical name. All bots' secrets need to be defined in a single environment variable named `SECRETS`, which should include a string representing a JSON object. In this JSON object, each key (bot's name) is mapped to a value (bot's secret). 30 | 31 | For example, let's assume we give the logical name '_samplebot_' to the bot we would like to test, and that its Web Chat secret is '_123_'. Then we should have an environment variable named `SECRETS` set to the following string: 32 | 33 | {"samplebot" : "123"} 34 | 35 | In case you would like to test a single bot most of the time, you can define an environment variable called `DefaultBot` to specify the logical name of your default bot. 36 | 37 | **Note:** If you are deploying the code sample using the "Deploy to Azure" option, you should set the variables in the Application Settings of your App Service. 38 | 39 | ## Running a Test 40 | 41 | There are several options for calling the service to run a _Test_, using HTTP `GET` or `POST`. There are also several ways to pass _Test_ parameters to the service. 42 | 43 | In all cases, the service needs to be aware of the bot to test. The target bot is identified by a logical name. This name can be passed as a 'bot' HTTP query parameter, e.g. '…?bot=_bot-logical-name_'. If no bot name is specified as a query parameter, the service uses the `DefaultBot` environment variable. 44 | 45 | ### Using HTTP `POST` with a _Transcript_ 46 | 47 | The simplest way to run a test is to `POST` an HTTP request to the `/test` route of the service. The request body should contain the contents of a _Transcript_ in JSON (application/json) format. 48 | 49 | Assuming our target bot is named '_samplebot_' and our service was deployed to Azure as '_testing123_', the request query may look like: 50 | 51 | `https://testing123.azurewebsites.net/test?bot=samplebot` 52 | 53 | In case you have `DeafultBot` set to '_samplebot_', the request may look like: 54 | 55 | `https://testing123.azurewebsites.net/test` 56 | 57 | ### Using HTTP `GET` with a _Transcript_ URL 58 | 59 | Instead of `POST`-ing the _Transcript_ as the request body, you can store it somewhere and give its URL to the service as a 'url' HTTP query parameter in a `GET` HTTP request. 60 | 61 | Let's assume that we have a Blob Storage account on Azure called '_samplestorageaccount_', and we uploaded a _Transcript_ file called '_Sample.transcript_' to a container called '_tests_'. The corresponding request query may look like: 62 | 63 | `https://testing123.azurewebsites.net/test?bot=samplebot&url=https://samplestorageaccount.blob.core.windows.net/tests/Sample.transcript` 64 | 65 | 66 | 67 | ## Contributing 68 | 69 | This project welcomes contributions and suggestions. Most contributions require you to agree to a 70 | Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us 71 | the rights to use your contribution. For details, visit https://cla.microsoft.com. 72 | 73 | When you submit a pull request, a CLA-bot will automatically determine whether you need to provide 74 | a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions 75 | provided by the bot. You will only need to do this once across all repos using our CLA. 76 | 77 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 78 | For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or 79 | contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. 80 | -------------------------------------------------------------------------------- /transcriptTransformationScript.js: -------------------------------------------------------------------------------- 1 | /** 2 | This script is used for removing attributes from a transcript exported by Bot Framework Emulator. 3 | Since the Bot Framework Emulator is connected directly to the bot, and not through the Bot Connector, 4 | there are differences between the messages in the exported transcript and the actual messages polled 5 | by the Functional Tests application. 6 | This script accepts one argument - a full path (including the file name) to a transcript file exported by 7 | Bot Framework Emulator, and creates a new transformed transcript file named "'"transfromed.transcript" 8 | in the script directory. 9 | Transformation is done by applying all defined handlers on each entry of the transcript object. 10 | **/ 11 | 12 | const fs = require('fs'); 13 | 14 | /** Here we can add all the handlers we need **/ 15 | // Handler 1 16 | function removeSchemaAttribute(currEntry) { 17 | if (currEntry['type'] === 'message') { 18 | if (currEntry.hasOwnProperty("attachments") && currEntry["attachments"][0].hasOwnProperty("content") && currEntry["attachments"][0]["content"].hasOwnProperty("$schema")) { 19 | delete currEntry["attachments"][0]["content"]["$schema"]; 20 | } 21 | } 22 | } 23 | 24 | // Handler 2 25 | function addSeparationAttribute(currEntry) { 26 | if (currEntry['type'] === 'message') { 27 | if (currEntry.hasOwnProperty("attachments") && currEntry["attachments"][0].hasOwnProperty("content") && currEntry["attachments"][0]["content"].hasOwnProperty("body") && currEntry["attachments"][0]["content"]["body"][0].hasOwnProperty("items")) { 28 | for (var item of currEntry["attachments"][0]["content"]["body"][0]["items"]) { 29 | if (item.hasOwnProperty("spacing") && item.hasOwnProperty("isSubtle")) { 30 | item["separation"] = "strong"; 31 | } 32 | } 33 | } 34 | 35 | } 36 | } 37 | 38 | function convertColumnsWidthToString(items) { 39 | for (var column of items["columns"]) { 40 | if (column.hasOwnProperty("width")) { 41 | column["width"] = column["width"] + ""; 42 | } 43 | } 44 | } 45 | // Handler 3 46 | function convertNumbersToString(currEntry) { 47 | if (currEntry['type'] === 'message') { 48 | if (currEntry.hasOwnProperty("attachments")) { 49 | for (var attachment of currEntry["attachments"]) { 50 | if (attachment.hasOwnProperty("content") && attachment["content"].hasOwnProperty("body")) { 51 | for (var bodyItem of attachment["content"]["body"]) { 52 | if (bodyItem.hasOwnProperty("columns")) { 53 | convertColumnsWidthToString(bodyItem); 54 | } else if (bodyItem.hasOwnProperty("items")) { 55 | for (var item of bodyItem["items"]) { 56 | if (item.hasOwnProperty("columns")) { 57 | convertColumnsWidthToString(item); 58 | } 59 | } 60 | } 61 | } 62 | } 63 | } 64 | } 65 | } 66 | } 67 | 68 | // Handler 4 69 | function convertColumnAttributesToCamelCase(currEntry) { 70 | if (currEntry['type'] === 'message') { 71 | if (currEntry.hasOwnProperty("attachments")) { 72 | const attachments = currEntry["attachments"]; 73 | attachments.forEach(attachment => { 74 | if (attachment.hasOwnProperty("content")) { 75 | const content = attachment["content"]; 76 | if (content.hasOwnProperty("body")) { 77 | const contentBody = content["body"][0]; 78 | if (contentBody.hasOwnProperty("items")) { 79 | const bodyItems = contentBody["items"]; 80 | bodyItems.forEach(bodyItem => { 81 | if (bodyItem.hasOwnProperty("columns")) { 82 | const columns = bodyItem["columns"]; 83 | columns.forEach(column => { 84 | if (column.hasOwnProperty("items")) { 85 | const colItems = column["items"]; 86 | const attributesToEdit = ["size", "weight", "color", "horizontalAlignment", "spacing"]; 87 | colItems.forEach(colItem => { 88 | attributesToEdit.forEach(attr => { 89 | if (colItem.hasOwnProperty(attr)) { 90 | colItem[attr] = colItem[attr].charAt(0).toLowerCase() + colItem[attr].slice(1, colItem[attr].length); 91 | } 92 | }); 93 | }); 94 | } 95 | }); 96 | } 97 | }); 98 | } 99 | } 100 | } 101 | }); 102 | } 103 | } 104 | } 105 | 106 | 107 | /** This is the main function - It iterates over all entries of the transcript, and applies all handlers on each entry **/ 108 | function main(path) { 109 | console.log("Started"); 110 | let contentBuffer; 111 | try { 112 | contentBuffer = fs.readFileSync(path); 113 | } 114 | catch (e) { 115 | console.log("Cannot open file", e.path); 116 | return; 117 | } 118 | let jsonTranscript = JSON.parse(contentBuffer); 119 | for (let i = 0; i < jsonTranscript.length; i++) { 120 | let currEntry = jsonTranscript[i]; 121 | // Here we call to all the handlers we defined 122 | removeSchemaAttribute(currEntry); 123 | addSeparationAttribute(currEntry); 124 | convertNumbersToString(currEntry); 125 | convertColumnAttributesToCamelCase(currEntry); 126 | 127 | } 128 | try { 129 | const filename = path.replace(/^.*[\\\/]/, '').replace(/\.[^/.]+$/, ''); // Extracts filename without extension from full path. 130 | fs.writeFileSync(filename + '_transformed.transcript', JSON.stringify(jsonTranscript)); 131 | console.log("Done"); 132 | } catch (e) { 133 | console.log("Cannot write file ", e); 134 | } 135 | } 136 | 137 | //Call main with file path as argument. 138 | main(process.argv[2]); 139 | -------------------------------------------------------------------------------- /.pipelines/[HealthBot]_[FunctionalTestService]_[Master].yml: -------------------------------------------------------------------------------- 1 | trigger: 2 | branches: 3 | include: 4 | - refs/heads/master 5 | batch: True 6 | name: $(BuildID)__$(Date:yyyy)-$(Date:MM)-$(Date:dd)_$(Hours)-$(Minutes) 7 | resources: 8 | repositories: 9 | - repository: onebranchTemplates 10 | type: git 11 | name: OneBranch.Pipelines/GovernedTemplates 12 | ref: refs/heads/main 13 | - repository: pes-utils 14 | type: git 15 | name: pes-utils 16 | ref: refs/heads/main 17 | - repository: HealthBotDevOps 18 | type: git 19 | name: HealthBotDevOps 20 | ref: refs/heads/master 21 | pipelines: 22 | - pipeline: '[HealthBotDevOps] [dockerfile] [master]' 23 | source: '[HealthBotDevOps] [dockerfile] [master]' 24 | trigger: 25 | branches: 26 | include: 27 | - refs/heads/master 28 | variables: 29 | - name: LinuxContainerImage 30 | value: mcr.microsoft.com/onebranch/azurelinux/build:3.0 31 | - name: WindowsContainerImage 32 | value: onebranch.azurecr.io/windows/ltsc2019/vse2022:latest 33 | extends: 34 | template: v2/OneBranch.NonOfficial.CrossPlat.yml@onebranchTemplates 35 | parameters: 36 | customTags: 'ES365AIMigrationTooling-BulkMigrated' 37 | stages: 38 | - stage: stage 39 | jobs: 40 | - job: Phase_1 41 | displayName: Phase 1 42 | cancelTimeoutInMinutes: 1 43 | pool: 44 | type: linux 45 | variables: 46 | - name: ob_outputDirectory 47 | value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT' 48 | - name: OB_build_container 49 | value: true 50 | steps: 51 | - checkout: self 52 | clean: true 53 | fetchTags: true 54 | - task: NodeTool@0 55 | displayName: 'Install Node.js 20.x' 56 | inputs: 57 | versionSource: 'spec' 58 | versionSpec: '20.x' 59 | - task: PowerShell@2 60 | displayName: 'Replace Tokens' 61 | inputs: 62 | targetType: 'inline' 63 | script: | 64 | $BUILD_NUMBER = (Get-Childitem env:\BUILD_NUMBER).Value 65 | 66 | #package.json 67 | $packageJsonLocation = 'package.json' 68 | $packageJson = Get-Content $packageJsonLocation -raw | ConvertFrom-Json 69 | $packageJson.description = 'Microsoft Health Bot Functional Tests- build# ' + $BUILD_NUMBER 70 | $packageJson | ConvertTo-Json | set-content $packageJsonLocation 71 | workingDirectory: '$(Build.SourcesDirectory)/BotFunctionalTestingService' 72 | env: 73 | BUILD_NUMBER: $(Build.BuildNumber) 74 | - task: Npm@1 75 | displayName: 'npm install (server)' 76 | inputs: 77 | workingDir: '$(Build.SourcesDirectory)/BotFunctionalTestingService' 78 | verbose: false 79 | - task: Npm@1 80 | displayName: 'npm update (server)' 81 | inputs: 82 | command: custom 83 | workingDir: '$(Build.SourcesDirectory)/BotFunctionalTestingService' 84 | verbose: false 85 | customCommand: update 86 | - task: Npm@1 87 | displayName: 'npm run build (server)' 88 | inputs: 89 | command: custom 90 | workingDir: '$(Build.SourcesDirectory)/BotFunctionalTestingService' 91 | verbose: false 92 | customCommand: 'run build' 93 | - task: DownloadPipelineArtifact@1 94 | displayName: 'Download Pipeline Artifact' 95 | inputs: 96 | buildType: specific 97 | project: '$(project)' 98 | pipeline: 74 99 | artifactName: 'health-bot-functional-tester-dockerfile-master' 100 | targetPath: '$(Build.SourcesDirectory)/dst' 101 | 102 | - task: ManifestGeneratorTask@0 103 | inputs: 104 | BuildDropPath: '$(Build.StagingDirectory)' 105 | 106 | - task: CopyFiles@2 107 | inputs: 108 | SourceFolder: '$(Build.StagingDirectory)/_manifest' 109 | Contents: '**' 110 | TargetFolder: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT/SBOM_Manifest' 111 | 112 | - task: DownloadPipelineArtifact@1 113 | displayName: 'Download Pipeline Artifact - master' 114 | inputs: 115 | buildType: specific 116 | project: '$(project)' 117 | pipeline: 74 118 | artifactName: 'health-bot-functional-tester-dockerfile-master' 119 | targetPath: '$(Build.SourcesDirectory)/dst' 120 | - task: CopyFiles@2 121 | inputs: 122 | SourceFolder: '$(Build.SourcesDirectory)/BotFunctionalTestingService' 123 | Contents: '**' 124 | TargetFolder: '$(ob_outputDirectory)/dist' 125 | - task: onebranch.pipeline.imagebuildinfo@1 126 | displayName: "Image build - Test" 127 | inputs: 128 | buildkit: 1 129 | repositoryName: functionaltestservice 130 | dockerFileRelPath: ./artifacts/Dockerfile 131 | dockerFileContextPath: ./artifacts/dist 132 | registry: hbscrtest.azurecr.io 133 | build_tag: '$(Build.BuildNumber)' # multiple tags are not supported 134 | enable_acr_push: true 135 | saveImageToPath: functionaltestservice-test.tar 136 | endpoint: hbs-acr-test 137 | enable_network: true 138 | enable_service_tree_acr_path: false 139 | arguments: '' 140 | - task: onebranch.pipeline.imagebuildinfo@1 141 | displayName: "Image build - Prod" 142 | inputs: 143 | buildkit: 1 144 | repositoryName: functionaltestservice 145 | dockerFileRelPath: ./artifacts/Dockerfile 146 | dockerFileContextPath: ./artifacts/dist 147 | registry: hbscrprod.azurecr.io 148 | build_tag: '$(Build.BuildNumber)' # multiple tags are not supported 149 | enable_acr_push: true 150 | saveImageToPath: functionaltestservice-prod.tar 151 | endpoint: hbs-acr-prod 152 | enable_network: true 153 | enable_service_tree_acr_path: false 154 | arguments: '' 155 | 156 | - template: pipelines/templates/prepare-for-release.yml@pes-utils 157 | parameters: 158 | env: test 159 | regionShort: eaus 160 | appName: bot-functional-tester 161 | imageTag: $(Build.BuildNumber) 162 | valuesFile: values-functional-tester 163 | 164 | - template: pipelines/templates/prepare-for-release-all-regions.yml@pes-utils 165 | parameters: 166 | appName: bot-functional-tester 167 | imageTag: $(Build.BuildNumber) 168 | valuesFile: values-functional-tester 169 | -------------------------------------------------------------------------------- /directlineclient.js: -------------------------------------------------------------------------------- 1 | const axios = require("axios"); 2 | var utils = require("./utils.js"); 3 | const logger = require("./logger"); 4 | 5 | // config items 6 | var pollInterval = 300; 7 | 8 | var directLineStartConversationUrl = `https://{directlineDomain}/v3/directline/conversations`; 9 | var directLineConversationUrlTemplate = `https://{directlineDomain}/v3/directline/conversations/{id}/activities`; 10 | 11 | DirectLineClient = function() { 12 | this.context = null; 13 | this.headers = {}; 14 | this.watermark = {}; 15 | } 16 | 17 | DirectLineClient.prototype.init = function(context, testData) { 18 | logger.log("DirectLine - init started"); 19 | var self = this; 20 | this.context = context; 21 | var headers = { 22 | Authorization: "Bearer " + testData.secret 23 | }; 24 | var startConversationOptions = { 25 | method: "POST", 26 | url: getDirectLineStartConversationUrl(testData.customDirectlineDomain), // CodeQL [SM04580] this is a closed api that is only accessible to an internal testing service, so the ssrf risk is mitigated 27 | headers: headers 28 | }; 29 | logger.log(`Init conversation request: ${JSON.stringify(logger.censorSecrets(startConversationOptions, ['headers.Authorization']))}`); // Censor the secret in logs 30 | var promise = axios.request(startConversationOptions) // CodeQL [SM04580] this is a closed api that is only accessible to an internal testing service, so the ssrf risk is mitigated 31 | .then(function({ data }) { 32 | logger.log("init response: " + utils.stringify(logger.censorSecrets(data, ['token']))); 33 | self.watermark[data.conversationId] = null; 34 | self.headers[data.conversationId] = headers; 35 | return data; 36 | }); 37 | return promise; 38 | } 39 | 40 | DirectLineClient.prototype.sendMessage = function(conversationId, message, customDirectlineDomain) { 41 | logger.log("sendMessage started"); 42 | logger.log("conversationId: " + conversationId); 43 | logger.log("message: " + utils.stringify(message)); 44 | var self = this; 45 | if (!conversationId) { 46 | throw new Error("DirectLineClient got invalid conversationId."); 47 | } 48 | 49 | var promise; 50 | if (isValidMessage(message)) { 51 | var postMessageOptions = { 52 | method: "POST", 53 | url: getConversationUrl(conversationId, customDirectlineDomain), // CodeQL [SM04580] this is a closed api that is only accessible to an internal testing service, so the ssrf risk is mitigated 54 | headers: self.headers[conversationId], 55 | data: message 56 | }; 57 | 58 | logger.log(`Send message request: ${JSON.stringify(logger.censorSecrets(postMessageOptions, ['headers.Authorization']))}`); 59 | promise = axios.request(postMessageOptions) // CodeQL [SM04580] this is a closed api that is only accessible to an internal testing service, so the ssrf risk is mitigated 60 | .then(function({ data }) { 61 | logger.log("sendMessage response: " + utils.stringify(data)); 62 | return data; 63 | }); 64 | } 65 | else { 66 | logger.log("sendMessage: message is invalid, not sending."); 67 | promise = Promise.resolve(null); 68 | } 69 | 70 | return promise; 71 | } 72 | 73 | DirectLineClient.prototype.pollMessages = function(conversationId, nMessages, bUserMessageIncluded, maxTimeout, customDirectlineDomain) { 74 | logger.log("pollMessages started"); 75 | logger.log("conversationId: " + conversationId); 76 | logger.log("nMessages: " + nMessages); 77 | logger.log("bUserMessageIncluded: " + bUserMessageIncluded); 78 | logger.log("maxTimeout: " + maxTimeout); 79 | var self = this; 80 | if (!conversationId) { 81 | throw new Error("DirectLineClient got invalid conversationId."); 82 | } 83 | 84 | var getMessagesOptions = { 85 | method: "GET", 86 | url: getConversationUrl(conversationId, customDirectlineDomain) + (this.watermark[conversationId] ? "?watermark=" + this.watermark[conversationId] : ""), // CodeQL [SM04580] this is a closed api that is only accessible to an internal testing service, so the ssrf risk is mitigated 87 | headers: self.headers[conversationId] 88 | }; 89 | 90 | var retries = 0; 91 | var maxRetries = (maxTimeout - 1) / pollInterval + 1; 92 | var messages; 93 | var nExpectedActivities = bUserMessageIncluded ? nMessages + 1 : nMessages; 94 | var promise = new Promise(function(resolve, reject) { 95 | var polling = function() { 96 | if (retries < maxRetries) { 97 | logger.log(`Poll messages request: ${JSON.stringify(logger.censorSecrets(getMessagesOptions, ['headers.Authorization']))}`); // CodeQL [SM04580] this is a closed api that is only accessible to an internal testing service, so the ssrf risk is mitigated 98 | axios.request(getMessagesOptions) 99 | .then(function({ data }) { 100 | messages = data.activities; 101 | logger.log(`Got ${messages.length} total activities (including user's response)`); 102 | if (messages.length < nExpectedActivities) { 103 | logger.log(`We have less than expected ${nExpectedActivities} activities - retry number ${retries + 1}...`); 104 | retries++; 105 | setTimeout(polling, pollInterval); 106 | } 107 | else { 108 | self.watermark[conversationId] = data.watermark; 109 | logger.log(`pollMessages messages: ${utils.stringify(messages)}`) 110 | resolve(messages); 111 | } 112 | }) 113 | .catch(function(err) { 114 | logger.log(`failed to get activities for on retry number ${retries + 1}. retrying...`); 115 | logger.log(err); 116 | retries++; 117 | setTimeout(polling, pollInterval); 118 | }); 119 | } 120 | else { 121 | logger.log(`pollMessages messages: ${utils.stringify(messages)}`) 122 | reject(new Error(`Could not obtain ${nMessages} responses`)); 123 | } 124 | } 125 | setTimeout(polling, pollInterval); 126 | }); 127 | return promise; 128 | } 129 | 130 | function isValidMessage(message) { 131 | return message && message.hasOwnProperty("type"); 132 | } 133 | 134 | function getConversationUrl(conversationId, customDirectlineDomain) { 135 | return directLineConversationUrlTemplate.replace("{directlineDomain}", customDirectlineDomain || process.env["directlineDomain"] || "directline.botframework.com" ).replace("{id}", conversationId); 136 | } 137 | 138 | function getDirectLineStartConversationUrl(customDirectlineDomain) { 139 | return directLineStartConversationUrl.replace("{directlineDomain}", customDirectlineDomain || process.env["directlineDomain"] || "directline.botframework.com" ); 140 | } 141 | 142 | module.exports = new DirectLineClient(); 143 | 144 | 145 | 146 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | var _ = require("underscore"); 2 | const crypto = require('crypto'); 3 | 4 | var expect = require("chai").expect; 5 | var diff = require("deep-object-diff").diff; 6 | 7 | var directline = require("./directlineclient"); 8 | var utils = require("./utils.js"); 9 | 10 | var Result = require("./result"); 11 | const logger = require("./logger"); 12 | 13 | var config = require("./config.json"); 14 | 15 | const initInterval = 500; 16 | 17 | class Test { 18 | static async perform(context, testData) { 19 | return await test(context, testData); 20 | } 21 | 22 | static async run(context, testData) { 23 | var testResult = await this.perform(context, testData); 24 | const eventData = { test: testData.name, details: testResult.message, conversationId: testResult.conversationId }; 25 | 26 | if (testResult.success) { 27 | logger.event("TestSucceeded", eventData); 28 | context.success(testResult); 29 | } 30 | else { 31 | logger.event("TestFailed", eventData); 32 | context.failure(testResult.code, testResult.message); 33 | } 34 | } 35 | } 36 | 37 | async function test(context, testData) { 38 | logger.log("test started"); 39 | // Break the conversation into messages from the user side vs. replies from the bot side 40 | // Each conversation step contains an array of user messages (typically one) and an array of bot replies (typically one, but it's normal to have more than one) 41 | // For each conversation step, first send the user message and then wait for the expected reply 42 | var testUserId = "test-user-" + crypto.randomBytes(4).toString("hex"); 43 | var conversationSteps = createConversationSteps(testData); 44 | try { 45 | // retrying initialization: 46 | let initResult; 47 | let tolerance = parseInt(process.env.failureTolerance) ? parseInt(process.env.failureTolerance) : config.defaults.failureTolerance; 48 | for (let i = 1; i <= tolerance; i++) { 49 | try { 50 | initResult = await directline.init(context, testData); 51 | break; 52 | } catch (err) { 53 | if (i === tolerance){ 54 | logger.log("testData: " + utils.stringify(testData)); 55 | logger.log("failed initializing %d times",tolerance ); // after last init attempt 56 | throw err; 57 | } 58 | else{ 59 | logger.log(`failed to initialize user ID ${testUserId} on retry number ${i}, retrying in ${initInterval / 1000} seconds...`); 60 | await new Promise((resolve) => setTimeout(resolve, initInterval)); 61 | } 62 | } 63 | } 64 | var conversationResult = await testConversation(context, testUserId, conversationSteps, initResult.conversationId, testData.timeout, testData.customDirectlineDomain); 65 | var message = `${getTestTitle(testData)} passed successfully (${conversationResult.count} ${conversationResult.count == 1 ? "step" : "steps"} passed)`; 66 | return new Result({ success: true, message, conversationId: initResult.conversationId }); 67 | } 68 | catch (err) { 69 | var reason; 70 | if (err.hasOwnProperty("details")) { 71 | reason = err.details; 72 | if (reason && reason.hasOwnProperty("message")) { 73 | reason.message = getTestTitle(testData) + ": " + reason.message; 74 | } 75 | } 76 | else { 77 | reason = getTestTitle(testData) + ": " + err.message; 78 | } 79 | return new Result({ success: false, message: reason, code: 500 }); 80 | } 81 | } 82 | 83 | function createConversationSteps(testData) { 84 | conversation = []; 85 | // Assuming that each user message is followed by at least one bot reply 86 | 87 | // Check whether the first message is from the bot 88 | if (!isUserMessage(testData, testData.messages[0])) { 89 | // If the first message is from the but, start with a special step with no user message 90 | conversation.push(new conversationStep(null)); 91 | } 92 | for (var i = 0; i < testData.messages.length; i++) { 93 | var message = testData.messages[i]; 94 | if (isUserMessage(testData, message)) { 95 | // User message - start a new step 96 | conversation.push(new conversationStep(message)); 97 | } 98 | else { 99 | // Bot message - add the bot reply to the current step 100 | conversation[conversation.length - 1].botReplies.push(message); 101 | } 102 | } 103 | return conversation; 104 | } 105 | 106 | function isUserMessage(testData, message) { 107 | return (testData && testData.userId) ? (message.from.id == testData.userId) : (message.recipient ? (message.recipient.role == "bot") : (message.from.role != "bot")); 108 | } 109 | 110 | function conversationStep(message) { 111 | this.userMessage = message; 112 | this.botReplies = []; 113 | } 114 | 115 | function testConversation(context, testUserId, conversationSteps, conversationId, defaultTimeout, customDirectlineDomain) { 116 | logger.log("testConversation started"); 117 | logger.log("testUserId: " + testUserId); 118 | logger.log("conversationSteps: " + utils.stringify(conversationSteps)); 119 | logger.log("conversationId: " + conversationId); 120 | logger.log("defaultTimeout: " + defaultTimeout); 121 | return new Promise(function(resolve, reject) { 122 | var index = 0; 123 | function nextStep() { 124 | if (index < conversationSteps.length) { 125 | logger.log("Testing conversation step " + index); 126 | var stepData = conversationSteps[index]; 127 | index++; 128 | var userMessage = createUserMessage(stepData.userMessage, testUserId); 129 | return testStep(context, conversationId, userMessage, stepData.botReplies, defaultTimeout, customDirectlineDomain).then(nextStep, reject); 130 | } 131 | else { 132 | logger.log("testConversation end"); 133 | resolve({count: index}); 134 | } 135 | } 136 | return nextStep(); 137 | }); 138 | } 139 | 140 | function createUserMessage(message, testUserId) { 141 | var userMessage = _.pick(message, "type", "text", "value", "locale"); 142 | userMessage.from = { 143 | id: testUserId, 144 | name: "Test User" 145 | }; 146 | return userMessage; 147 | } 148 | 149 | function testStep(context, conversationId, userMessage, expectedReplies, timeoutMilliseconds, customDirectlineDomain) { 150 | logger.log("testStep started"); 151 | logger.log("conversationId: " + conversationId); 152 | logger.log("userMessage: " + utils.stringify(userMessage)); 153 | logger.log("expectedReplies: " + utils.stringify(expectedReplies)); 154 | logger.log("timeoutMilliseconds: " + timeoutMilliseconds); 155 | return directline.sendMessage(conversationId, userMessage, customDirectlineDomain) 156 | .then(function(response) { 157 | var nMessages = expectedReplies.hasOwnProperty("length") ? expectedReplies.length : 1; 158 | var bUserMessageIncluded = response != null; 159 | return directline.pollMessages(conversationId, nMessages, bUserMessageIncluded, timeoutMilliseconds, customDirectlineDomain); 160 | }) 161 | .then(function(messages) { 162 | return compareMessages(context, userMessage, expectedReplies, messages); 163 | }) 164 | .catch(function(err) { 165 | var message = `User message '${userMessage.text}' response failed - ${err.message}`; 166 | if (err.hasOwnProperty("details")) { 167 | err.details.message = message; 168 | } 169 | else { 170 | err.message = message; 171 | } 172 | throw err; 173 | }); 174 | } 175 | 176 | function compareMessages(context, userMessage, expectedReplies, actualMessages) { 177 | logger.log("compareMessages started"); 178 | logger.log("actualMessages: " + utils.stringify(actualMessages)); 179 | // Filter out messages from the (test) user, leaving only bot replies 180 | var botReplies = _.reject(actualMessages, 181 | function(message) { 182 | return message.from.id == userMessage.from.id; 183 | }); 184 | 185 | expect(botReplies, `reply to user message '${userMessage.text}'`).to.have.lengthOf(expectedReplies.length); 186 | 187 | for (var i = 0; i < expectedReplies.length; i++) { 188 | var assert = expectedReplies[i].assert || "to.be.equal"; 189 | var expectedReply = expectedReplies[i]; 190 | var botReply = botReplies[i]; 191 | 192 | if (botReply.hasOwnProperty("text")) { 193 | var expr = 'expect(botReply.text, "user message number ' + (i+1) + ' ").' + assert + '(expectedReply.text)'; 194 | eval(expr); 195 | } 196 | if (botReply.hasOwnProperty("attachments")) { 197 | try { 198 | expect(botReply.attachments,`attachments of reply number ${i+1} to user message '${userMessage.text}'`).to.deep.equal(expectedReply.attachments); 199 | } 200 | catch (err) { 201 | var exception = new Error(err.message); 202 | exception.details = {message: err.message, expected: err.expected, actual: err.actual, diff: diff(err.expected, err.actual)}; 203 | throw exception; 204 | } 205 | } 206 | } 207 | return true; 208 | } 209 | 210 | function getTestTitle(testData) { 211 | return `Test ${testData.name? `'${testData.name}'` : `#${testData.index || 0}`}`; 212 | } 213 | 214 | module.exports = Test; 215 | 216 | --------------------------------------------------------------------------------