├── .dockerignore ├── .eslintignore ├── .eslintrc.js ├── .gitignore ├── .npmignore ├── Dockerfile ├── LICENSE ├── README.md ├── bin └── botium-cli.js ├── package-merge-use-botium-npm.json ├── package.json ├── report.js ├── rollup.config.js └── src ├── agent └── index.js ├── box └── index.js ├── emulator ├── console │ └── index.js └── index.js ├── hello └── index.js ├── import └── index.js ├── index.js ├── init-alexa-avs └── index.js ├── init-alexa-smapi └── index.js ├── init-dev ├── asserter │ ├── botium.json │ ├── spec │ │ └── convos │ │ │ └── echo.convo.txt │ └── src │ │ └── MyCustomAsserter.js ├── connector │ ├── botium.json │ ├── spec │ │ └── convos │ │ │ └── echo.convo.txt │ └── src │ │ └── MyCustomConnector.js ├── index.js └── logichook │ ├── botium.json │ ├── spec │ └── convos │ │ └── echo.convo.txt │ └── src │ ├── MyCustomLogicHook.js │ └── MyGlobalLogicHook.js ├── init └── index.js ├── metrics.js ├── nlp ├── extract.js ├── index.js └── split.js ├── proxy └── index.js └── run └── index.js /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | botiumwork 4 | dist 5 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | **/node_modules/* -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "extends": "standard" 3 | }; -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (http://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # Typescript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | dist 61 | /.idea 62 | package-lock.json 63 | test/ -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (http://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # Typescript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | temp 61 | botiumwork 62 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:15-alpine3.13 2 | 3 | RUN apk add --no-cache \ 4 | coreutils \ 5 | curl curl-dev \ 6 | dos2unix \ 7 | git \ 8 | g++ \ 9 | make \ 10 | python2 \ 11 | bash \ 12 | krb5 krb5-dev \ 13 | libstdc++ \ 14 | chromium \ 15 | chromium-chromedriver \ 16 | harfbuzz \ 17 | nss \ 18 | freetype \ 19 | tini \ 20 | ttf-freefont \ 21 | && rm -rf /var/cache/* \ 22 | && mkdir /var/cache/apk 23 | ENV CHROME_BIN=/usr/bin/chromium-browser CHROME_PATH=/usr/lib/chromium/ CHROMEDRIVER_FILEPATH=/usr/bin/chromedriver 24 | 25 | COPY ./package.json /app/botium-cli/package.json 26 | COPY ./package-merge-use-botium-npm.json /app/botium-cli/package-merge-use-botium-npm.json 27 | COPY ./report.js /app/botium-cli/report.js 28 | RUN cd /app/botium-cli && npx json-merger -p package-merge-use-botium-npm.json > package-npm.json && mv package-npm.json package.json 29 | RUN cd /app/botium-cli && BOTIUM_ANALYTICS=false yarn install --no-optional --ignore-engines 30 | RUN apk del curl-dev g++ make python2 dos2unix 31 | COPY . /app/botium-cli 32 | 33 | WORKDIR /app/workdir 34 | VOLUME /app/workdir 35 | ENTRYPOINT ["node", "/app/botium-cli/bin/botium-cli.js"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Botium GmbH 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Botium CLI - The Selenium for Chatbots 2 | ====================================== 3 | 4 | [![NPM](https://nodei.co/npm/botium-cli.png?downloads=true&downloadRank=true&stars=true)](https://nodei.co/npm/botium-cli/) 5 | 6 | [ ![Codeship Status for codeforequity-at/botium-cli](https://app.codeship.com/projects/4d7fd410-18ab-0136-6ab1-6e2b4bb62b94/status?branch=master)](https://app.codeship.com/projects/283938) 7 | [![npm version](https://badge.fury.io/js/botium-cli.svg)](https://badge.fury.io/js/botium-cli) 8 | [![license](https://img.shields.io/github/license/mashape/apistatus.svg)]() 9 | [![docs](https://readthedocs.org/projects/botium-docs/badge/)](https://botium-docs.readthedocs.io/) 10 | 11 | [Botium is the Selenium for chatbots](https://www.botium.ai). Botium CLI is the swiss army knife of Botium. 12 | 13 | **_IF YOU LIKE WHAT YOU SEE, PLEASE CONSIDER GIVING US A STAR ON GITHUB!_** 14 | 15 | **UPDATE 2020/11/05:** Botium has a FREE, hosted plan available! The new Botium Box Mini is our ❤️ to the community. [Take it for a test drive 🚗 ...](https://www.botium.ai/pricing/) 16 | 17 | [![](http://img.youtube.com/vi/ciVxojvRfng/0.jpg)](https://www.youtube.com/watch?v=ciVxojvRfng "Botium Box Mini") 18 | 19 | # Documentation 20 | 21 | See [here](https://botium-docs.readthedocs.io/) for Botium documentation. 22 | -------------------------------------------------------------------------------- /bin/botium-cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | const yargsCmd = require('yargs') 3 | const _ = require('lodash') 4 | const debug = require('debug')('botium-cli') 5 | 6 | const handleConfig = (argv) => { 7 | argv.verbose = argv.v = process.env.BOTIUM_VERBOSE === '1' || argv.verbose 8 | 9 | if (argv.verbose) { 10 | require('debug').enable('botium*') 11 | } 12 | 13 | if (!process.env.BOTIUM_CONFIG) { 14 | process.env.BOTIUM_CONFIG = argv.config 15 | } 16 | debug(`Using Botium configuration file ${process.env.BOTIUM_CONFIG}`) 17 | 18 | const envConvoDirs = Object.keys(process.env).filter(e => e.startsWith('BOTIUM_CONVOS')).map(e => process.env[e]).filter(e => e) 19 | if (envConvoDirs && envConvoDirs.length > 0) { 20 | argv.convos = envConvoDirs 21 | } else { 22 | if (argv.convos && _.isString(argv.convos)) { 23 | argv.convos = [argv.convos] 24 | } 25 | } 26 | 27 | return true 28 | } 29 | 30 | const wrapHandler = (builder) => { 31 | const origHandler = builder.handler 32 | builder.handler = (argv) => { 33 | if (handleConfig(argv)) { 34 | origHandler(argv) 35 | } 36 | } 37 | return builder 38 | } 39 | 40 | const runIfModuleAvailable = (checkModule, fn, fn1) => { 41 | try { 42 | require(checkModule) 43 | return fn() 44 | } catch (err) { 45 | if (fn1) return fn1() 46 | } 47 | } 48 | 49 | const yargs = yargsCmd.usage('Botium CLI\n\nUsage: $0 [options]') // eslint-disable-line 50 | .help('help').alias('help', 'h') 51 | .version('version', require('../package.json').version).alias('version', 'V') 52 | .showHelpOnFail(true) 53 | .strict(true) 54 | .demandCommand(1, 'You need at least one command before moving on') 55 | .command(wrapHandler(require('../src/run'))) 56 | .command(wrapHandler(require('../src/hello'))) 57 | .command(wrapHandler(require('../src/nlp'))) 58 | .command(wrapHandler(require('../src/nlp/extract'))) 59 | .command(wrapHandler(require('../src/nlp/split'))) 60 | .command(wrapHandler(require('../src/emulator'))) 61 | .command(wrapHandler(require('../src/box'))) 62 | .command(wrapHandler(require('../src/init'))) 63 | .command(wrapHandler(require('../src/init-dev'))) 64 | .command(wrapHandler(require('../src/proxy'))) 65 | .command(wrapHandler(require('../src/agent'))) 66 | .option('verbose', { 67 | alias: 'v', 68 | describe: 'Enable verbose output (also read from env variable "BOTIUM_VERBOSE" - "1" means verbose)', 69 | type: 'boolean', 70 | default: false 71 | }) 72 | .option('convos', { 73 | alias: 'C', 74 | describe: 'Path to a directory holding your convo files. Can be specified more than once, ending in "--" ("... --convos dir1 dir2 dir3 -- ...") (also read from env variables starting with "BOTIUM_CONVOS")', 75 | array: true, 76 | default: '.' 77 | }) 78 | .option('config', { 79 | alias: 'c', 80 | envPrefix: 'BOTIUM_CONFIG', 81 | describe: 'Path to the Botium configuration file (also read from env variable "BOTIUM_CONFIG")', 82 | nargs: 1, 83 | default: './botium.json' 84 | }) 85 | .command(wrapHandler(require('../src/import')('botium-connector-alexa-smapi', 'alexaimport', 'Import convos and utterances from Alexa SMAPI'))) 86 | .command(wrapHandler(require('../src/import')('botium-connector-dialogflow', 'dialogflowimport', 'Import convos and utterances from Google Dialogflow'))) 87 | .command(wrapHandler(require('../src/import')('botium-connector-watson', 'watsonimport', 'Import convos and utterances from IBM Watson Assistant'))) 88 | .command(wrapHandler(require('../src/import')('botium-connector-lex', 'leximport', 'Import convos and utterances from Amazon Lex'))) 89 | .command(wrapHandler(require('../src/import')('botium-connector-luis', 'luisimport', 'Import convos and utterances from Microsoft LUIS'))) 90 | .command(wrapHandler(require('../src/import')('botium-connector-qnamaker', 'qnamakerimport', 'Import convos and utterances from QnAMaker'))) 91 | .command(wrapHandler(require('../src/import')('botium-connector-rasa', 'rasaimport', 'Import convos and utterances from Rasa'))) 92 | 93 | runIfModuleAvailable('botium-crawler', () => yargs.command(wrapHandler(require('botium-crawler/src/crawler-run'))), () => yargs.command({ command: 'crawler-run', describe: 'Install NPM module "botium-crawler" to enable this command', handler: () => ({}) })) 94 | runIfModuleAvailable('botium-crawler', () => yargs.command(wrapHandler(require('botium-crawler/src/crawler-feedbacks'))), () => yargs.command({ command: 'crawler-feedbacks', describe: 'Install NPM module "botium-crawler" to enable this command', handler: () => ({}) })) 95 | runIfModuleAvailable('botium-connector-alexa-avs', () => yargs.command(wrapHandler(require('../src/init-alexa-avs'))), () => yargs.command({ command: 'init-alexa-avs', describe: 'Install NPM module "botium-connector-alexa-avs" to enable this command', handler: () => ({}) })) 96 | runIfModuleAvailable('botium-connector-alexa-smapi', () => yargs.command(wrapHandler(require('../src/init-alexa-smapi'))), () => yargs.command({ command: 'init-alexa-smapi', describe: 'Install NPM module "botium-connector-alexa-smapi" to enable this command', handler: () => ({}) })) 97 | 98 | yargs.argv // eslint-disable-line -------------------------------------------------------------------------------- /package-merge-use-botium-npm.json: -------------------------------------------------------------------------------- 1 | { 2 | "$merge": { 3 | "source": { 4 | "$import": "package.json" 5 | }, 6 | "with": { 7 | "dependencies": { 8 | "botium-asserter-basiclink": "0.0.7", 9 | "botium-asserter-watson-toneanalyzer": "0.0.1", 10 | "botium-connector-alexa-avs": "0.0.13", 11 | "botium-connector-alexa-smapi": "0.0.12", 12 | "botium-connector-botframework": "0.0.2", 13 | "botium-connector-botkit": "0.0.5", 14 | "botium-connector-botkit-websocket": "0.2.0", 15 | "botium-connector-botpress": "0.0.4", 16 | "botium-connector-chatlayer": "0.0.8", 17 | "botium-connector-cognigy": "0.0.6", 18 | "botium-connector-dialogflow": "0.1.0", 19 | "botium-connector-dialogflowcx": "0.0.3", 20 | "botium-connector-directline3": "0.0.26", 21 | "botium-connector-einsteinbot": "0.0.4", 22 | "botium-connector-fbwebhook": "0.0.6", 23 | "botium-connector-fbmessengerbots": "0.0.1", 24 | "botium-connector-google-assistant": "0.0.8", 25 | "botium-connector-holmes": "0.0.2", 26 | "botium-connector-inbenta": "0.0.3", 27 | "botium-connector-koreai-webhook": "0.0.3", 28 | "botium-connector-lex": "0.1.0", 29 | "botium-connector-liveperson": "0.0.1", 30 | "botium-connector-luis": "0.0.7", 31 | "botium-connector-nlpjs": "0.0.2", 32 | "botium-connector-ondewo": "0.0.2", 33 | "botium-connector-oracleda": "0.0.3", 34 | "botium-connector-pandorabots": "0.0.2", 35 | "botium-connector-qnamaker": "0.1.4", 36 | "botium-connector-rasa": "0.0.10", 37 | "botium-connector-sapcai": "0.0.7", 38 | "botium-connector-teneo": "0.0.5", 39 | "botium-connector-ubitec": "0.0.7", 40 | "botium-connector-watson": "0.0.23", 41 | "botium-connector-webdriverio": "0.3.17", 42 | "botium-connector-webdrivervoice": "0.0.3", 43 | "botium-connector-websocket": "0.0.9", 44 | "botium-connector-wechat": "0.0.1", 45 | "botium-connector-whatsapp": "0.0.3", 46 | "botium-connector-witai": "0.0.3", 47 | "botium-connector-xatkit": "0.2.0", 48 | "botium-crawler": "0.0.14", 49 | "botium-logichook-perfectoreporting": "0.0.5" 50 | } 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "botium-cli", 3 | "version": "1.1.0", 4 | "description": "Botium - The Selenium for Chatbots", 5 | "main": "dist/botium-cli-cjs.js", 6 | "module": "dist/botium-cli-es.js", 7 | "engines": { 8 | "node": ">=10" 9 | }, 10 | "bin": { 11 | "botium-cli": "./bin/botium-cli.js" 12 | }, 13 | "scripts": { 14 | "postinstall": "node ./report.js", 15 | "build": "npm run eslint && rollup -c", 16 | "builddocker": "docker build . -t botium/botium-cli:$npm_package_version", 17 | "publishdocker": "docker tag botium/botium-cli:$npm_package_version botium/botium-cli:latest && docker push botium/botium-cli:$npm_package_version && docker push botium/botium-cli:latest", 18 | "eslint": "eslint \"./src/**/*.js\" \"./bin/**/*.js\"", 19 | "eslint:fix": "eslint --fix \"./src/**/*.js\" \"./bin/**/*.js\"", 20 | "test": "echo \"no tests for botium-cli yet\" && exit 0", 21 | "update-dependencies": "npm-check-updates -u --reject rollup --timeout 60000" 22 | }, 23 | "repository": { 24 | "type": "git", 25 | "url": "git+https://github.com/codeforequity-at/botium-cli.git" 26 | }, 27 | "author": "Florian Treml", 28 | "license": "MIT", 29 | "bugs": { 30 | "url": "https://github.com/codeforequity-at/botium-cli/issues" 31 | }, 32 | "homepage": "https://www.botium.ai", 33 | "devDependencies": { 34 | "@babel/core": "^7.21.4", 35 | "@babel/node": "^7.20.7", 36 | "@babel/plugin-transform-runtime": "^7.21.4", 37 | "@babel/preset-env": "^7.21.4", 38 | "eslint": "^8.38.0", 39 | "eslint-config-standard": "^17.0.0", 40 | "eslint-plugin-import": "^2.27.5", 41 | "eslint-plugin-n": "^15.7.0", 42 | "eslint-plugin-promise": "^6.1.1", 43 | "eslint-plugin-standard": "^4.1.0", 44 | "license-checker": "^25.0.1", 45 | "npm-check-updates": "^16.10.9", 46 | "rollup": "^2.60.0", 47 | "rollup-plugin-babel": "^4.4.0", 48 | "@rollup/plugin-commonjs": "^24.1.0", 49 | "@rollup/plugin-json": "^6.0.0" 50 | }, 51 | "dependencies": { 52 | "@babel/runtime": "^7.21.0", 53 | "botium-connector-echo": "0.0.19", 54 | "botium-core": "1.13.16", 55 | "chai": "^4.3.7", 56 | "debug": "^4.3.4", 57 | "figlet": "^1.6.0", 58 | "fs-extra": "^11.1.1", 59 | "is-json": "^2.0.1", 60 | "lodash": "^4.17.21", 61 | "mime-types": "^2.1.35", 62 | "mkdirp": "^3.0.0", 63 | "mocha": "^10.2.0", 64 | "mochawesome": "^7.1.3", 65 | "promise-retry": "^2.0.1", 66 | "readline": "^1.3.0", 67 | "request": "^2.88.2", 68 | "slug": "^8.2.2", 69 | "terminal-kit": "^3.0.0", 70 | "update-dependencies": "^1.0.2", 71 | "yargs": "^17.7.1" 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /report.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const os = require('os') 3 | 4 | const botiumAnalyticsHost = process.env.BOTIUM_ANALYTICS_HOST || 'v1.license.botium.cyaraportal.us' 5 | const botiumAnalyticsPort = process.env.BOTIUM_ANALYTICS_PORT || 443 6 | const https = botiumAnalyticsPort === 443 ? require('https') : require('http') 7 | const execTimeout = 10000 8 | 9 | function logIfVerbose (toLog, stream) { 10 | if (process.env.BOTIUM_ANALYTICS_VERBOSE === 'true') { 11 | (stream || console.log)(toLog) 12 | } 13 | } 14 | 15 | async function reportPostInstall () { 16 | if (process.env.BOTIUM_ANALYTICS === 'false') return 17 | 18 | const packageJson = require(path.join(__dirname, 'package.json')) 19 | 20 | const infoPayload = { 21 | rawPlatform: os.platform(), 22 | rawArch: os.arch(), 23 | library: packageJson.name, 24 | version: packageJson.version 25 | } 26 | 27 | const data = JSON.stringify(infoPayload) 28 | logIfVerbose(`Botium analytics payload: ${data}`) 29 | 30 | const reqOptions = { 31 | host: botiumAnalyticsHost, 32 | port: botiumAnalyticsPort, 33 | method: 'POST', 34 | path: '/metrics/installation/core', 35 | headers: { 36 | 'Content-Type': 'application/json', 37 | 'Content-Length': data.length 38 | }, 39 | timeout: execTimeout 40 | } 41 | await new Promise((resolve, reject) => { 42 | const req = https.request(reqOptions, (res) => { 43 | logIfVerbose(`Response status: ${res.statusCode}`) 44 | resolve() 45 | }) 46 | 47 | req.on('error', error => { 48 | logIfVerbose(error, console.error) 49 | reject(error) 50 | }) 51 | 52 | req.on('timeout', error => { 53 | logIfVerbose(error, console.error) 54 | reject(error) 55 | }) 56 | 57 | req.write(data) 58 | req.end() 59 | }) 60 | } 61 | 62 | if (require.main === module) { 63 | try { 64 | reportPostInstall().catch(e => { 65 | logIfVerbose(`\n\n${e}`, console.error) 66 | }).finally(() => { 67 | process.exit(0) 68 | }) 69 | } catch (e) { 70 | logIfVerbose(`\n\nTop level error: ${e}`, console.error) 71 | process.exit(0) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /rollup.config.js: -------------------------------------------------------------------------------- 1 | import babel from 'rollup-plugin-babel' 2 | import commonjs from '@rollup/plugin-commonjs' 3 | import json from '@rollup/plugin-json' 4 | 5 | export default { 6 | input: 'src/index.js', 7 | output: [ 8 | { 9 | file: 'dist/botium-cli-es.js', 10 | format: 'es', 11 | sourcemap: true 12 | }, 13 | { 14 | file: 'dist/botium-cli-cjs.js', 15 | format: 'cjs', 16 | sourcemap: true 17 | } 18 | ], 19 | plugins: [ 20 | commonjs({ 21 | exclude: 'node_modules/**' 22 | }), 23 | babel({ 24 | exclude: 'node_modules/**', 25 | runtimeHelpers: true 26 | }), 27 | json() 28 | ] 29 | } 30 | -------------------------------------------------------------------------------- /src/agent/index.js: -------------------------------------------------------------------------------- 1 | const util = require('util') 2 | const debug = require('debug')('botium-cli-agent') 3 | 4 | const handler = (argv) => { 5 | debug(`command options: ${util.inspect(argv)}`) 6 | 7 | if (argv.port) { 8 | process.env.PORT = argv.port 9 | } 10 | if (argv.apitoken) { 11 | process.env.BOTIUM_API_TOKEN = argv.apitoken 12 | } 13 | require('botium-core/src/grid/agent/agent') 14 | } 15 | 16 | module.exports = { 17 | command: 'agent', 18 | describe: 'Launch Botium agent', 19 | builder: (yargs) => { 20 | yargs.option('port', { 21 | describe: 'Local port the agent is listening to', 22 | number: true, 23 | default: 46100 24 | }) 25 | yargs.option('apitoken', { 26 | describe: 'API Token for clients to connect (also read from env variable "BOTIUM_API_TOKEN")' 27 | }) 28 | }, 29 | handler 30 | } 31 | -------------------------------------------------------------------------------- /src/box/index.js: -------------------------------------------------------------------------------- 1 | const util = require('util') 2 | const yargsCmd = require('yargs') 3 | const request = require('request') 4 | const Mocha = require('mocha') 5 | const _ = require('lodash') 6 | const addContext = require('mochawesome/addContext') 7 | const debug = require('debug')('botium-cli-box') 8 | const { parseReporterOptions, parseReporter, outputTypes } = require('../run/index') 9 | 10 | const handleArrayParam = (argv, paramName, envNameStart) => { 11 | argv[paramName] = argv[paramName] ? _.isArray(argv[paramName]) ? argv[paramName] : [argv[paramName]] : [] 12 | const envParams = Object.keys(process.env).filter(e => e.startsWith(envNameStart)).map(e => process.env[e]).filter(e => e) 13 | if (envParams) { 14 | argv[paramName] = argv[paramName].concat(envParams) 15 | } 16 | } 17 | 18 | const buildBotiumCaps = (arg) => { 19 | return arg && arg.reduce((acc, c) => { 20 | const [key, val] = c.split('=') 21 | if (key && val) { 22 | acc[key] = val 23 | } else { 24 | acc[key] = true 25 | } 26 | return acc 27 | }, {}) 28 | } 29 | 30 | const handler = (argv) => { 31 | debug(`command options: ${util.inspect(argv)}`) 32 | 33 | const reporterOptions = parseReporterOptions(argv.reporterOptions) 34 | debug(`Mocha Reporter "${argv.output}", options: ${util.inspect(reporterOptions)}`) 35 | 36 | argv.webhook = process.env.BOTIUM_BOX_WEBHOOK || argv.webhook 37 | if (!argv.webhook) { 38 | return yargsCmd.showHelp() 39 | } 40 | 41 | argv.buildid = process.env.BOTIUM_BOX_BUILDID || argv.buildid 42 | argv.buildcomment = process.env.BOTIUM_BOX_BUILDCOMMENT || argv.buildcomment 43 | 44 | handleArrayParam(argv, 'tags', 'BOTIUM_BOX_TAGS') 45 | handleArrayParam(argv, 'caps', 'BOTIUM_BOX_CAPS') 46 | handleArrayParam(argv, 'sources', 'BOTIUM_BOX_SOURCES') 47 | handleArrayParam(argv, 'envs', 'BOTIUM_BOX_ENVS') 48 | 49 | const boxPostParams = { 50 | WAIT: '1', 51 | REPORTER: 'json' 52 | } 53 | if (argv.buildid) boxPostParams.BUILDID = argv.buildid 54 | if (argv.buildcomment) boxPostParams.BUILDCOMMENT = argv.buildcomment 55 | if (argv.tags) boxPostParams.TAG = argv.tags.map(t => `${t}`) 56 | if (argv.caps) boxPostParams.CAPS = buildBotiumCaps(argv.caps) 57 | if (argv.sources) boxPostParams.SOURCES = buildBotiumCaps(argv.sources) 58 | if (argv.envs) boxPostParams.ENVS = buildBotiumCaps(argv.envs) 59 | 60 | const requestOptions = { 61 | uri: argv.webhook, 62 | method: 'POST', 63 | json: boxPostParams, 64 | timeout: argv.timeout * 1000 65 | } 66 | 67 | debug(`Botium Box calling ${argv.webhook} command options: ${util.inspect(requestOptions)}`) 68 | request(requestOptions, (err, response, body) => { 69 | if (err) { 70 | console.log(`ERROR: ${err}`) 71 | return process.exit(1) 72 | } 73 | if (!body || !body.id) { 74 | console.log(`ERROR: NO JSON RESPONSE ${util.inspect(body)}`) 75 | return process.exit(1) 76 | } 77 | debug(`Botium Box sent response: ${util.inspect(body)}`) 78 | 79 | const mocha = new Mocha({ 80 | reporter: parseReporter(argv.output), 81 | reporterOptions 82 | }) 83 | 84 | const suite = Mocha.Suite.create(mocha.suite, body.name) 85 | let runner = null 86 | 87 | if (!body.results || body.results.length === 0) { 88 | const test = new Mocha.Test(body.name, (testcaseDone) => { 89 | if (body.status === 'READY') { 90 | testcaseDone() 91 | } else { 92 | testcaseDone(`Botium Box returned status: ${body.status}`) 93 | } 94 | }) 95 | suite.addTest(test) 96 | } else { 97 | body.results.forEach((result) => { 98 | const test = new Mocha.Test(result.testcaseName, (testcaseDone) => { 99 | if (result.testcaseSource) addContext(runner, { title: 'Conversation Log', value: result.testcaseSource }) 100 | 101 | if (result.success) { 102 | testcaseDone() 103 | } else { 104 | testcaseDone(new Error(result.err || 'Botium Box returned error')) 105 | } 106 | }) 107 | suite.addTest(test) 108 | }) 109 | } 110 | 111 | runner = mocha.run((failures) => { 112 | process.on('exit', () => { 113 | process.exit(failures) 114 | }) 115 | }) 116 | }) 117 | } 118 | 119 | module.exports = { 120 | command: 'box [output]', 121 | describe: 'Run Test Project on Botium Box and output test report with Mocha test runner', 122 | builder: (yargs) => { 123 | yargs.positional('output', { 124 | describe: 'Output report type (select Mocha reporter)', 125 | choices: outputTypes, 126 | default: 'spec' 127 | }) 128 | yargs.option('webhook', { 129 | describe: 'Botium Box Webhook Link for the Test Project, including the Botium Box API Key (also read from env variable "BOTIUM_BOX_WEBHOOK") (required)' 130 | }) 131 | yargs.option('buildid', { 132 | describe: 'Botium Box Build Identifier for the Test Project Build (also read from env variable "BOTIUM_BOX_BUILDID")' 133 | }) 134 | yargs.option('buildcomment', { 135 | describe: 'Botium Box Build Comment for Test Project Build (also read from env variable "BOTIUM_BOX_BUILDCOMMENT")' 136 | }) 137 | yargs.option('tags', { 138 | describe: 'Botium Box Tags to attach to the Test Project, can be specified more than once, ending in "--" ("... --tags Tag1 Tag2 Tag3 -- ...") (also read from env variables starting with "BOTIUM_BOX_TAGS")', 139 | array: true 140 | }) 141 | yargs.option('caps', { 142 | describe: 'Botium Capabilities to attach to the Test Project Build, in the form of "CAPABILITYNAME=CAPABILITYVALUE", can be specified more than once, ending in "--" ("... --caps CAP1=value1 CAP2=value2 -- ...") (also read from env variables starting with "BOTIUM_BOX_CAPS")', 143 | array: true 144 | }) 145 | yargs.option('sources', { 146 | describe: 'Botium Sources to attach to the Test Project Build, in the form of "SOURCESNAME=SOURCESVALUE", can be specified more than once, ending in "--" ("... --sources SOURCE1=value1 SOURCE2=value2 -- ...") (also read from env variables starting with "BOTIUM_BOX_SOURCES")', 147 | array: true 148 | }) 149 | yargs.option('envs', { 150 | describe: 'Botium Environment Variables to attach to the Test Project Build, in the form of "ENVNAME=ENVVALUE", can be specified more than once, ending in "--" ("... --envs ENV1=value1 ENV2=value2 -- ...") (also read from env variables starting with "BOTIUM_BOX_ENVS")', 151 | array: true 152 | }) 153 | yargs.option('reporter-options', { 154 | describe: 'Options for mocha reporter, either as JSON, or as key-value pairs ("option1=value1,option2=value2,..."). For details see documentation of the used mocha reporter.' 155 | }) 156 | yargs.option('timeout', { 157 | describe: 'Timeout in seconds for calling the Botium Box Webhook', 158 | number: true, 159 | default: 300 160 | }) 161 | }, 162 | handler 163 | } 164 | -------------------------------------------------------------------------------- /src/emulator/console/index.js: -------------------------------------------------------------------------------- 1 | const util = require('util') 2 | const fs = require('fs') 3 | const path = require('path') 4 | const term = require('terminal-kit').terminal 5 | const { mkdirpSync } = require('mkdirp') 6 | const figlet = require('figlet') 7 | const repl = require('repl') 8 | const slug = require('slug') 9 | const BotDriver = require('botium-core').BotDriver 10 | 11 | module.exports = (outputDir) => { 12 | const driver = new BotDriver() 13 | const compiler = driver.BuildCompiler() 14 | let container = null 15 | 16 | driver.Build().then((c) => { 17 | container = c 18 | return container.Start() 19 | }).then(() => { 20 | const conversation = [] 21 | 22 | driver.on('MESSAGE_RECEIVEDFROMBOT', (container, msg) => { 23 | if (msg) { 24 | if (!msg.sender) msg.sender = 'bot' 25 | if (msg.messageText) { 26 | term.cyan('BOT SAYS ' + (msg.channel ? '(' + msg.channel + '): ' : ': ') + msg.messageText + '\n') 27 | } 28 | if (msg.media && msg.media.length > 0) { 29 | term.cyan('BOT SENDS MEDIA ATTACHMENTS ' + (msg.channel ? '(' + msg.channel + '): ' : ': ') + '\n') 30 | msg.media.forEach(m => { 31 | term.cyan(' * URL: ' + m.mediaUri) 32 | m.mimeType && term.cyan(' MIMETYPE: ' + m.mimeType) 33 | m.altText && term.cyan(' ALTTEXT: ' + m.altText) 34 | term('\n') 35 | }) 36 | } 37 | if (msg.buttons && msg.buttons.length > 0) { 38 | term.cyan('BOT SENDS BUTTONS ' + (msg.channel ? '(' + msg.channel + '): ' : ': ') + '\n') 39 | msg.buttons.forEach(b => { 40 | term.cyan(' * TEXT: ' + b.text) 41 | b.payload && term.cyan(' PAYLOAD: ' + b.payload) 42 | term('\n') 43 | }) 44 | } 45 | if (msg.cards && msg.cards.length > 0) { 46 | term.cyan('BOT SENDS CARDS ' + (msg.channel ? '(' + msg.channel + '): ' : ': ') + '\n') 47 | msg.cards.forEach(c => { 48 | term.cyan(' ***********************************************\n') 49 | c.text && term.cyan(' * ' + c.text + '\n') 50 | c.image && term.cyan(' * IMAGE: ' + c.image.mediaUri + '\n') 51 | if (c.buttons && c.buttons.length > 0) { 52 | term.cyan(' ***********************************************\n') 53 | c.buttons.forEach(b => { 54 | term.cyan(' * BUTTON: ' + b.text) 55 | b.payload && term.cyan(' PAYLOAD: ' + b.payload) 56 | term('\n') 57 | }) 58 | } 59 | term.cyan(' ***********************************************\n') 60 | }) 61 | } 62 | 63 | if (!msg.messageText && !msg.media && !msg.buttons && !msg.cards && msg.sourceData) { 64 | term.cyan('BOT SAYS RICH MESSAGE ' + (msg.channel ? '(' + msg.channel + '): ' : ': \n')) 65 | term.cyan(JSON.stringify(msg.sourceData, null, 2)) 66 | term('\n') 67 | } 68 | conversation.push(msg) 69 | } 70 | }) 71 | 72 | term.fullscreen(true) 73 | term.yellow( 74 | figlet.textSync('BOTIUM', { horizontalLayout: 'full' }) 75 | ) 76 | term('\n') 77 | const helpText = 'Enter "#SAVE " to save your conversation into your convo-directory, #EXIT to quit or just a message to send to your Chatbot!\n' 78 | 79 | term.green('Chatbot online.\n') 80 | term.green(helpText) 81 | 82 | const evaluator = (line) => { 83 | if (line) line = line.trim() 84 | if (!line) return 85 | 86 | if (line.toLowerCase() === '#exit') { 87 | term.yellow('Botium stopping ...\n') 88 | container.Stop().then(() => container.Clean()).then(() => term.green('Botium stopped')).then(() => process.exit(0)).catch((err) => term.red(err)) 89 | } else if (line.toLowerCase().startsWith('#save')) { 90 | const name = line.substr(5).trim() 91 | if (!name) { 92 | term.red(helpText) 93 | return 94 | } 95 | const filename = path.resolve(outputDir, slug(name) + '.convo.txt') 96 | 97 | try { 98 | fs.accessSync(filename, fs.constants.R_OK) 99 | term.red('File ' + filename + ' already exists. Please choose another conversation name.\n') 100 | return 101 | } catch (err) { 102 | } 103 | 104 | try { 105 | mkdirpSync(outputDir) 106 | 107 | const scriptData = compiler.Decompile([{ header: { name }, conversation }], 'SCRIPTING_FORMAT_TXT') 108 | fs.writeFileSync(filename, scriptData) 109 | term.green('Conversation written to file ' + filename + '\n') 110 | conversation.length = 0 111 | } catch (err) { 112 | term.red(err) 113 | } 114 | } else if (line.startsWith('#')) { 115 | const channel = line.substr(0, line.indexOf(' ')) 116 | const text = line.substr(line.indexOf(' ') + 1) 117 | 118 | const msg = { messageText: text, sender: 'me', channel } 119 | 120 | container.UserSays(msg).catch((err) => term.red(util.inspect(err))) 121 | conversation.push(msg) 122 | } else { 123 | const msg = { messageText: line, sender: 'me' } 124 | 125 | container.UserSays(msg).catch((err) => term.red(util.inspect(err))) 126 | conversation.push(msg) 127 | } 128 | } 129 | repl.start({ prompt: '', eval: evaluator }) 130 | }).catch((err) => term.red(util.inspect(err))) 131 | } 132 | -------------------------------------------------------------------------------- /src/emulator/index.js: -------------------------------------------------------------------------------- 1 | const util = require('util') 2 | const yargsCmd = require('yargs') 3 | const debug = require('debug')('botium-cli-emulator') 4 | 5 | const handler = (argv) => { 6 | debug(`command options: ${util.inspect(argv)}`) 7 | 8 | if (!argv.ui) { 9 | return yargsCmd.showHelp() 10 | } 11 | 12 | if (argv.ui === 'console') { 13 | const emulator = require('./console') 14 | emulator(argv.convos[0]) 15 | } else { 16 | return yargsCmd.showHelp() 17 | } 18 | } 19 | 20 | module.exports = { 21 | command: 'emulator [ui]', 22 | describe: 'Launch Botium emulator', 23 | builder: (yargs) => { 24 | yargs.positional('ui', { 25 | describe: 'Emulator UI (terminal-based)', 26 | choices: ['console'], 27 | default: 'console' 28 | }) 29 | }, 30 | handler 31 | } 32 | -------------------------------------------------------------------------------- /src/hello/index.js: -------------------------------------------------------------------------------- 1 | const util = require('util') 2 | const BotDriver = require('botium-core').BotDriver 3 | const debug = require('debug')('botium-cli-hello') 4 | 5 | const fgRed = '\x1b[31m' 6 | const fgGreen = '\x1b[32m' 7 | 8 | const handler = async (argv) => { 9 | debug(`command options: ${util.inspect(argv)}`) 10 | const driver = new BotDriver() 11 | const compiler = driver.BuildCompiler() 12 | const convo = { 13 | header: { 14 | name: 'Connectivity check' 15 | }, 16 | conversation: [] 17 | } 18 | 19 | try { 20 | const container = await _startContainer(driver) 21 | 22 | const userMessage = { 23 | sender: 'me', 24 | messageText: argv.messageText 25 | } 26 | convo.conversation.push(userMessage) 27 | await container.UserSays(userMessage) 28 | convo.conversation.push(await container.WaitBotSays()) 29 | 30 | await _stopContainer(container) 31 | 32 | const script = compiler.Decompile([convo], 'SCRIPTING_FORMAT_TXT') 33 | console.log(`${script}-------------------------`) 34 | console.log(fgGreen, 'Connectivity check SUCCESS') 35 | } catch (err) { 36 | console.log(err.message || err) 37 | console.log(fgRed, 'Connectivity check FAILED') 38 | process.exit(1) 39 | } 40 | } 41 | 42 | const _startContainer = async (driver) => { 43 | const myContainer = await driver.Build() 44 | debug('Conversation container built, now starting') 45 | try { 46 | await myContainer.Start() 47 | debug('Conversation container started.') 48 | return myContainer 49 | } catch (err) { 50 | try { 51 | await myContainer.Stop() 52 | } catch (err) { 53 | debug(`Conversation Stop failed: ${err}`) 54 | } 55 | try { 56 | await myContainer.Clean() 57 | } catch (err) { 58 | debug(`Conversation Clean failed: ${err}`) 59 | } 60 | throw err 61 | } 62 | } 63 | 64 | const _stopContainer = async (container) => { 65 | if (container) { 66 | try { 67 | await container.Stop() 68 | } catch (err) { 69 | debug(`Conversation Stop failed: ${err}`) 70 | } 71 | try { 72 | await container.Clean() 73 | } catch (err) { 74 | debug(`Conversation Clean failed: ${err}`) 75 | } 76 | } 77 | debug('Conversation container stopped.') 78 | } 79 | 80 | module.exports = { 81 | command: 'hello', 82 | describe: 'Say "hello" to check connectivity with the configured chatbot.', 83 | builder: (yargs) => { 84 | yargs.option('messageText', { 85 | describe: 'The defined message text is sent to the bot.', 86 | default: 'hello' 87 | }) 88 | }, 89 | handler 90 | } 91 | -------------------------------------------------------------------------------- /src/import/index.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | const slug = require('slug') 3 | const fs = require('fs') 4 | const path = require('path') 5 | const { mkdirpSync } = require('mkdirp') 6 | const { BotDriver } = require('botium-core') 7 | 8 | const writeConvo = (compiler, convo, outputDir) => { 9 | const filename = path.resolve(outputDir, slug(convo.header.name) + '.convo.txt') 10 | 11 | mkdirpSync(outputDir) 12 | 13 | const scriptData = compiler.Decompile([convo], 'SCRIPTING_FORMAT_TXT') 14 | 15 | fs.writeFileSync(filename, scriptData) 16 | return filename 17 | } 18 | 19 | const writeUtterances = (compiler, utterance, samples, outputDir) => { 20 | const filename = path.resolve(outputDir, slug(utterance) + '.utterances.txt') 21 | 22 | mkdirpSync(outputDir) 23 | 24 | const scriptData = [utterance, ...samples].join('\n') 25 | 26 | fs.writeFileSync(filename, scriptData) 27 | return filename 28 | } 29 | 30 | module.exports = (connector, command, describe) => { 31 | try { 32 | require(connector) 33 | } catch (err) { 34 | return { 35 | command, 36 | describe: `Install NPM module "${connector}" to enable this command`, 37 | handler: () => ({}) 38 | } 39 | } 40 | 41 | const { Handler, Args } = require(connector).Import 42 | 43 | return { 44 | command, 45 | describe, 46 | builder: (yargs) => { 47 | for (const arg of Object.keys(Args)) { 48 | if (Args[arg].skipCli) continue 49 | yargs.option(arg, Args[arg]) 50 | } 51 | }, 52 | handler: async (argv) => { 53 | const outputDir = argv.convos[0] 54 | 55 | let convos = [] 56 | let utterances = [] 57 | try { 58 | ({ convos, utterances } = await Handler(argv)) 59 | } catch (err) { 60 | console.log(`FAILED: ${err.message}`) 61 | return 62 | } 63 | 64 | const driver = new BotDriver() 65 | const compiler = await driver.BuildCompiler() 66 | 67 | for (const convo of convos) { 68 | try { 69 | const filename = writeConvo(compiler, convo, outputDir) 70 | console.log(`SUCCESS: wrote convo to file ${filename}`) 71 | } catch (err) { 72 | console.log(`WARNING: writing convo "${convo.header.name}" failed: ${err.message}`) 73 | } 74 | } 75 | for (const utterance of utterances) { 76 | try { 77 | const filename = writeUtterances(compiler, utterance.name, utterance.utterances, outputDir) 78 | console.log(`SUCCESS: wrote utterances to file ${filename}`) 79 | } catch (err) { 80 | console.log(`WARNING: writing utterances "${utterance.name}" failed: ${err.message}`) 81 | } 82 | } 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | run: () => require('./run'), 3 | emulator: () => require('./emulator'), 4 | box: () => require('./box') 5 | } 6 | -------------------------------------------------------------------------------- /src/init-alexa-avs/index.js: -------------------------------------------------------------------------------- 1 | const util = require('util') 2 | const debug = require('debug')('botium-bindings-cli') 3 | const start = require('botium-connector-alexa-avs/src/tools/CreateCapabilitiesImpl').execute 4 | 5 | const handler = (argv) => { 6 | debug(`command options: ${util.inspect(argv)}`) 7 | start() 8 | } 9 | 10 | module.exports = { 11 | command: 'init-alexa-avs', 12 | describe: 'Run the "Botium Connector Alexa AVS Initialization Tool"', 13 | handler 14 | } 15 | -------------------------------------------------------------------------------- /src/init-alexa-smapi/index.js: -------------------------------------------------------------------------------- 1 | const args = require('botium-connector-alexa-smapi/src/init').args 2 | 3 | module.exports = Object.assign({}, args, { command: 'init-alexa-smapi' }) 4 | -------------------------------------------------------------------------------- /src/init-dev/asserter/botium.json: -------------------------------------------------------------------------------- 1 | { 2 | "botium": { 3 | "Capabilities": { 4 | "PROJECTNAME": "Botium Asserters Sample", 5 | "CONTAINERMODE": "echo", 6 | "ASSERTERS": [ 7 | { 8 | "ref": "MYASSERTER", 9 | "src": "./src/MyCustomAsserter.js", 10 | "global": false, 11 | "args": { 12 | "globalArg1": "globalArg1Value", 13 | "globalArg2": "globalArg2Value" 14 | } 15 | } 16 | ] 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /src/init-dev/asserter/spec/convos/echo.convo.txt: -------------------------------------------------------------------------------- 1 | echo 2 | 3 | #begin 4 | MYASSERTER arg1|arg2 5 | 6 | #me 7 | Hello 8 | 9 | #bot 10 | MYASSERTER arg1|arg2 11 | 12 | #me 13 | Hello again! 14 | 15 | #bot 16 | MYASSERTER arg1|arg2 17 | 18 | #end 19 | MYASSERTER arg1|arg2 20 | -------------------------------------------------------------------------------- /src/init-dev/asserter/src/MyCustomAsserter.js: -------------------------------------------------------------------------------- 1 | const utils = require('util') 2 | 3 | module.exports = class MyCustomAsserter { 4 | constructor (context, caps, globalArgs) { 5 | this.context = context 6 | this.caps = caps 7 | this.globalArgs = globalArgs 8 | console.log(`MyCustomAsserter constructor, globalArgs: ${utils.inspect(globalArgs)}`) 9 | } 10 | 11 | assertConvoBegin ({ convo, args, isGlobal }) { 12 | console.log(`MyCustomAsserter assertConvoBegin: ${convo.header.name}`) 13 | return Promise.resolve() 14 | } 15 | 16 | assertConvoStep ({ convo, convoStep, args, isGlobal, botMsg }) { 17 | console.log(`MyCustomAsserter assertConvoStep ${convo.header.name}/${convoStep.stepTag}, args: ${utils.inspect(args)}, botMessage: ${botMsg.messageText}`) 18 | return Promise.resolve() 19 | } 20 | 21 | assertConvoEnd ({ convo, transcript, args, isGlobal }) { 22 | console.log(`MyCustomAsserter assertConvoEnd ${convo.header.name}, conversation length: ${transcript.steps.length} steps`) 23 | return Promise.resolve() 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/init-dev/connector/botium.json: -------------------------------------------------------------------------------- 1 | { 2 | "botium": { 3 | "Capabilities": { 4 | "PROJECTNAME": "Botium Connectors Sample", 5 | "CONTAINERMODE": "./src/MyCustomConnector.js", 6 | "MYCUSTOMCONNECTOR_PREFIX": "You said: " 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /src/init-dev/connector/spec/convos/echo.convo.txt: -------------------------------------------------------------------------------- 1 | echo 2 | 3 | #me 4 | Hello 5 | 6 | #bot 7 | You said: Hello 8 | 9 | #me 10 | Hello again! 11 | 12 | #bot 13 | You said: Hello again! 14 | -------------------------------------------------------------------------------- /src/init-dev/connector/src/MyCustomConnector.js: -------------------------------------------------------------------------------- 1 | class MyCustomConnector { 2 | constructor ({ queueBotSays, caps }) { 3 | this.queueBotSays = queueBotSays 4 | this.caps = caps 5 | } 6 | 7 | UserSays (msg) { 8 | const requestObject = this._msgToRequestObject(msg) 9 | const responseObject = this._doRequestChatbotApi(requestObject) 10 | const botMsg = this._responseObjectToMsg(responseObject) 11 | console.log(`MyCustomConnector: ${msg.messageText} => ${botMsg.messageText}`) 12 | setTimeout(() => this.queueBotSays(botMsg), 0) 13 | } 14 | 15 | _msgToRequestObject (msg) { 16 | // TODO convert generic msg to chatbot specific requestObject 17 | return msg.messageText 18 | } 19 | 20 | _doRequestChatbotApi (requestObject) { 21 | // TODO request the Chatbot API using chatbot specific requestO0bject 22 | // and return bot response as responseObject 23 | return (this.caps.MYCUSTOMCONNECTOR_PREFIX || '') + requestObject 24 | } 25 | 26 | _responseObjectToMsg (msg) { 27 | // TODO convert chatbot specific requestObject to generic msg 28 | return { messageText: msg } 29 | } 30 | } 31 | 32 | module.exports = { 33 | PluginVersion: 1, 34 | PluginClass: MyCustomConnector 35 | } 36 | -------------------------------------------------------------------------------- /src/init-dev/index.js: -------------------------------------------------------------------------------- 1 | const util = require('util') 2 | const yargsCmd = require('yargs') 3 | const path = require('path') 4 | const fs = require('fs-extra') 5 | const debug = require('debug')('botium-cli-emulator') 6 | 7 | const handler = (argv) => { 8 | debug(`command options: ${util.inspect(argv)}`) 9 | 10 | if (!argv.project) { 11 | return yargsCmd.showHelp() 12 | } 13 | const srcPath = path.resolve(__dirname, argv.project) 14 | const targetPath = path.resolve(process.cwd()) 15 | fs.copySync(srcPath, targetPath) 16 | console.log(`Botium development project written to "${targetPath}". You should now run "botium-cli run --verbose --convos spec/convos" to verify.`) 17 | } 18 | 19 | module.exports = { 20 | command: 'init-dev [project]', 21 | describe: 'Setup a development project for Botium connectors, asserters or logic hooks in the current directory', 22 | builder: (yargs) => { 23 | yargs.positional('project', { 24 | describe: 'Project type', 25 | choices: ['connector', 'asserter', 'logichook'], 26 | default: 'asserter' 27 | }) 28 | }, 29 | handler 30 | } 31 | -------------------------------------------------------------------------------- /src/init-dev/logichook/botium.json: -------------------------------------------------------------------------------- 1 | { 2 | "botium": { 3 | "Capabilities": { 4 | "PROJECTNAME": "Botium Logic Hook Sample", 5 | "CONTAINERMODE": "echo", 6 | "LOGIC_HOOKS": [ 7 | { 8 | "ref": "MYLOGICHOOK", 9 | "src": "./src/MyCustomLogicHook.js", 10 | "global": false, 11 | "args": { 12 | "globalArg1": "globalArg1Value", 13 | "globalArg2": "globalArg2Value" 14 | } 15 | }, 16 | { 17 | "ref": "MYGLOBALLOGICHOOK", 18 | "src": "./src/MyGlobalLogicHook.js", 19 | "global": true, 20 | "args": { 21 | "globalArg1": "globalArg1Value", 22 | "globalArg2": "globalArg2Value" 23 | } 24 | } 25 | ] 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /src/init-dev/logichook/spec/convos/echo.convo.txt: -------------------------------------------------------------------------------- 1 | echo 2 | 3 | #me 4 | Hello 5 | 6 | #bot 7 | You said: Hello 8 | MYLOGICHOOK arg1|arg2 9 | 10 | #me 11 | Hello again! 12 | 13 | #bot 14 | You said: Hello again! 15 | MYLOGICHOOK arg1|arg2 16 | 17 | #end 18 | MYLOGICHOOK arg1|arg2 19 | -------------------------------------------------------------------------------- /src/init-dev/logichook/src/MyCustomLogicHook.js: -------------------------------------------------------------------------------- 1 | const utils = require('util') 2 | 3 | /** 4 | * This is a custom logic hook 5 | * It has to be referenced in the convo file to get active 6 | */ 7 | module.exports = class MyCustomLogicHook { 8 | constructor (context, caps, globalArgs) { 9 | this.context = context 10 | this.caps = caps 11 | this.globalArgs = globalArgs 12 | console.log(`MyCustomLogicHook constructor, globalArgs: ${utils.inspect(globalArgs)}`) 13 | } 14 | 15 | onConvoBegin ({ convo, args }) { 16 | console.log(`MyCustomLogicHook onConvoBegin: ${convo.header.name}`) 17 | } 18 | 19 | onConvoEnd ({ convo, transcript, args }) { 20 | console.log(`MyCustomLogicHook onConvoEnd ${convo.header.name}, conversation length: ${transcript.steps.length} steps`) 21 | } 22 | 23 | onMeStart ({ convo, convoStep, args }) { 24 | console.log(`MyCustomLogicHook onMeStart ${convo.header.name}/${convoStep.stepTag}, args: ${utils.inspect(args)}, meMessage: ${convoStep.messageText}`) 25 | } 26 | 27 | onMeEnd ({ convo, convoStep, args }) { 28 | console.log(`MyCustomLogicHook onMeEnd ${convo.header.name}/${convoStep.stepTag}, args: ${utils.inspect(args)}, meMessage: ${convoStep.messageText}`) 29 | } 30 | 31 | onBotStart ({ convo, convoStep, args }) { 32 | console.log(`MyCustomLogicHook onBotStart ${convo.header.name}/${convoStep.stepTag}, args: ${utils.inspect(args)}, expected: ${convoStep.messageText}`) 33 | } 34 | 35 | onBotEnd ({ convo, convoStep, botMsg, args }) { 36 | console.log(`MyCustomLogicHook onBotEnd ${convo.header.name}/${convoStep.stepTag}, args: ${utils.inspect(args)}, expected: ${convoStep.messageText}, meMessage: ${botMsg.messageText}`) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/init-dev/logichook/src/MyGlobalLogicHook.js: -------------------------------------------------------------------------------- 1 | const utils = require('util') 2 | 3 | /** 4 | * This is a custom global logic hook 5 | * The hook functions are evaluated for all convos 6 | */ 7 | module.exports = class MyGlobalLogicHook { 8 | constructor (context, caps, globalArgs) { 9 | this.context = context 10 | this.caps = caps 11 | this.globalArgs = globalArgs 12 | console.log(`MyGlobalLogicHook constructor, globalArgs: ${utils.inspect(globalArgs)}`) 13 | } 14 | 15 | onConvoBegin ({ convo, args }) { 16 | console.log(`MyGlobalLogicHook onConvoBegin: ${convo.header.name}`) 17 | } 18 | 19 | onConvoEnd ({ convo, transcript, args }) { 20 | console.log(`MyGlobalLogicHook onConvoEnd ${convo.header.name}, conversation length: ${transcript.steps.length} steps`) 21 | } 22 | 23 | onMeStart ({ convo, convoStep, args }) { 24 | console.log(`MyGlobalLogicHook onMeStart ${convo.header.name}/${convoStep.stepTag}, args: ${utils.inspect(args)}, meMessage: ${convoStep.messageText}`) 25 | } 26 | 27 | onMeEnd ({ convo, convoStep, args }) { 28 | console.log(`MyGlobalLogicHook onMeEnd ${convo.header.name}/${convoStep.stepTag}, args: ${utils.inspect(args)}, meMessage: ${convoStep.messageText}`) 29 | } 30 | 31 | onBotStart ({ convo, convoStep, args }) { 32 | console.log(`MyGlobalLogicHook onBotStart ${convo.header.name}/${convoStep.stepTag}, args: ${utils.inspect(args)}, expected: ${convoStep.messageText}`) 33 | } 34 | 35 | onBotEnd ({ convo, convoStep, botMsg, args }) { 36 | console.log(`MyGlobalLogicHook onBotEnd ${convo.header.name}/${convoStep.stepTag}, args: ${utils.inspect(args)}, expected: ${convoStep.messageText}, meMessage: ${botMsg.messageText}`) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/init/index.js: -------------------------------------------------------------------------------- 1 | const util = require('util') 2 | const fs = require('fs') 3 | const path = require('path') 4 | const { mkdirpSync } = require('mkdirp') 5 | const debug = require('debug')('botium-bindings-cli') 6 | 7 | const handler = (argv) => { 8 | debug(`command options: ${util.inspect(argv)}`) 9 | 10 | const botiumConvoDir = argv.convos[0] 11 | if (!fs.existsSync(botiumConvoDir)) { 12 | mkdirpSync(botiumConvoDir) 13 | } 14 | 15 | const botiumJsonFile = argv.config 16 | if (fs.existsSync(botiumJsonFile)) { 17 | console.log(`Botium Configuration File "${botiumJsonFile}" already present, skipping ...`) 18 | } else { 19 | fs.writeFileSync(botiumJsonFile, JSON.stringify({ 20 | botium: { 21 | Capabilities: { 22 | PROJECTNAME: 'My Botium Project', 23 | CONTAINERMODE: 'echo' 24 | }, 25 | Sources: { }, 26 | Envs: { } 27 | } 28 | }, null, 2)) 29 | console.log(`Botium Configuration File written to "${botiumJsonFile}".`) 30 | } 31 | 32 | const botiumEchoSample = path.resolve(botiumConvoDir, 'give_me_a_picture.convo.txt') 33 | if (fs.existsSync(botiumEchoSample)) { 34 | console.log(`Botium Convo File "${botiumEchoSample}" already present, skipping ...`) 35 | } else { 36 | fs.writeFileSync(botiumEchoSample, 37 | `give me picture 38 | 39 | #me 40 | Hello, Bot! 41 | 42 | #bot 43 | You said: Hello, Bot! 44 | 45 | #me 46 | give me a picture 47 | 48 | #bot 49 | Here is a picture 50 | MEDIA logo.png 51 | ` 52 | ) 53 | console.log(`Botium Convo File written to "${botiumEchoSample}".`) 54 | } 55 | console.log(`Botium initialization ready. You should now run "botium-cli run --verbose --convos ${botiumConvoDir}" to verify.`) 56 | } 57 | 58 | module.exports = { 59 | command: 'init', 60 | describe: 'Setup a directory for Botium usage', 61 | handler 62 | } 63 | -------------------------------------------------------------------------------- /src/metrics.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const os = require('os') 3 | 4 | const botiumAnalyticsHost = process.env.BOTIUM_ANALYTICS_HOST || 'v1.license.botium.cyaraportal.us' 5 | const botiumAnalyticsPort = process.env.BOTIUM_ANALYTICS_PORT || 443 6 | const https = botiumAnalyticsPort === 443 ? require('https') : require('http') 7 | const execTimeout = 10000 8 | 9 | function logIfVerbose (toLog, stream) { 10 | if (process.env.BOTIUM_ANALYTICS_VERBOSE === 'true') { 11 | (stream || console.log)(toLog) 12 | } 13 | } 14 | 15 | async function reportUsage (metrics) { 16 | if (process.env.BOTIUM_ANALYTICS === 'false') return 17 | 18 | const packageJson = require(path.join(__dirname, '..', 'package.json')) 19 | 20 | const infoPayload = Object.assign({ 21 | rawPlatform: os.platform(), 22 | rawArch: os.arch(), 23 | library: packageJson.name, 24 | version: packageJson.version 25 | }, metrics) 26 | 27 | const data = JSON.stringify(infoPayload) 28 | logIfVerbose(`Botium analytics payload: ${data}`) 29 | 30 | const reqOptions = { 31 | host: botiumAnalyticsHost, 32 | port: botiumAnalyticsPort, 33 | method: 'POST', 34 | path: '/metrics/usage/core', 35 | headers: { 36 | 'Content-Type': 'application/json', 37 | 'Content-Length': data.length 38 | }, 39 | timeout: execTimeout 40 | } 41 | await new Promise((resolve, reject) => { 42 | const req = https.request(reqOptions, (res) => { 43 | logIfVerbose(`Response status: ${res.statusCode}`) 44 | resolve() 45 | }) 46 | 47 | req.on('error', error => { 48 | logIfVerbose(error, console.error) 49 | resolve() 50 | }) 51 | 52 | req.on('timeout', error => { 53 | logIfVerbose(error, console.error) 54 | resolve() 55 | }) 56 | 57 | req.write(data) 58 | req.end() 59 | }) 60 | } 61 | 62 | module.exports = { 63 | reportUsage: (metrics) => { 64 | reportUsage(metrics).catch(e => { 65 | logIfVerbose(`metrics error: ${e}`, console.error) 66 | }) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/nlp/extract.js: -------------------------------------------------------------------------------- 1 | const util = require('util') 2 | const path = require('path') 3 | const fs = require('fs') 4 | const slug = require('slug') 5 | const { BotDriver } = require('botium-core') 6 | const debug = require('debug')('botium-cli-nlp') 7 | 8 | const { getConnector } = require('./index') 9 | 10 | const extract = async (argv) => { 11 | const driver = new BotDriver() 12 | const pluginConnector = getConnector(driver.caps.CONTAINERMODE) 13 | 14 | if (!pluginConnector || !pluginConnector.NLP) { 15 | console.log(`NLP Analytics not supported by connector ${driver.caps.CONTAINERMODE}`) 16 | } 17 | const { ExtractIntentUtterances } = pluginConnector.NLP 18 | 19 | let extractedIntents = null 20 | try { 21 | extractedIntents = await ExtractIntentUtterances({}) 22 | debug(`Extracted intent utterances: ${JSON.stringify(extractedIntents.intents.map(i => ({ intentName: i.intentName, utterances: i.utterances.length })), null, 2)}`) 23 | } catch (err) { 24 | console.log(`Failed to extract utterances: ${err.message}`) 25 | return 26 | } 27 | 28 | for (const e of extractedIntents.intents) { 29 | const filename = path.resolve(argv.convos[0], `${slug(e.intentName)}.utterances.txt`) 30 | try { 31 | fs.writeFileSync(filename, [ 32 | e.intentName, 33 | ...e.utterances 34 | ].join('\r\n')) 35 | console.log(`Extracted intent utterances for ${e.intentName} to ${filename}`) 36 | } catch (err) { 37 | console.log(`Failed to extract intent utterances for ${e.intentName} to ${filename}: ${err.message}`) 38 | } 39 | } 40 | } 41 | 42 | const handler = (argv) => { 43 | debug(`command options: ${util.inspect(argv)}`) 44 | extract(argv) 45 | } 46 | 47 | module.exports = { 48 | command: 'nlpextract', 49 | describe: 'Extract utterances from connector workspace for NLP Analytics', 50 | builder: (yargs) => { 51 | }, 52 | handler 53 | } 54 | -------------------------------------------------------------------------------- /src/nlp/index.js: -------------------------------------------------------------------------------- 1 | const util = require('util') 2 | const yargsCmd = require('yargs') 3 | const fs = require('fs') 4 | const _ = require('lodash') 5 | const { BotDriver } = require('botium-core') 6 | const debug = require('debug')('botium-cli-nlp') 7 | 8 | const getConnector = (containermode) => { 9 | if (!containermode) { 10 | console.log('CONTAINERMODE capability empty, Botium connector cannot be loaded.') 11 | process.exit(1) 12 | } else if (containermode.startsWith('botium-connector')) { 13 | return require(containermode) 14 | } else { 15 | return require(`botium-connector-${containermode}`) 16 | } 17 | } 18 | 19 | const splitArray = (array = [], nPieces = 1) => { 20 | const splitArray = [] 21 | let atArrPos = 0 22 | for (let i = 0; i < nPieces; i++) { 23 | const splitArrayLength = Math.ceil((array.length - atArrPos) / (nPieces - i)) 24 | splitArray.push([]) 25 | splitArray[i] = array.slice(atArrPos, splitArrayLength + atArrPos) 26 | atArrPos += splitArrayLength 27 | } 28 | return splitArray 29 | } 30 | 31 | const makeFolds = (intents, k, shuffle, intentFilter) => { 32 | const result = [] 33 | 34 | for (let i = 0; i < k; i++) { 35 | const chunk = intents.map(intent => { 36 | if (intent.utterances.length < k) { 37 | debug(`Intent ${intent.intentName} has too less utterances (${intent.utterances.length}), no folds created.`) 38 | return { 39 | intentName: intent.intentName, 40 | train: intent.utterances 41 | } 42 | } 43 | if (intentFilter && intentFilter.indexOf(intent.intentName) < 0) { 44 | debug(`Intent ${intent.intentName} is not in filter, no folds created.`) 45 | return { 46 | intentName: intent.intentName, 47 | train: intent.utterances 48 | } 49 | } 50 | const chunks = splitArray(shuffle ? _.shuffle(intent.utterances) : intent.utterances, k) 51 | return { 52 | intentName: intent.intentName, 53 | test: chunks[i], 54 | train: _.flatten(_.filter(chunks, (s, chunkIndex) => chunkIndex !== i)) 55 | } 56 | }) 57 | result.push(chunk) 58 | } 59 | return result 60 | } 61 | 62 | const handler = (argv) => { 63 | debug(`command options: ${util.inspect(argv)}`) 64 | 65 | const driver = new BotDriver() 66 | const compiler = driver.BuildCompiler() 67 | const pluginConnector = getConnector(driver.caps.CONTAINERMODE) 68 | 69 | if (!pluginConnector || !pluginConnector.NLP) { 70 | console.log(`NLP Analytics not supported by connector ${driver.caps.CONTAINERMODE}`) 71 | } 72 | const { ExtractIntentUtterances, TrainIntentUtterances, CleanupIntentUtterances } = pluginConnector.NLP 73 | 74 | if (argv.algorithm === 'validate') { 75 | // eslint-disable-next-line no-unexpected-multiline 76 | (async () => { 77 | argv.output = argv.output || 'validate.csv' 78 | argv.outputPredictions = argv.outputPredictions || 'validate-predictions.csv' 79 | if (!argv.train || !argv.test) { 80 | return yargsCmd.showHelp() 81 | } 82 | 83 | compiler.ReadScriptsFromDirectory(argv.train) 84 | const trainingIntents = Object.keys(compiler.utterances).map(intentName => ({ 85 | intentName, 86 | utterances: compiler.utterances[intentName].utterances 87 | })) 88 | debug(`Read training intent utterances: ${JSON.stringify(trainingIntents.map(i => ({ intentName: i.intentName, utterances: i.utterances.length })), null, 2)}`) 89 | 90 | const testCompiler = driver.BuildCompiler() 91 | testCompiler.ReadScriptsFromDirectory(argv.test) 92 | const testIntents = Object.keys(testCompiler.utterances).map(intentName => ({ 93 | intentName, 94 | utterances: testCompiler.utterances[intentName].utterances 95 | })) 96 | debug(`Read test intent utterances: ${JSON.stringify(testIntents.map(i => ({ intentName: i.intentName, utterances: i.utterances.length })), null, 2)}`) 97 | 98 | const predictionDetails = [] 99 | const validateMatrix = {} 100 | 101 | console.log('Starting training.') 102 | let trainResult = null 103 | try { 104 | trainResult = await TrainIntentUtterances({}, trainingIntents, {}) 105 | } catch (err) { 106 | console.log(`Validation training failed: ${err.message}`) 107 | return 108 | } 109 | 110 | console.log('Starting testing.') 111 | try { 112 | const intentPromises = testIntents.map(async (testIntent) => { 113 | testIntent.predictions = {} 114 | 115 | const intentDriver = new BotDriver(trainResult.caps) 116 | const intentContainer = await intentDriver.Build() 117 | 118 | for (const utt of testIntent.utterances) { 119 | try { 120 | await intentContainer.Start() 121 | 122 | await intentContainer.UserSaysText(utt) 123 | const botMsg = await intentContainer.WaitBotSays() 124 | 125 | const predictedIntentName = _.get(botMsg, 'nlp.intent.name') || 'None' 126 | const mappedIntentName = (trainResult.trainedIntents && (trainResult.trainedIntents.find(ti => ti.mapFromIntentName === predictedIntentName) || {}).intentName) || predictedIntentName 127 | 128 | if (mappedIntentName) { 129 | testIntent.predictions[mappedIntentName] = (testIntent.predictions[mappedIntentName] || 0) + 1 130 | } 131 | 132 | testIntent.techok = (testIntent.techok || 0) + 1 133 | predictionDetails.push({ 134 | match: (testIntent.intentName === mappedIntentName) ? 'Y' : 'N', 135 | utterance: utt, 136 | expectedIntent: testIntent.intentName, 137 | predictedIntent: predictedIntentName 138 | }) 139 | 140 | await intentContainer.Stop() 141 | } catch (err) { 142 | testIntent.techfailures = (testIntent.techfailures || 0) + 1 143 | predictionDetails.push({ 144 | match: 'N', 145 | utterance: utt, 146 | expectedIntent: testIntent.intentName, 147 | predictedIntent: null 148 | }) 149 | 150 | console.log(`Validate: Failed sending utterance "${utt}" - ${err.message}`) 151 | } 152 | } 153 | await intentContainer.Clean() 154 | }) 155 | await Promise.all(intentPromises) 156 | 157 | const expectedIntents = {} 158 | for (const testIntent of testIntents) { 159 | expectedIntents[testIntent.intentName] = testIntent.predictions 160 | } 161 | const allIntentNames = testIntents.map(fi => fi.intentName).concat(['None']) 162 | for (const intentName of allIntentNames) { 163 | expectedIntents[intentName] = expectedIntents[intentName] || { } 164 | } 165 | 166 | const matrix = [] 167 | for (const testIntent of testIntents) { 168 | const totalPredicted = allIntentNames.reduce((agg, otherIntentName) => { 169 | return agg + (expectedIntents[otherIntentName][testIntent.intentName] || 0) 170 | }, 0) 171 | const totalExpected = allIntentNames.reduce((agg, otherIntentName) => { 172 | return agg + (testIntent.predictions[otherIntentName] || 0) 173 | }, 0) 174 | 175 | const score = { 176 | techok: testIntent.techok || 0, 177 | techfailures: testIntent.techfailures || 0 178 | } 179 | 180 | if (totalPredicted === 0) { 181 | score.precision = 0 182 | } else { 183 | score.precision = (testIntent.predictions[testIntent.intentName] || 0) / totalPredicted 184 | } 185 | if (totalExpected === 0) { 186 | score.recall = 0 187 | } else { 188 | score.recall = (testIntent.predictions[testIntent.intentName] || 0) / totalExpected 189 | } 190 | if (score.precision === 0 && score.recall === 0) { 191 | score.F1 = 0 192 | } else { 193 | score.F1 = 2 * ((score.precision * score.recall) / (score.precision + score.recall)) 194 | } 195 | 196 | matrix.push({ 197 | intent: testIntent.intentName, 198 | predictions: testIntent.predictions, 199 | score 200 | }) 201 | } 202 | 203 | Object.assign(validateMatrix, { 204 | techok: matrix.reduce((sum, r) => sum + r.score.techok, 0), 205 | techfailures: matrix.reduce((sum, r) => sum + r.score.techfailures, 0), 206 | precision: matrix.reduce((sum, r) => sum + r.score.precision, 0) / matrix.length, 207 | recall: matrix.reduce((sum, r) => sum + r.score.recall, 0) / matrix.length, 208 | matrix 209 | }) 210 | if (validateMatrix.precision === 0 && validateMatrix.recall === 0) { 211 | validateMatrix.F1 = 0 212 | } else { 213 | validateMatrix.F1 = 2 * ((validateMatrix.precision * validateMatrix.recall) / (validateMatrix.precision + validateMatrix.recall)) 214 | } 215 | } catch (err) { 216 | console.log(`Validate testing failed: ${err.message}`) 217 | } finally { 218 | try { 219 | await CleanupIntentUtterances({}, trainResult) 220 | } catch (err) { 221 | console.log(`Validate training failed: ${err.message}`) 222 | } 223 | } 224 | 225 | console.log('############# Summary #############') 226 | console.log(`Validate: Precision=${validateMatrix.precision.toFixed(4)} Recall=${validateMatrix.recall.toFixed(4)} F1-Score=${validateMatrix.F1.toFixed(4)} Tech.OK=${validateMatrix.techok} Tech.Failures=${validateMatrix.techfailures}`) 227 | 228 | const csvLines = [ 229 | ['intent', 'precision', 'recall', 'F1', 'Tech.OK', 'Tech.Failures'].join(';') 230 | ] 231 | for (let m = 0; m < validateMatrix.matrix.length; m++) { 232 | const matrix = validateMatrix.matrix[m] 233 | csvLines.push([ 234 | matrix.intent, 235 | matrix.score.precision.toFixed(4), 236 | matrix.score.recall.toFixed(4), 237 | matrix.score.F1.toFixed(4), 238 | matrix.score.techok, 239 | matrix.score.techfailures 240 | ].join(';')) 241 | } 242 | try { 243 | fs.writeFileSync(argv.output, csvLines.join('\r\n')) 244 | console.log(`Wrote output file ${argv.output}`) 245 | } catch (err) { 246 | console.log(`Failed to write output file ${argv.output} - ${err.message}`) 247 | } 248 | 249 | const csvLinesPredictions = [ 250 | ['match', 'utterance', 'expectedIntent', 'predictedIntent'].join(';') 251 | ].concat(predictionDetails.map(d => [ 252 | d.match, 253 | d.utterance || '', 254 | d.expectedIntent || '', 255 | d.predictedIntent || '' 256 | ].join(';') 257 | )) 258 | try { 259 | fs.writeFileSync(argv.outputPredictions, csvLinesPredictions.join('\r\n')) 260 | console.log(`Wrote predictions output file ${argv.outputPredictions}`) 261 | } catch (err) { 262 | console.log(`Failed to write predictions output file ${argv.outputPredictions} - ${err.message}`) 263 | } 264 | })() 265 | } 266 | if (argv.algorithm === 'k-fold') { 267 | // eslint-disable-next-line no-unexpected-multiline 268 | (async () => { 269 | argv.output = argv.output || 'k-fold.csv' 270 | argv.outputPredictions = argv.outputPredictions || 'k-fold-predictions.csv' 271 | 272 | let extractedIntents = null 273 | if (argv.extract) { 274 | try { 275 | debug('Extracting utterances ...') 276 | extractedIntents = await ExtractIntentUtterances({}) 277 | } catch (err) { 278 | console.log(`K-Fold failed to extract utterances: ${err.message}`) 279 | return 280 | } 281 | } else { 282 | argv.convos.forEach((convodir) => { 283 | compiler.ReadScriptsFromDirectory(convodir) 284 | }) 285 | extractedIntents = { 286 | intents: Object.keys(compiler.utterances).map(intentName => ({ 287 | intentName, 288 | utterances: compiler.utterances[intentName].utterances 289 | })) 290 | } 291 | } 292 | const originalIntents = extractedIntents.intents 293 | debug(`Extracted intent utterances: ${JSON.stringify(originalIntents.map(i => ({ intentName: i.intentName, utterances: i.utterances.length })), null, 2)}`) 294 | 295 | const folds = makeFolds(originalIntents, argv.k, argv.shuffle, argv.intents) 296 | console.log(`Created ${argv.k} folds (shuffled: ${argv.shuffle})`) 297 | 298 | const foldMatrices = [] 299 | const predictionDetails = [] 300 | 301 | for (let k = 0; k < folds.length; k++) { 302 | const foldIntents = folds[k] 303 | 304 | const trainingData = foldIntents.map(fi => ({ 305 | intentName: fi.intentName, 306 | utterances: fi.train 307 | })) 308 | 309 | console.log(`Starting training for fold ${k + 1}`) 310 | let trainResult = null 311 | try { 312 | trainResult = await TrainIntentUtterances({}, trainingData, extractedIntents) 313 | } catch (err) { 314 | console.log(`K-Fold training for fold ${k + 1} failed: ${err.message}`) 315 | return 316 | } 317 | 318 | console.log(`Starting testing for fold ${k + 1}`) 319 | try { 320 | const testIntents = foldIntents.filter(fi => fi.test) 321 | 322 | const intentPromises = testIntents.map(async (foldIntent) => { 323 | foldIntent.predictions = {} 324 | 325 | const foldDriver = new BotDriver(trainResult.caps) 326 | const foldContainer = await foldDriver.Build() 327 | 328 | for (const utt of foldIntent.test) { 329 | try { 330 | await foldContainer.Start() 331 | 332 | await foldContainer.UserSaysText(utt) 333 | const botMsg = await foldContainer.WaitBotSays() 334 | 335 | const predictedIntentName = _.get(botMsg, 'nlp.intent.name') || 'None' 336 | const mappedIntentName = (trainResult.trainedIntents && (trainResult.trainedIntents.find(ti => ti.mapFromIntentName === predictedIntentName) || {}).intentName) || predictedIntentName 337 | 338 | if (mappedIntentName) { 339 | foldIntent.predictions[mappedIntentName] = (foldIntent.predictions[mappedIntentName] || 0) + 1 340 | } 341 | 342 | foldIntent.techok = (foldIntent.techok || 0) + 1 343 | predictionDetails.push({ 344 | fold: k, 345 | match: (foldIntent.intentName === mappedIntentName) ? 'Y' : 'N', 346 | utterance: utt, 347 | expectedIntent: foldIntent.intentName, 348 | predictedIntent: predictedIntentName 349 | }) 350 | 351 | await foldContainer.Stop() 352 | } catch (err) { 353 | foldIntent.techfailures = (foldIntent.techfailures || 0) + 1 354 | predictionDetails.push({ 355 | fold: k, 356 | match: 'N', 357 | utterance: utt, 358 | expectedIntent: foldIntent.intentName, 359 | predictedIntent: null 360 | }) 361 | 362 | console.log(`K-Fold Round ${k + 1}: Failed sending utterance "${utt}" - ${err.message}`) 363 | } 364 | } 365 | await foldContainer.Clean() 366 | }) 367 | await Promise.all(intentPromises) 368 | 369 | const expectedIntents = {} 370 | for (const testIntent of testIntents) { 371 | expectedIntents[testIntent.intentName] = testIntent.predictions 372 | } 373 | const allIntentNames = foldIntents.map(fi => fi.intentName).concat(['None']) 374 | for (const intentName of allIntentNames) { 375 | expectedIntents[intentName] = expectedIntents[intentName] || { } 376 | } 377 | 378 | const matrix = [] 379 | for (const testIntent of testIntents) { 380 | const totalPredicted = allIntentNames.reduce((agg, otherIntentName) => { 381 | return agg + (expectedIntents[otherIntentName][testIntent.intentName] || 0) 382 | }, 0) 383 | const totalExpected = allIntentNames.reduce((agg, otherIntentName) => { 384 | return agg + (testIntent.predictions[otherIntentName] || 0) 385 | }, 0) 386 | 387 | const score = { 388 | techok: testIntent.techok || 0, 389 | techfailures: testIntent.techfailures || 0 390 | } 391 | 392 | if (totalPredicted === 0) { 393 | score.precision = 0 394 | } else { 395 | score.precision = (testIntent.predictions[testIntent.intentName] || 0) / totalPredicted 396 | } 397 | if (totalExpected === 0) { 398 | score.recall = 0 399 | } else { 400 | score.recall = (testIntent.predictions[testIntent.intentName] || 0) / totalExpected 401 | } 402 | if (score.precision === 0 && score.recall === 0) { 403 | score.F1 = 0 404 | } else { 405 | score.F1 = 2 * ((score.precision * score.recall) / (score.precision + score.recall)) 406 | } 407 | 408 | matrix.push({ 409 | intent: testIntent.intentName, 410 | predictions: testIntent.predictions, 411 | score 412 | }) 413 | } 414 | 415 | const foldMatrix = { 416 | techok: matrix.reduce((sum, r) => sum + r.score.techok, 0), 417 | techfailures: matrix.reduce((sum, r) => sum + r.score.techfailures, 0), 418 | precision: matrix.reduce((sum, r) => sum + r.score.precision, 0) / matrix.length, 419 | recall: matrix.reduce((sum, r) => sum + r.score.recall, 0) / matrix.length, 420 | matrix 421 | } 422 | if (foldMatrix.precision === 0 && foldMatrix.recall === 0) { 423 | foldMatrix.F1 = 0 424 | } else { 425 | foldMatrix.F1 = 2 * ((foldMatrix.precision * foldMatrix.recall) / (foldMatrix.precision + foldMatrix.recall)) 426 | } 427 | foldMatrices.push(foldMatrix) 428 | 429 | console.log(`K-Fold Round ${k + 1}: Precision=${foldMatrix.precision.toFixed(4)} Recall=${foldMatrix.recall.toFixed(4)} F1-Score=${foldMatrix.F1.toFixed(4)} Tech.OK=${foldMatrix.techok} Tech.Failures=${foldMatrix.techfailures}`) 430 | } catch (err) { 431 | console.log(`K-Fold testing for fold ${k + 1} failed: ${err.message}`) 432 | } finally { 433 | try { 434 | await CleanupIntentUtterances({}, trainResult) 435 | } catch (err) { 436 | console.log(`K-Fold training for fold ${k + 1} failed: ${err.message}`) 437 | } 438 | } 439 | } 440 | const avgPrecision = foldMatrices.reduce((sum, r) => sum + r.precision, 0) / foldMatrices.length 441 | const avgRecall = foldMatrices.reduce((sum, r) => sum + r.recall, 0) / foldMatrices.length 442 | let avgF1 = 0 443 | if (avgPrecision !== 0 || avgRecall !== 0) { 444 | avgF1 = 2 * ((avgPrecision * avgRecall) / (avgPrecision + avgRecall)) 445 | } 446 | 447 | console.log('############# Summary #############') 448 | for (let k = 0; k < foldMatrices.length; k++) { 449 | const foldMatrix = foldMatrices[k] 450 | console.log(`K-Fold Round ${k + 1}: Precision=${foldMatrix.precision.toFixed(4)} Recall=${foldMatrix.recall.toFixed(4)} F1-Score=${foldMatrix.F1.toFixed(4)} Tech.OK=${foldMatrix.techok} Tech.Failures=${foldMatrix.techfailures}`) 451 | } 452 | console.log(`K-Fold Avg: Precision=${avgPrecision.toFixed(4)} Recall=${avgRecall.toFixed(4)} F1-Score=${avgF1.toFixed(4)}`) 453 | 454 | const csvLines = [ 455 | ['fold', 'intent', 'precision', 'recall', 'F1', 'Tech.OK', 'Tech.Failures'].join(';') 456 | ] 457 | for (let k = 0; k < foldMatrices.length; k++) { 458 | const foldMatrix = foldMatrices[k] 459 | for (let m = 0; m < foldMatrix.matrix.length; m++) { 460 | const matrix = foldMatrix.matrix[m] 461 | csvLines.push([ 462 | `${k + 1}`, 463 | matrix.intent, 464 | matrix.score.precision.toFixed(4), 465 | matrix.score.recall.toFixed(4), 466 | matrix.score.F1.toFixed(4), 467 | matrix.score.techok, 468 | matrix.score.techfailures 469 | ].join(';')) 470 | } 471 | } 472 | try { 473 | fs.writeFileSync(argv.output, csvLines.join('\r\n')) 474 | console.log(`Wrote output file ${argv.output}`) 475 | } catch (err) { 476 | console.log(`Failed to write output file ${argv.output} - ${err.message}`) 477 | } 478 | 479 | const csvLinesPredictions = [ 480 | ['fold', 'match', 'utterance', 'expectedIntent', 'predictedIntent'].join(';') 481 | ].concat(predictionDetails.map(d => [ 482 | `${d.fold + 1}`, 483 | d.match, 484 | d.utterance || '', 485 | d.expectedIntent || '', 486 | d.predictedIntent || '' 487 | ].join(';') 488 | )) 489 | try { 490 | fs.writeFileSync(argv.outputPredictions, csvLinesPredictions.join('\r\n')) 491 | console.log(`Wrote predictions output file ${argv.outputPredictions}`) 492 | } catch (err) { 493 | console.log(`Failed to write predictions output file ${argv.outputPredictions} - ${err.message}`) 494 | } 495 | })() 496 | } 497 | } 498 | 499 | module.exports = { 500 | command: 'nlpanalytics ', 501 | describe: 'Run Botium NLP Analytics', 502 | builder: (yargs) => { 503 | yargs.positional('algorithm', { 504 | describe: 'NLP Analytics Algorithm to use. "validate" - train with training set, evaluate with test set. "k-fold" - k-fold monte-carlo cross-validation with full data set', 505 | choices: ['validate', 'k-fold'] 506 | }) 507 | yargs.option('k', { 508 | describe: 'K for K-Fold', 509 | number: true, 510 | default: 5 511 | }) 512 | yargs.option('shuffle', { 513 | describe: 'Shuffle utterances before K-Fold (monte carlo)', 514 | boolean: true, 515 | default: true 516 | }) 517 | yargs.option('extract', { 518 | describe: 'Extract utterances from connector workspace (otherwise load from --convos directory or .) - k-fold only.', 519 | boolean: true, 520 | default: false 521 | }) 522 | yargs.option('intents', { 523 | describe: 'Only evaluate for the given intents' 524 | }) 525 | yargs.option('output', { 526 | describe: 'Output scores to CSV file (default validate.csv or k-fold.csv)' 527 | }) 528 | yargs.option('outputPredictions', { 529 | describe: 'Output intent predictions to CSV file (default validate-predictions.csv or k-fold-predictions.csv)' 530 | }) 531 | yargs.option('train', { 532 | describe: 'Folder holding the training set - validate only, required.' 533 | }) 534 | yargs.option('test', { 535 | describe: 'Folder holding the test set - validate only, required.' 536 | }) 537 | }, 538 | handler, 539 | getConnector 540 | } 541 | -------------------------------------------------------------------------------- /src/nlp/split.js: -------------------------------------------------------------------------------- 1 | const util = require('util') 2 | const path = require('path') 3 | const fs = require('fs') 4 | const yargsCmd = require('yargs') 5 | const slug = require('slug') 6 | const { mkdirpSync } = require('mkdirp') 7 | const _ = require('lodash') 8 | const { BotDriver } = require('botium-core') 9 | const debug = require('debug')('botium-cli-nlp') 10 | 11 | const writeUtterances = (filename, intentName, utterances) => { 12 | fs.writeFileSync(filename, [ 13 | intentName, 14 | ...utterances 15 | ].join('\r\n')) 16 | } 17 | 18 | const split = async (argv) => { 19 | const trainDir = argv.train 20 | try { 21 | mkdirpSync(trainDir) 22 | } catch (err) { 23 | console.log(`Failed to create training data directory ${trainDir}: ${err.message}`) 24 | process.exit(1) 25 | } 26 | const testDir = argv.test 27 | try { 28 | mkdirpSync(testDir) 29 | } catch (err) { 30 | console.log(`Failed to create test data directory ${testDir}: ${err.message}`) 31 | process.exit(1) 32 | } 33 | 34 | const driver = new BotDriver() 35 | const compiler = driver.BuildCompiler() 36 | 37 | for (const convodir of argv.convos) { 38 | try { 39 | compiler.ReadScriptsFromDirectory(convodir) 40 | } catch (err) { 41 | console.log(`Failed to read convo directory ${convodir}: ${err.message}`) 42 | process.exit(1) 43 | } 44 | } 45 | 46 | const intents = Object.keys(compiler.utterances).map(intentName => ({ 47 | intentName, 48 | utterances: compiler.utterances[intentName].utterances 49 | })) 50 | 51 | for (const intent of intents) { 52 | const utterancesShuffled = _.shuffle(intent.utterances) 53 | const uttCount = utterancesShuffled.length 54 | 55 | const uttCountTest = Math.floor((argv.percentage / 100) * uttCount) 56 | const utterancesTest = utterancesShuffled.slice(0, uttCountTest) 57 | const utterancesTrain = utterancesShuffled.slice(uttCountTest) 58 | console.log(`Split intent ${intent.intentName} into ${utterancesTest.length} test and ${utterancesTrain.length} training utterances`) 59 | 60 | const filenameTest = path.join(testDir, `${slug(intent.intentName)}.utterances.txt`) 61 | try { 62 | writeUtterances(filenameTest, intent.intentName, utterancesTest || []) 63 | console.log(`Wrote intent test utterances for ${intent.intentName} to ${filenameTest}`) 64 | } catch (err) { 65 | console.log(`Failed to write intent test utterances for ${intent.intentName} to ${filenameTest}: ${err.message}`) 66 | } 67 | 68 | const filenameTrain = path.join(trainDir, `${slug(intent.intentName)}.utterances.txt`) 69 | try { 70 | writeUtterances(filenameTrain, intent.intentName, utterancesTrain || []) 71 | console.log(`Wrote intent training utterances for ${intent.intentName} to ${filenameTrain}`) 72 | } catch (err) { 73 | console.log(`Failed to write intent training utterances for ${intent.intentName} to ${filenameTrain}: ${err.message}`) 74 | } 75 | } 76 | } 77 | 78 | const handler = (argv) => { 79 | debug(`command options: ${util.inspect(argv)}`) 80 | if (argv.percentage < 0 || argv.percentage > 100) { 81 | return yargsCmd.showHelp() 82 | } 83 | split(argv) 84 | } 85 | 86 | module.exports = { 87 | command: 'nlpsplit', 88 | describe: 'Split utterances into a training and a test set', 89 | builder: (yargs) => { 90 | yargs.option('percentage', { 91 | describe: 'Percentage to put into test set (between 0 and 100)', 92 | number: true, 93 | default: 20 94 | }) 95 | yargs.option('train', { 96 | describe: 'Folder to put training set' 97 | }) 98 | yargs.option('test', { 99 | describe: 'Folder to put test set' 100 | }) 101 | }, 102 | handler 103 | } 104 | -------------------------------------------------------------------------------- /src/proxy/index.js: -------------------------------------------------------------------------------- 1 | const util = require('util') 2 | const { buildRedisHandler, startProxy } = require('botium-core/src/grid/inbound/proxy') 3 | const debug = require('debug')('botium-cli-proxy') 4 | 5 | const handler = (argv) => { 6 | debug(`command options: ${util.inspect(argv)}`) 7 | 8 | console.log(argv.redisurl) 9 | 10 | const redisHandler = buildRedisHandler(argv.redisurl, argv.topic) 11 | 12 | startProxy({ 13 | port: argv.port, 14 | endpoint: '/', 15 | processEvent: redisHandler 16 | }).catch((err) => { 17 | debug(err) 18 | console.log(`Failed to start inbound proxy: ${err.message || err}`) 19 | process.exit(1) 20 | }) 21 | } 22 | 23 | module.exports = { 24 | command: 'inbound-proxy', 25 | describe: 'Run Botium endpoint for accepting inbound messages, forwarding it to Redis', 26 | builder: (yargs) => { 27 | yargs.option('port', { 28 | describe: 'Local port the inbound proxy is listening to', 29 | number: true, 30 | default: 45100 31 | }) 32 | yargs.option('redisurl', { 33 | describe: 'Redis url to forward inbound messages', 34 | default: 'redis://127.0.0.1:6379' 35 | }) 36 | yargs.option('topic', { 37 | describe: 'Redis topic to forward inbound messages', 38 | default: 'SIMPLEREST_INBOUND_SUBSCRIPTION' 39 | }) 40 | }, 41 | handler 42 | } 43 | -------------------------------------------------------------------------------- /src/run/index.js: -------------------------------------------------------------------------------- 1 | const util = require('util') 2 | const path = require('path') 3 | const fs = require('fs') 4 | const Mocha = require('mocha') 5 | const { mkdirpSync } = require('mkdirp') 6 | const isJSON = require('is-json') 7 | const slug = require('slug') 8 | const mime = require('mime-types') 9 | const promiseRetry = require('promise-retry') 10 | const { BotDriver, RetryHelper } = require('botium-core') 11 | const expect = require('chai').expect 12 | const addContext = require('mochawesome/addContext') 13 | const { reportUsage } = require('../metrics') 14 | const debug = require('debug')('botium-cli-run') 15 | 16 | const outputTypes = [ 17 | 'dot', 18 | 'nyan', 19 | 'landing', 20 | 'tap', 21 | 'json', 22 | 'json-stream', 23 | 'xunit', 24 | 'spec', 25 | 'list', 26 | 'min', 27 | 'doc', 28 | 'progress', 29 | 'csv', 30 | 'mochawesome' 31 | ] 32 | 33 | const parseReporterOptions = (args) => { 34 | if (!args) return 35 | 36 | if (isJSON(args)) { 37 | return JSON.parse(args) 38 | } 39 | 40 | return args.split(',').reduce((acc, option) => { 41 | if (option.indexOf('=') > 0) { 42 | const [key, value] = option.split('=') 43 | acc[key] = value 44 | } else { 45 | acc[option] = true 46 | } 47 | return acc 48 | }, {}) 49 | } 50 | 51 | const parseReporter = (output) => { 52 | return output === 'csv' ? CsvReporter : output 53 | } 54 | 55 | class CsvReporter { 56 | constructor (runner) { 57 | Mocha.reporters.Base.call(this, runner) 58 | 59 | const quote = (str) => str ? str.replace('"', '""') : '' 60 | 61 | runner.on('start', function () { 62 | console.log('STATUS,SUITE,TEST,DURATION,MESSAGE') 63 | }) 64 | 65 | runner.on('pass', function (test) { 66 | console.log(`"OK","${quote(test.parent && test.parent.title)}","${quote(test.title)}",${test.duration},""`) 67 | }) 68 | 69 | runner.on('fail', function (test, err) { 70 | console.log(`"NOK","${quote(test.parent && test.parent.title)}","${quote(test.title)}",${test.duration},"${quote(err.message)}"`) 71 | }) 72 | } 73 | } 74 | 75 | const wrapBotiumError = (err) => { 76 | if (err.cause && err.cause.prettify) { 77 | return new Error(err.message + '\r\n' + err.cause.prettify()) 78 | } else { 79 | return new Error(err.message) 80 | } 81 | } 82 | 83 | const handler = (argv) => { 84 | debug(`command options: ${util.inspect(argv)}`) 85 | 86 | const reporterOptions = parseReporterOptions(argv.reporterOptions) 87 | debug(`Mocha Reporter "${argv.output}", options: ${util.inspect(reporterOptions)}`) 88 | 89 | argv.testsuitename = process.env.BOTIUM_TESTSUITENAME || argv.testsuitename 90 | argv.timeout = process.env.BOTIUM_TIMEOUT || argv.timeout 91 | argv.timeout = argv.timeout * 1000 92 | argv.attachments = process.env.BOTIUM_ATTACHMENTS || argv.attachments 93 | 94 | if (argv.attachments && !fs.existsSync(argv.attachments)) { 95 | try { 96 | mkdirpSync(argv.attachments) 97 | debug(`Created attachments directory "${argv.attachments}"`) 98 | } catch (err) { 99 | console.log(`Failed to create attachments directory ${argv.attachments}: ${err.message}`) 100 | process.exit(1) 101 | } 102 | } 103 | 104 | const driver = new BotDriver() 105 | const compiler = driver.BuildCompiler() 106 | compiler.scriptingEvents.fail = (err) => { 107 | expect.fail(null, null, err) 108 | } 109 | argv.convos.forEach((convodir) => { 110 | compiler.ReadScriptsFromDirectory(convodir, process.env.BOTIUM_FILTER || argv.filter) 111 | }) 112 | debug(`ready reading convos (${compiler.convos.length}), expanding convos ...`) 113 | if (process.env.BOTIUM_EXPANDUTTERANCES === '1' || argv.expandutterances) { 114 | debug('expanding utterances ...') 115 | compiler.ExpandUtterancesToConvos() 116 | } 117 | if (process.env.BOTIUM_EXPANDSCRIPTINGMEMORY === '1' || argv.expandscriptingmemory) { 118 | debug('expanding scripting memory ...') 119 | compiler.ExpandScriptingMemoryToConvos() 120 | } 121 | compiler.ExpandConvos() 122 | 123 | const usageMetrics = { 124 | metric: 'testexecution', 125 | connector: `${compiler.caps.CONTAINERMODE}`, 126 | testsuitename: argv.testsuitename, 127 | projectname: `${compiler.caps.PROJECTNAME}`, 128 | convoCount: compiler.convos.length, 129 | convoStepCount: compiler.convos.reduce((sum, convo) => sum + convo.conversation.length, 0), 130 | partialConvoCount: Object.keys(compiler.partialConvos).length, 131 | utterancesRefCount: Object.keys(compiler.utterances).length, 132 | utterancesCount: Object.keys(compiler.utterances).reduce((sum, uttName) => sum + compiler.utterances[uttName].utterances.length, 0), 133 | scriptingMemoriesCount: compiler.scriptingMemories.length 134 | } 135 | reportUsage(usageMetrics) 136 | 137 | debug(`ready expanding convos and utterances, number of test cases: (${compiler.convos.length}).`) 138 | 139 | const mocha = new Mocha({ 140 | reporter: parseReporter(argv.output), 141 | reporterOptions 142 | }) 143 | 144 | const suite = Mocha.Suite.create(mocha.suite, process.env.BOTIUM_TESTSUITENAME || argv.testsuitename) 145 | suite.timeout(argv.timeout) 146 | suite.beforeAll((done) => { 147 | driver.Build() 148 | .then((container) => { 149 | suite.container = container 150 | done() 151 | }) 152 | .catch(done) 153 | }) 154 | suite.beforeEach((done) => { 155 | suite.container ? suite.container.Start().then(() => done()).catch(done) : done() 156 | }) 157 | suite.afterEach((done) => { 158 | suite.container ? suite.container.Stop().then(() => done()).catch(done) : done() 159 | }) 160 | suite.afterAll((done) => { 161 | suite.container ? suite.container.Clean().then(() => done()).catch(done) : done() 162 | }) 163 | 164 | let runner = null 165 | 166 | compiler.convos.forEach((convo) => { 167 | debug(`adding test case ${convo.header.name} (from: ${util.inspect(convo.sourceTag)})`) 168 | const test = new Mocha.Test(convo.header.name, (testcaseDone) => { 169 | debug('running testcase ' + convo.header.name) 170 | 171 | const attachmentsLog = [] 172 | const listenerMe = (container, msg) => { 173 | if (msg.attachments) attachmentsLog.push(...msg.attachments) 174 | } 175 | const listenerBot = (container, msg) => { 176 | if (msg.attachments) attachmentsLog.push(...msg.attachments) 177 | } 178 | const listenerAttachments = (container, attachment) => { 179 | attachmentsLog.push(attachment) 180 | } 181 | driver.on('MESSAGE_SENTTOBOT', listenerMe) 182 | driver.on('MESSAGE_RECEIVEDFROMBOT', listenerBot) 183 | driver.on('MESSAGE_ATTACHMENT', listenerAttachments) 184 | 185 | const finish = (transcript, err) => { 186 | if (transcript) { 187 | addContext(runner, { title: 'Conversation Log', value: transcript.prettifyActual() }) 188 | } 189 | driver.eventEmitter.removeListener('MESSAGE_SENTTOBOT', listenerMe) 190 | driver.eventEmitter.removeListener('MESSAGE_RECEIVEDFROMBOT', listenerBot) 191 | driver.eventEmitter.removeListener('MESSAGE_ATTACHMENT', listenerAttachments) 192 | 193 | if (argv.attachments && attachmentsLog.length > 0) { 194 | debug(`Found ${attachmentsLog.length} attachments, saving to folder ${argv.attachments}`) 195 | attachmentsLog.forEach((a, i) => { 196 | const filename = slug(convo.header.name) + '_' + i + (a.name ? '_' + slug(a.name) : '') + (a.mimeType ? '.' + mime.extension(a.mimeType) : '') 197 | const outputTo = path.join(argv.attachments, filename) 198 | try { 199 | fs.writeFileSync(outputTo, Buffer.from(a.base64, 'base64')) 200 | } catch (err) { 201 | debug(`Failed to write attachment to ${outputTo}: ${err.message || util.inspect(err)}`) 202 | } 203 | }) 204 | } 205 | if (err) { 206 | testcaseDone(wrapBotiumError(err)) 207 | } else { 208 | testcaseDone() 209 | } 210 | } 211 | 212 | const retryHelper = new RetryHelper(suite.container.caps, 'CONVO') 213 | promiseRetry(async (retry, number) => { 214 | try { 215 | const transcript = await convo.Run(suite.container) 216 | debug(convo.header.name + ' ready, calling done function.') 217 | return transcript 218 | } catch (err) { 219 | if (retryHelper.shouldRetry(err)) { 220 | debug(`Running Convo "${convo.header.name}" trial #${number} failed, retry activated`) 221 | await suite.container.Stop() 222 | await suite.container.Start() 223 | debug(`Restarting container for Convo "${convo.header.name}" trial #${number} completed successfully.`) 224 | retry(err) 225 | } else { 226 | debug(`Running Convo "${convo.header.name}" trial #${number} failed finally`) 227 | throw err 228 | } 229 | } 230 | }, retryHelper.retrySettings) 231 | .then((transcript) => { 232 | debug(convo.header.name + ' ready, calling done function.') 233 | finish(transcript) 234 | }) 235 | .catch((err) => { 236 | debug(convo.header.name + ' failed: ' + util.inspect(err)) 237 | finish(err.transcript, err) 238 | }) 239 | }) 240 | test.timeout(argv.timeout) 241 | suite.addTest(test) 242 | }) 243 | 244 | runner = mocha.run((failures) => { 245 | process.on('exit', () => { 246 | process.exit(failures) 247 | }) 248 | }) 249 | } 250 | 251 | module.exports = { 252 | command: 'run [output]', 253 | describe: 'Run Botium convo files and output test report with Mocha test runner', 254 | builder: (yargs) => { 255 | yargs.positional('output', { 256 | describe: 'Output report type (select Mocha reporter)', 257 | choices: outputTypes, 258 | default: 'spec' 259 | }) 260 | yargs.option('testsuitename', { 261 | alias: 'n', 262 | describe: 'Name of the Test Suite (also read from env variable "BOTIUM_TESTSUITENAME")', 263 | default: 'Botium Test-Suite' 264 | }) 265 | yargs.option('filter', { 266 | describe: 'Filter the convo files to run with a "glob" filter (also read from env variable "BOTIUM_FILTER"), for example "**/*.en.*.txt"' 267 | }) 268 | yargs.option('expandutterances', { 269 | describe: 'Expand all utterances (except INCOMPREHENSION) to simple Question/Answer convos (also read from env variable "BOTIUM_EXPANDUTTERANCES" - "1" means yes)', 270 | default: false 271 | }) 272 | yargs.option('expandscriptingmemory', { 273 | describe: 'Expand scripting memory tables to separate convos (also read from env variable "BOTIUM_EXPANDSCRIPTINGMEMORY" - "1" means yes)', 274 | default: false 275 | }) 276 | yargs.option('attachments', { 277 | describe: 'Directory where to save message attachments, for example screenshots from webdriver (also read from env variable "BOTIUM_ATTACHMENTS")' 278 | }) 279 | yargs.option('timeout', { 280 | describe: 'Timeout in seconds for Botium functions (also read from env variable "BOTIUM_TIMEOUT")', 281 | number: true, 282 | default: 60 283 | }) 284 | yargs.option('reporter-options', { 285 | describe: 'Options for mocha reporter, either as JSON, or as key-value pairs ("option1=value1,option2=value2,..."). For details see documentation of the used mocha reporter.' 286 | }) 287 | }, 288 | handler, 289 | parseReporterOptions, 290 | parseReporter, 291 | outputTypes 292 | } 293 | --------------------------------------------------------------------------------