├── .dockerignore ├── examples ├── example-gated-deployment-secret.yml ├── example-rest-service-gated-deployment.yml ├── deployment-control.yml └── deployment-treatment.yml ├── helm └── kubernetes-gated-deployments │ ├── Chart.yaml │ ├── templates │ ├── service-account.yaml │ ├── _helpers.tpl │ ├── deployment.yaml │ └── rbac.yaml │ ├── .helmignore │ └── values.yaml ├── Dockerfile ├── CHANGELOG.md ├── custom-resource-manifest.json ├── CONTRIBUTING.md ├── lib ├── watchers │ ├── constants.js │ ├── gated-deployment-watcher.js │ ├── watcher.js │ ├── gated-deployment-watcher.test.js │ ├── watcher.test.js │ ├── deployment-watcher.js │ ├── watcher.integration.test.js │ └── deployment-watcher.test.js ├── analysis.js ├── plugins │ ├── plugin-factory.js │ ├── plugin-factory.test.js │ ├── plugin.js │ ├── plugin.test.js │ ├── newrelic-performance.js │ └── newrelic-performance.test.js ├── newrelic.js ├── deployment-helper.js ├── custom-resource-manager.js ├── analysis.test.js ├── newrelic.test.js ├── deployment-helper.test.js └── custom-resource-manager.test.js ├── release.sh ├── SECURITY.md ├── config ├── index.js └── environment.js ├── LICENSE ├── bin └── daemon.js ├── .gitignore ├── package.json ├── gated-deployments.yml ├── CODE_OF_CONDUCT.md ├── README.md ├── architecture.svg └── gated-deployment-path.svg /.dockerignore: -------------------------------------------------------------------------------- 1 | coverage 2 | .env 3 | node_modules 4 | architecture.svg 5 | gated-deployment-path.svg 6 | Dockerfile 7 | -------------------------------------------------------------------------------- /examples/example-gated-deployment-secret.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: newrelic-secrets 5 | type: Opaque 6 | data: 7 | example-rest-service: aW5zaWdodHNBcGlLZXk= 8 | -------------------------------------------------------------------------------- /helm/kubernetes-gated-deployments/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: 1.1.0 3 | description: A Helm chart for Kubernetes Gated Deployments 4 | name: kubernetes-gated-deployments 5 | version: 1.0.0 6 | -------------------------------------------------------------------------------- /helm/kubernetes-gated-deployments/templates/service-account.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 | name: {{ .Release.Name }}-service-account 5 | namespace: {{ .Release.Namespace }} 6 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:10.15.1-alpine 2 | 3 | RUN npm install npm@6.4.1 -g 4 | 5 | # Setup source directory 6 | RUN mkdir /app 7 | WORKDIR /app 8 | COPY package.json package-lock.json /app/ 9 | RUN npm ci 10 | 11 | # Copy app to source directory 12 | COPY . /app 13 | 14 | ENV NODE_ENV production 15 | ENV NPM_CONFIG_LOGLEVEL info 16 | 17 | USER node 18 | 19 | CMD ["npm", "start"] 20 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. 4 | 5 | ## 1.1.0 (2019-09-10) 6 | 7 | 8 | ### Features 9 | 10 | * add standard-version and release script ([#2](https://github.com/godaddy/kubernetes-gated-deployments/issues/2)) ([4050dca](https://github.com/godaddy/kubernetes-gated-deployments/commit/4050dca)) 11 | -------------------------------------------------------------------------------- /helm/kubernetes-gated-deployments/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | .vscode/ 23 | -------------------------------------------------------------------------------- /custom-resource-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "kind": "CustomResourceDefinition", 3 | "spec": { 4 | "scope": "Namespaced", 5 | "version": "v1", 6 | "group": "kubernetes-client.io", 7 | "names": { 8 | "shortNames": [ 9 | "gd" 10 | ], 11 | "kind": "GatedDeployment", 12 | "plural": "gateddeployments", 13 | "singular": "gateddeployment" 14 | } 15 | }, 16 | "apiVersion": "apiextensions.k8s.io/v1beta1", 17 | "metadata": { 18 | "name": "gateddeployments.kubernetes-client.io" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /examples/example-rest-service-gated-deployment.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 'kubernetes-client.io/v1' 2 | kind: GatedDeployment 3 | metadata: 4 | name: example-rest-service 5 | deploymentDescriptor: 6 | control: 7 | name: example-rest-service-control 8 | treatment: 9 | name: example-rest-service-treatment 10 | decisionPlugins: 11 | - name: newRelicPerformance 12 | accountId: 807783 13 | secretName: newrelic-secrets 14 | secretKey: example-rest-service 15 | appName: example-rest-service 16 | minSamples: 50 17 | maxTime: 600 18 | testPath: /shopper/products 19 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Kubernetes Gated Deployments 2 | 3 | Thanks for taking the time to contribute! 4 | 5 | ## How to contribute 6 | 7 | * Fork the repository on GitHub 8 | * Make the changes in your forked repository 9 | * Add or update any relevant unit and integration tests 10 | * Ensure your changes adhere to [Javascript Standard Style](https://github.com/standard/standard/) with `npm run lint` 11 | * Ensure tests pass with `npm test` 12 | * Push changes to your fork and submit a pull request to the `master` branch of this repository 13 | * Your pull request will be merged by the project maintainers after two approvals -------------------------------------------------------------------------------- /lib/watchers/constants.js: -------------------------------------------------------------------------------- 1 | const StreamEvents = { 2 | DATA: 'data', 3 | END: 'end' 4 | } 5 | 6 | const WatchEvents = { 7 | ADDED: 'ADDED', 8 | DELETED: 'DELETED', 9 | MODIFIED: 'MODIFIED' 10 | } 11 | 12 | const WatcherStates = { 13 | READY: 'READY', 14 | RUNNING: 'RUNNING', 15 | ENDED: 'ENDED' 16 | } 17 | 18 | const DEPLOYMENT_ANNOTATION_NAME = 'gatedDeployStatus' 19 | const EVENT_LOCK = 'eventLock' 20 | const EXPERIMENT_ANNOTATION_NAME = 'gatedDeployExperiment' 21 | 22 | module.exports = { 23 | DEPLOYMENT_ANNOTATION_NAME, 24 | EVENT_LOCK, 25 | EXPERIMENT_ANNOTATION_NAME, 26 | StreamEvents, 27 | WatchEvents, 28 | WatcherStates 29 | } 30 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | SHA=$(git rev-parse --short HEAD) 6 | TAG=$(git describe) 7 | 8 | docker build -t godaddy/kubernetes-gated-deployments:$SHA . 9 | docker tag godaddy/kubernetes-gated-deployments:$SHA godaddy/kubernetes-gated-deployments:$TAG 10 | 11 | perl -i -pe "s/tag: [a-zA-Z0-9\.]*/tag: $TAG/" helm/kubernetes-gated-deployments/values.yaml 12 | perl -i -pe "s/appVersion: [a-zA-Z0-9\.]*/appVersion: $TAG/" helm/kubernetes-gated-deployments/Chart.yaml 13 | git commit helm/kubernetes-gated-deployments/values.yaml helm/kubernetes-gated-deployments/Chart.yaml -m "chore(release): godaddy/kubernetes-gated-deployments:$TAG" 14 | 15 | echo "" 16 | echo "Run the following to publish:" 17 | echo "" 18 | echo " git push --follow-tags origin master && docker push godaddy/kubernetes-gated-deployments:$TAG" 19 | echo "" 20 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Reporting Security Issues 2 | 3 | We take security very seriously at GoDaddy. We appreciate your efforts to 4 | responsibly disclose your findings, and will make every effort to acknowledge 5 | your contributions. 6 | 7 | ## Where should I report security issues? 8 | 9 | In order to give the community time to respond and upgrade, we strongly urge you 10 | report all security issues privately. 11 | 12 | To report a security issue in one of our Open Source projects email us directly 13 | at **oss@godaddy.com** and include the word "SECURITY" in the subject line. 14 | 15 | This mail is delivered to our Open Source Security team. 16 | 17 | After the initial reply to your report, the team will keep you informed of the 18 | progress being made towards a fix and announcement, and may ask for additional 19 | information or guidance. 20 | -------------------------------------------------------------------------------- /config/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const kube = require('kubernetes-client') 4 | const pino = require('pino') 5 | 6 | const envConfig = require('./environment') 7 | const CustomResourceManager = require('../lib/custom-resource-manager') 8 | const customResourceManifest = require('../custom-resource-manifest.json') 9 | 10 | let kubeClientConfig 11 | try { 12 | kubeClientConfig = kube.config.getInCluster() 13 | } catch (err) { 14 | kubeClientConfig = kube.config.fromKubeconfig() 15 | } 16 | const kubeClient = new kube.Client({ config: kubeClientConfig }) 17 | 18 | const logger = pino({ 19 | serializers: { 20 | err: pino.stdSerializers.err 21 | }, 22 | level: envConfig.logLevel 23 | }) 24 | 25 | const customResourceManager = new CustomResourceManager({ 26 | kubeClient, 27 | logger 28 | }) 29 | 30 | module.exports = { 31 | customResourceManager, 32 | customResourceManifest, 33 | ...envConfig, 34 | kubeClient, 35 | logger 36 | } 37 | -------------------------------------------------------------------------------- /config/environment.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | /* eslint-disable no-process-env */ 4 | 5 | const environment = process.env.NODE_ENV 6 | ? process.env.NODE_ENV.toLowerCase() : 'development' 7 | 8 | // Validate environment 9 | const validEnvironments = new Set(['development', 'test', 'production']) 10 | if (!validEnvironments.has(environment)) { 11 | throw new Error(`Invalid environment: ${environment}`) 12 | } 13 | 14 | // Load env file only when development env 15 | if (environment === 'development') { 16 | require('dotenv').config() 17 | } 18 | 19 | const pollerIntervalMilliseconds = process.env.POLLER_INTERVAL_MILLISECONDS 20 | ? Number(process.env.POLLER_INTERVAL_MILLISECONDS) : 30000 21 | 22 | const controllerNamespace = process.env.CONTROLLER_NAMESPACE || 'kubernetes-gated-deployments' 23 | const logLevel = process.env.LOG_LEVEL || 'info' 24 | 25 | module.exports = { 26 | controllerNamespace, 27 | environment, 28 | logLevel, 29 | pollerIntervalMilliseconds 30 | } 31 | -------------------------------------------------------------------------------- /examples/deployment-control.yml: -------------------------------------------------------------------------------- 1 | kind: Deployment 2 | apiVersion: apps/v1 3 | metadata: 4 | name: example-rest-service-control 5 | labels: 6 | name: example-rest-service-control 7 | spec: 8 | replicas: 4 9 | selector: 10 | matchLabels: 11 | name: example-rest-service-control 12 | template: 13 | metadata: 14 | labels: 15 | name: example-rest-service-control 16 | service: example-rest-service 17 | spec: 18 | containers: 19 | - name: example-rest-service-control 20 | image: silasbw/example-rest-service:latest 21 | imagePullPolicy: Always 22 | ports: 23 | - containerPort: 8080 24 | livenessProbe: 25 | httpGet: 26 | path: "/healthcheck" 27 | port: 8080 28 | initialDelaySeconds: 15 29 | timeoutSeconds: 1 30 | readinessProbe: 31 | httpGet: 32 | path: "/healthcheck" 33 | port: 8080 34 | initialDelaySeconds: 15 35 | timeoutSeconds: 1 36 | -------------------------------------------------------------------------------- /examples/deployment-treatment.yml: -------------------------------------------------------------------------------- 1 | kind: Deployment 2 | apiVersion: apps/v1 3 | metadata: 4 | name: example-rest-service-treatment 5 | labels: 6 | name: example-rest-service-treatment 7 | spec: 8 | replicas: 4 9 | selector: 10 | matchLabels: 11 | name: example-rest-service-treatment 12 | template: 13 | metadata: 14 | labels: 15 | name: example-rest-service-treatment 16 | service: example-rest-service 17 | spec: 18 | containers: 19 | - name: example-rest-service-treatment 20 | image: silasbw/example-rest-service:latest 21 | imagePullPolicy: Always 22 | ports: 23 | - containerPort: 8080 24 | livenessProbe: 25 | httpGet: 26 | path: "/healthcheck" 27 | port: 8080 28 | initialDelaySeconds: 15 29 | timeoutSeconds: 1 30 | readinessProbe: 31 | httpGet: 32 | path: "/healthcheck" 33 | port: 8080 34 | initialDelaySeconds: 15 35 | timeoutSeconds: 1 36 | -------------------------------------------------------------------------------- /helm/kubernetes-gated-deployments/values.yaml: -------------------------------------------------------------------------------- 1 | # Default values for kubernetes-gated-deployments. 2 | # This is a YAML-formatted file. 3 | # Declare variables to be passed into your templates. 4 | 5 | logLevel: info 6 | pollerIntervalMilliseconds: 30000 7 | 8 | replicaCount: 1 9 | 10 | image: 11 | repository: godaddy/kubernetes-gated-deployments 12 | tag: 1.1.0 13 | pullPolicy: IfNotPresent 14 | 15 | nameOverride: "" 16 | fullnameOverride: "" 17 | 18 | resources: {} 19 | # We usually recommend not to specify default resources and to leave this as a conscious 20 | # choice for the user. This also increases chances charts run on environments with little 21 | # resources, such as Minikube. If you do want to specify resources, uncomment the following 22 | # lines, adjust them as necessary, and remove the curly braces after 'resources:'. 23 | # limits: 24 | # cpu: 100m 25 | # memory: 128Mi 26 | # requests: 27 | # cpu: 100m 28 | # memory: 128Mi 29 | 30 | nodeSelector: {} 31 | 32 | tolerations: [] 33 | 34 | affinity: {} 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 GoDaddy Operating Company, LLC. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /bin/daemon.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | 'use strict' 4 | 5 | // make-promises-safe installs an process.on('unhandledRejection') handler 6 | // that prints the stacktrace and exits the process 7 | // with an exit code of 1, just like any uncaught exception. 8 | require('make-promises-safe') 9 | 10 | const GatedDeploymentWatcher = require('../lib/watchers/gated-deployment-watcher') 11 | 12 | const { 13 | kubeClient, 14 | customResourceManager, 15 | customResourceManifest, 16 | logger, 17 | pollerIntervalMilliseconds, 18 | controllerNamespace 19 | } = require('../config') 20 | 21 | async function main () { 22 | logger.info('loading kube specs') 23 | await kubeClient.loadSpec() 24 | logger.info('successfully loaded kube specs') 25 | logger.info('updating CRD') 26 | await customResourceManager.upsertResource({ customResourceManifest }) 27 | logger.info('successfully updated CRD') 28 | 29 | const gatedDeploymentWatcher = new GatedDeploymentWatcher({ 30 | controllerNamespace, 31 | kubeClient, 32 | logger, 33 | pollerIntervalMilliseconds 34 | }) 35 | 36 | logger.info('starting app') 37 | gatedDeploymentWatcher.start() 38 | logger.info('successfully started app') 39 | } 40 | 41 | main() 42 | -------------------------------------------------------------------------------- /lib/analysis.js: -------------------------------------------------------------------------------- 1 | const mwu = require('mann-whitney-utest') 2 | 3 | const results = { 4 | harm: 'harm', 5 | noHarm: 'noHarm', 6 | notSignificant: 'notSignificant' 7 | } 8 | 9 | const DEFAULT_HARM_THRESHOLD = 1.5 10 | 11 | // This z score corresponds to p = 0.05 (two tailed) 12 | const DEFAULT_Z_SCORE_THRESHOLD = 1.96 13 | 14 | function test ({ 15 | controlSamples, 16 | treatmentSamples, 17 | minSamples, 18 | harmThreshold = DEFAULT_HARM_THRESHOLD, 19 | zScoreThreshold = DEFAULT_Z_SCORE_THRESHOLD 20 | }) { 21 | const samples = [controlSamples, treatmentSamples] 22 | if (controlSamples.length < minSamples || treatmentSamples.length < minSamples) { 23 | return { u: [0, 0], analysisResult: results.notSignificant } 24 | } 25 | const u = mwu.test(samples) 26 | const zScore = mwu.criticalValue(u, samples) 27 | if (zScore <= zScoreThreshold) return { u, analysisResult: results.noHarm } 28 | 29 | // If significantly different, test if treatment is worse by harmThreshold 30 | const [controlU, treatmentU] = u 31 | const analysisResult = (treatmentU / controlU) > harmThreshold ? results.harm : results.noHarm 32 | return { u, analysisResult } 33 | } 34 | 35 | module.exports = { 36 | test, 37 | results 38 | } 39 | -------------------------------------------------------------------------------- /lib/plugins/plugin-factory.js: -------------------------------------------------------------------------------- 1 | const newRelicPerformance = require('./newrelic-performance') 2 | 3 | const pluginNameMap = { 4 | newRelicPerformance 5 | } 6 | 7 | /** 8 | * Creates an array of plugins specified by the decisionPluginConfig 9 | * @param {Object} options the options object 10 | * @param {string} options.namespace the kubernetes namespace of the controller 11 | * @param {Object} options.kubeClient the kubernetes client 12 | * @param {Object} options.logger the system logger 13 | * @param {string} options.controlName the name of the control deployment 14 | * @param {string} options.treatmentName the name of the treatment deployment 15 | * @param {array} options.decisionPluginConfig the list of decision plugin configs 16 | * @returns {array} Array of Plugin objects 17 | */ 18 | function buildPluginsFromConfig ({ 19 | namespace, 20 | kubeClient, 21 | logger, 22 | controlName, 23 | treatmentName, 24 | decisionPluginConfig 25 | }) { 26 | return Promise.all(decisionPluginConfig.map(config => { 27 | const pluginClass = pluginNameMap[config.name] 28 | if (pluginClass) { 29 | return pluginClass.build({ 30 | namespace, 31 | kubeClient, 32 | logger, 33 | controlName, 34 | treatmentName, 35 | config 36 | }) 37 | } else { 38 | throw new Error(`Invalid plugin: ${config.name}`) 39 | } 40 | })) 41 | } 42 | 43 | module.exports = { 44 | buildPluginsFromConfig 45 | } 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | 9 | # Diagnostic reports (https://nodejs.org/api/report.html) 10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 11 | 12 | # Runtime data 13 | pids 14 | *.pid 15 | *.seed 16 | *.pid.lock 17 | 18 | # Directory for instrumented libs generated by jscoverage/JSCover 19 | lib-cov 20 | 21 | # Coverage directory used by tools like istanbul 22 | coverage 23 | *.lcov 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # TypeScript v1 declaration files 45 | typings/ 46 | 47 | # TypeScript cache 48 | *.tsbuildinfo 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Optional REPL history 57 | .node_repl_history 58 | 59 | # Output of 'npm pack' 60 | *.tgz 61 | 62 | # Yarn Integrity file 63 | .yarn-integrity 64 | 65 | # dotenv environment variables file 66 | .env 67 | .env.test 68 | 69 | # parcel-bundler cache (https://parceljs.org/) 70 | .cache 71 | 72 | # next.js build output 73 | .next 74 | 75 | # nuxt.js build output 76 | .nuxt 77 | 78 | # vuepress build output 79 | .vuepress/dist 80 | 81 | # Serverless directories 82 | .serverless/ 83 | 84 | # FuseBox cache 85 | .fusebox/ 86 | 87 | # DynamoDB Local files 88 | .dynamodb/ -------------------------------------------------------------------------------- /lib/plugins/plugin-factory.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | const chai = require('chai') 3 | const sinon = require('sinon') 4 | const sinonChai = require('sinon-chai') 5 | 6 | const newRelicPerformance = require('./newrelic-performance') 7 | const pluginFactory = require('../plugins/plugin-factory') 8 | 9 | chai.use(sinonChai) 10 | 11 | const { expect } = chai 12 | 13 | describe('PluginFactory', () => { 14 | describe('buildPluginsFromConfig', () => { 15 | it('builds all plugins specified in config', async () => { 16 | const newrelicStub = sinon.stub(newRelicPerformance, 'build').resolves('mockPluginClass') 17 | 18 | const result = await pluginFactory.buildPluginsFromConfig({ 19 | namespace: 'mockNS', 20 | kubeClient: 'mockClient', 21 | logger: 'mockLogger', 22 | controlName: 'control', 23 | treatmentName: 'treatment', 24 | decisionPluginConfig: [{ 25 | name: 'newRelicPerformance' 26 | }] 27 | }) 28 | 29 | expect(result).to.deep.equal(['mockPluginClass']) 30 | expect(newrelicStub).to.have.been.calledWith({ 31 | namespace: 'mockNS', 32 | kubeClient: 'mockClient', 33 | logger: 'mockLogger', 34 | controlName: 'control', 35 | treatmentName: 'treatment', 36 | config: { name: 'newRelicPerformance' } 37 | }) 38 | }) 39 | 40 | it('throws an error on an unknown plugin', () => { 41 | expect(() => pluginFactory.buildPluginsFromConfig({ 42 | namespace: 'mockNS', 43 | kubeClient: 'mockClient', 44 | logger: 'mockLogger', 45 | decisionPluginConfig: [{ 46 | name: 'foo' 47 | }] 48 | })).to.throw(Error) 49 | }) 50 | }) 51 | }) 52 | -------------------------------------------------------------------------------- /helm/kubernetes-gated-deployments/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "kubernetes-gated-deployments.name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | If release name contains chart name it will be used as a full name. 13 | */}} 14 | {{- define "kubernetes-gated-deployments.fullname" -}} 15 | {{- if .Values.fullnameOverride -}} 16 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} 17 | {{- else -}} 18 | {{- $name := default .Chart.Name .Values.nameOverride -}} 19 | {{- if contains $name .Release.Name -}} 20 | {{- .Release.Name | trunc 63 | trimSuffix "-" -}} 21 | {{- else -}} 22 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 23 | {{- end -}} 24 | {{- end -}} 25 | {{- end -}} 26 | 27 | {{/* 28 | Create chart name and version as used by the chart label. 29 | */}} 30 | {{- define "kubernetes-gated-deployments.chart" -}} 31 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} 32 | {{- end -}} 33 | 34 | {{/* 35 | Common labels 36 | */}} 37 | {{- define "kubernetes-gated-deployments.labels" -}} 38 | app.kubernetes.io/name: {{ include "kubernetes-gated-deployments.name" . }} 39 | helm.sh/chart: {{ include "kubernetes-gated-deployments.chart" . }} 40 | app.kubernetes.io/instance: {{ .Release.Name }} 41 | {{- if .Chart.AppVersion }} 42 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 43 | {{- end }} 44 | app.kubernetes.io/managed-by: {{ .Release.Service }} 45 | {{- end -}} 46 | -------------------------------------------------------------------------------- /helm/kubernetes-gated-deployments/templates/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "kubernetes-gated-deployments.fullname" . }} 5 | namespace: {{ .Release.Namespace }} 6 | labels: 7 | {{ include "kubernetes-gated-deployments.labels" . | indent 4 }} 8 | spec: 9 | replicas: {{ .Values.replicaCount }} 10 | selector: 11 | matchLabels: 12 | app.kubernetes.io/name: {{ include "kubernetes-gated-deployments.name" . }} 13 | app.kubernetes.io/instance: {{ .Release.Name }} 14 | template: 15 | metadata: 16 | labels: 17 | app.kubernetes.io/name: {{ include "kubernetes-gated-deployments.name" . }} 18 | app.kubernetes.io/instance: {{ .Release.Name }} 19 | spec: 20 | serviceAccountName: {{ .Release.Name }}-service-account 21 | containers: 22 | - name: {{ .Chart.Name }} 23 | image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" 24 | imagePullPolicy: {{ .Values.image.pullPolicy }} 25 | resources: 26 | {{- toYaml .Values.resources | nindent 12 }} 27 | env: 28 | - name: CONTROLLER_NAMESPACE 29 | value: {{ .Release.Namespace }} 30 | - name: LOG_LEVEL 31 | value: {{ .Values.logLevel }} 32 | - name: POLLER_INTERVAL_MILLISECONDS 33 | value: {{ .Values.pollerIntervalMilliseconds | quote }} 34 | {{- with .Values.nodeSelector }} 35 | nodeSelector: 36 | {{- toYaml . | nindent 8 }} 37 | {{- end }} 38 | {{- with .Values.affinity }} 39 | affinity: 40 | {{- toYaml . | nindent 8 }} 41 | {{- end }} 42 | {{- with .Values.tolerations }} 43 | tolerations: 44 | {{- toYaml . | nindent 8 }} 45 | {{- end }} 46 | -------------------------------------------------------------------------------- /helm/kubernetes-gated-deployments/templates/rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: rbac.authorization.k8s.io/v1 2 | kind: ClusterRoleBinding 3 | metadata: 4 | name: {{ .Release.Name }}-cluster-role-binding 5 | subjects: 6 | - kind: ServiceAccount 7 | name: {{ .Release.Name }}-service-account 8 | namespace: {{ .Release.Namespace }} 9 | roleRef: 10 | kind: ClusterRole 11 | name: {{ .Release.Name }}-cluster-role 12 | apiGroup: rbac.authorization.k8s.io 13 | --- 14 | apiVersion: rbac.authorization.k8s.io/v1 15 | kind: RoleBinding 16 | metadata: 17 | name: {{ .Release.Name }}-read-secrets-role-binding 18 | namespace: {{ .Release.Namespace }} 19 | subjects: 20 | - kind: ServiceAccount 21 | name: {{ .Release.Name }}-service-account 22 | namespace: {{ .Release.Namespace }} 23 | roleRef: 24 | kind: Role 25 | name: {{ .Release.Name }}-read-secrets-role 26 | apiGroup: rbac.authorization.k8s.io 27 | --- 28 | apiVersion: rbac.authorization.k8s.io/v1 29 | kind: ClusterRole 30 | metadata: 31 | name: {{ .Release.Name }}-cluster-role 32 | rules: 33 | - apiGroups: ["apps"] 34 | resources: ["deployments"] 35 | verbs: ["get", "patch", "watch"] 36 | - apiGroups: ["apiextensions.k8s.io"] 37 | resources: ["customresourcedefinitions"] 38 | verbs: ["create"] 39 | - apiGroups: ["apiextensions.k8s.io"] 40 | resources: ["customresourcedefinitions"] 41 | resourceNames: ["gateddeployments.kubernetes-client.io"] 42 | verbs: ["get", "update"] 43 | - apiGroups: ["kubernetes-client.io"] 44 | resources: ["gateddeployments"] 45 | verbs: ["get", "watch", "list"] 46 | --- 47 | apiVersion: rbac.authorization.k8s.io/v1 48 | kind: Role 49 | metadata: 50 | name: {{ .Release.Name }}-read-secrets-role 51 | namespace: {{ .Release.Namespace }} 52 | rules: 53 | - apiGroups: [""] 54 | resources: ["secrets"] 55 | verbs: ["get"] 56 | -------------------------------------------------------------------------------- /lib/plugins/plugin.js: -------------------------------------------------------------------------------- 1 | const moment = require('moment') 2 | 3 | const DecisionResults = { 4 | PASS: 'PASS', 5 | FAIL: 'FAIL', 6 | WAIT: 'WAIT' 7 | } 8 | 9 | class Plugin { 10 | /** 11 | * Creates a new Plugin 12 | * @param {string} controlName the name of the control deployment 13 | * @param {string} treatmentName the name of the treatment deployment 14 | * @param {int} maxTime=600 the maximum time to let the experiment run 15 | */ 16 | constructor (controlName, treatmentName, maxTime = 600) { 17 | this._controlName = controlName 18 | this._treatmentName = treatmentName 19 | this._maxTime = maxTime 20 | } 21 | 22 | /** 23 | * Build a new plugin asynchronously. Implemented in subclasses 24 | */ 25 | static async build () { 26 | throw new Error('Must be overridden') 27 | } 28 | 29 | /** 30 | * Called when an experiment starts. Sets the start time; subclasses 31 | * may have more logic 32 | * @param {int} startTime the utc time in seconds at experiment start 33 | */ 34 | onExperimentStart (startTime) { 35 | this._startTime = startTime 36 | } 37 | 38 | /** 39 | * Called when an experiment stops. Clears the start time 40 | */ 41 | onExperimentStop () { 42 | this._startTime = null 43 | } 44 | 45 | /** 46 | * Called every polling interval to get the plugin's decision about the experiment 47 | * @returns {DecisionResults} the DecisionResults outcome of the analysis 48 | */ 49 | onExperimentPoll () { 50 | if (this._maxTime > 0 && moment().utc().diff(this._startTime, 's') >= this._maxTime) { 51 | return DecisionResults.PASS 52 | } 53 | 54 | return this._poll() 55 | } 56 | 57 | /** 58 | * Called by onExperimentPoll. Overridden by subclasses to run their 59 | * decision logic 60 | */ 61 | async _poll () { 62 | throw new Error('Must be overridden') 63 | } 64 | } 65 | 66 | module.exports = { 67 | DecisionResults, 68 | Plugin 69 | } 70 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kubernetes-gated-deployments", 3 | "version": "1.1.0", 4 | "description": "Gate Kubernetes deployments with AB tests", 5 | "main": "bin/daemon.js", 6 | "scripts": { 7 | "coverage": "nyc ./node_modules/mocha/bin/_mocha --recursive lib", 8 | "lint": "standard", 9 | "release": "standard-version -t '' && ./release.sh", 10 | "start": "./bin/daemon.js", 11 | "nodemon": "nodemon ./bin/daemon.js", 12 | "test": "mocha --recursive lib" 13 | }, 14 | "repository": { 15 | "type": "git", 16 | "url": "https://github.com/godaddy/kubernetes-gated-deployments" 17 | }, 18 | "keywords": [ 19 | "kubernetes", 20 | "aws" 21 | ], 22 | "author": "GoDaddy Operating Company, LLC", 23 | "license": "MIT", 24 | "engines": { 25 | "node": ">=10.0.0" 26 | }, 27 | "dependencies": { 28 | "async-lock": "^1.2.0", 29 | "clone": "^2.1.2", 30 | "json-stream": "^1.0.0", 31 | "kubernetes-client": "^6.5.0", 32 | "make-promises-safe": "^4.0.0", 33 | "mann-whitney-utest": "^1.0.5", 34 | "moment": "^2.24.0", 35 | "object-hash": "^1.3.1", 36 | "pino": "^5.9.0", 37 | "request-promise": "^4.2.4" 38 | }, 39 | "devDependencies": { 40 | "chai": "^4.1.2", 41 | "dotenv": "^6.1.0", 42 | "mocha": "^6.1.4", 43 | "nock": "^10.0.6", 44 | "nodemon": "^1.18.10", 45 | "nyc": "^14.1.0", 46 | "sinon": "^7.2.3", 47 | "sinon-chai": "^3.3.0", 48 | "standard": "^12.0.1", 49 | "standard-version": "^7.0.0" 50 | }, 51 | "nyc": { 52 | "check-coverage": true, 53 | "reporter": [ 54 | "cobertura", 55 | "json-summary", 56 | "lcov", 57 | "text", 58 | "text-summary" 59 | ], 60 | "exclude": [ 61 | "config/", 62 | "coverage/", 63 | "bin/", 64 | "**/*.test.js" 65 | ], 66 | "lines": 4, 67 | "functions": 4, 68 | "all": true, 69 | "cache": false, 70 | "temp-directory": "./coverage/.nyc_output" 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /lib/plugins/plugin.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | const chai = require('chai') 3 | const sinon = require('sinon') 4 | const sinonChai = require('sinon-chai') 5 | const moment = require('moment') 6 | 7 | const { DecisionResults, Plugin } = require('./plugin') 8 | 9 | chai.use(sinonChai) 10 | 11 | const { expect } = chai 12 | 13 | describe('Plugin', () => { 14 | describe('constructor', () => { 15 | it('sets the max time', () => { 16 | const plugin = new Plugin('', '', 20) 17 | expect(plugin._maxTime).to.equal(20) 18 | }) 19 | }) 20 | 21 | describe('onExperimentStart', () => { 22 | it('sets the start time', () => { 23 | const plugin = new Plugin() 24 | plugin.onExperimentStart(50) 25 | expect(plugin._startTime).to.equal(50) 26 | }) 27 | }) 28 | 29 | describe('onExperimentStop', () => { 30 | it('clears the start time', () => { 31 | const plugin = new Plugin() 32 | plugin.onExperimentStart(50) 33 | expect(plugin._startTime).to.equal(50) 34 | plugin.onExperimentStop() 35 | expect(plugin._startTime).to.equal(null) 36 | }) 37 | }) 38 | 39 | describe('onExperimentPoll', () => { 40 | it('does not compare time if maxTime is 0', async () => { 41 | const plugin = new Plugin('control', 'treatment', 0) 42 | plugin._poll = sinon.stub().resolves(DecisionResults.WAIT) 43 | plugin.onExperimentStart(moment().utc().subtract('60', 's')) 44 | 45 | const result = await plugin.onExperimentPoll() 46 | expect(result).to.equal(DecisionResults.WAIT) 47 | expect(plugin._poll).to.have.callCount(1) 48 | }) 49 | 50 | it('passes the experiment when maxtime is reached', async () => { 51 | const plugin = new Plugin('control', 'treatment', 60) 52 | plugin._poll = sinon.stub().resolves(DecisionResults.WAIT) 53 | plugin.onExperimentStart(moment().utc().subtract('80', 's')) 54 | 55 | const result = await plugin.onExperimentPoll() 56 | expect(result).to.equal(DecisionResults.PASS) 57 | expect(plugin._poll).to.have.callCount(0) 58 | }) 59 | 60 | it('defers to _poll when maxtime is not yet reached', async () => { 61 | const plugin = new Plugin('control', 'treatment', 60) 62 | plugin._poll = sinon.stub().resolves(DecisionResults.WAIT) 63 | plugin.onExperimentStart(moment().utc().subtract('30', 's')) 64 | 65 | const result = await plugin.onExperimentPoll() 66 | expect(result).to.equal(DecisionResults.WAIT) 67 | expect(plugin._poll).to.have.callCount(1) 68 | }) 69 | }) 70 | }) 71 | -------------------------------------------------------------------------------- /gated-deployments.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Namespace 3 | metadata: 4 | name: kubernetes-gated-deployments 5 | --- 6 | apiVersion: rbac.authorization.k8s.io/v1 7 | kind: ClusterRoleBinding 8 | metadata: 9 | name: kubernetes-gated-deployments-cluster-role-binding 10 | subjects: 11 | - kind: ServiceAccount 12 | name: kubernetes-gated-deployments-service-account 13 | namespace: kubernetes-gated-deployments 14 | roleRef: 15 | kind: ClusterRole 16 | name: kubernetes-gated-deployments-cluster-role 17 | apiGroup: rbac.authorization.k8s.io 18 | --- 19 | apiVersion: rbac.authorization.k8s.io/v1 20 | kind: RoleBinding 21 | metadata: 22 | name: kubernetes-gated-deployments-read-secrets-role-binding 23 | namespace: kubernetes-gated-deployments 24 | subjects: 25 | - kind: ServiceAccount 26 | name: kubernetes-gated-deployments-service-account 27 | namespace: kubernetes-gated-deployments 28 | roleRef: 29 | kind: Role 30 | name: kubernetes-gated-deployments-read-secrets-role 31 | apiGroup: rbac.authorization.k8s.io 32 | --- 33 | apiVersion: rbac.authorization.k8s.io/v1 34 | kind: ClusterRole 35 | metadata: 36 | name: kubernetes-gated-deployments-cluster-role 37 | rules: 38 | - apiGroups: ["apps"] 39 | resources: ["deployments"] 40 | verbs: ["get", "patch", "watch"] 41 | - apiGroups: ["apiextensions.k8s.io"] 42 | resources: ["customresourcedefinitions"] 43 | verbs: ["create"] 44 | - apiGroups: ["apiextensions.k8s.io"] 45 | resources: ["customresourcedefinitions"] 46 | resourceNames: ["gateddeployments.kubernetes-client.io"] 47 | verbs: ["get", "update"] 48 | - apiGroups: ["kubernetes-client.io"] 49 | resources: ["gateddeployments"] 50 | verbs: ["get", "watch", "list"] 51 | --- 52 | apiVersion: rbac.authorization.k8s.io/v1 53 | kind: Role 54 | metadata: 55 | name: kubernetes-gated-deployments-read-secrets-role 56 | namespace: kubernetes-gated-deployments 57 | rules: 58 | - apiGroups: [""] 59 | resources: ["secrets"] 60 | verbs: ["get"] 61 | --- 62 | apiVersion: v1 63 | kind: ServiceAccount 64 | metadata: 65 | name: kubernetes-gated-deployments-service-account 66 | namespace: kubernetes-gated-deployments 67 | --- 68 | apiVersion: apps/v1 69 | kind: Deployment 70 | metadata: 71 | labels: 72 | name: kubernetes-gated-deployments 73 | name: kubernetes-gated-deployments 74 | namespace: kubernetes-gated-deployments 75 | spec: 76 | replicas: 1 77 | selector: 78 | matchLabels: 79 | name: kubernetes-gated-deployments 80 | template: 81 | metadata: 82 | labels: 83 | name: kubernetes-gated-deployments 84 | service: kubernetes-gated-deployments 85 | spec: 86 | serviceAccountName: kubernetes-gated-deployments-service-account 87 | containers: 88 | - image: "godaddy/kubernetes-gated-deployments:1.0.0" 89 | imagePullPolicy: Always 90 | name: kubernetes-gated-deployments 91 | -------------------------------------------------------------------------------- /lib/newrelic.js: -------------------------------------------------------------------------------- 1 | const moment = require('moment') 2 | const rp = require('request-promise') 3 | const util = require('util') 4 | 5 | const AVERAGE_TEMPLATE = 'SELECT average(duration), count(*) FROM Transaction SINCE ' + 6 | "'%s' UNTIL now WHERE host LIKE '%s' AND appName = '%s' AND `request.uri` LIKE '%s'" 7 | 8 | const SAMPLES_TEMPLATE = "SELECT duration FROM Transaction SINCE '%s' UNTIL now WHERE " + 9 | "host LIKE '%s' AND appName = '%s' AND `request.uri` LIKE '%s' LIMIT 1000" 10 | 11 | // 12 | // https://docs.newrelic.com/docs/insights/insights-api/get-data/query-insights-event-data-api 13 | // 14 | class NewRelicClient { 15 | constructor ({ accountId, key }) { 16 | this.accountId = accountId 17 | this.key = key 18 | } 19 | 20 | async _query ({ nrql }) { 21 | const result = await rp({ 22 | method: 'GET', 23 | uri: `https://insights-api.newrelic.com/v1/accounts/${this.accountId}/query`, 24 | json: true, 25 | qs: { 26 | nrql 27 | }, 28 | headers: { 29 | 'X-Query-Key': this.key 30 | } 31 | }) 32 | return result.results 33 | } 34 | 35 | /** 36 | * Return performance of a group (e.g., "treatment" or "control") 37 | * given a hostPrefix, following the convention that host names for the group 38 | * start with the hostPrefix (e.g., the deployment name, 39 | * "example-rest-service-treatment" or "example-rest-service-control"). 40 | * @returns {Promise} Promise object resolving to performance data. 41 | */ 42 | queryAverage ({ since, hostPrefix, appName, pathName }) { 43 | // 44 | // https://docs.newrelic.com/docs/insights/use-insights-ui/time-settings/set-time-range-insights-dashboards-charts 45 | // http://momentjs.com/docs/#/displaying/format/ 46 | // 47 | const utcSince = moment(since).utc().format('YYYY-MM-DD HH:mm:ss') 48 | const nrql = util.format(AVERAGE_TEMPLATE, utcSince, `${hostPrefix}%`, appName, pathName) 49 | return this._query({ nrql }) 50 | } 51 | 52 | async querySamples ({ since, hostPrefix, appName, pathName }) { 53 | const utcSince = moment(since).utc().format('YYYY-MM-DD HH:mm:ss') 54 | const nrql = util.format(SAMPLES_TEMPLATE, utcSince, `${hostPrefix}%`, appName, pathName) 55 | const results = await this._query({ nrql }) 56 | return results[0].events.map(({ duration }) => duration) 57 | } 58 | } 59 | 60 | /** 61 | * Creates and returns a NewRelicClient with account id and key in secrets 62 | * @param {string} namespace - namespace to find secret in 63 | * @param {Object} kubeClient - Client for interacting with kubernetes cluster. 64 | * @param {Object} config - NewRelic config from deployment descriptor that contains 65 | * accountId, secretName, and secretKey 66 | */ 67 | async function getClientFromSecret ({ 68 | namespace, 69 | kubeClient, 70 | config 71 | }) { 72 | const kubeNamespace = kubeClient.api.v1.namespaces(namespace) 73 | const encodedNewRelicKey = (await kubeNamespace.secrets(config.secretName).get()).body.data[config.secretKey] 74 | const newRelicKey = Buffer.from(encodedNewRelicKey, 'base64').toString() 75 | const newRelicClient = new NewRelicClient({ 76 | accountId: config.accountId, 77 | key: newRelicKey 78 | }) 79 | return newRelicClient 80 | } 81 | 82 | module.exports = { 83 | NewRelicClient, 84 | getClientFromSecret 85 | } 86 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies within all project spaces, and it also applies when 49 | an individual is representing the project or its community in public spaces. 50 | Examples of representing a project or community include using an official 51 | project e-mail address, posting via an official social media account, or acting 52 | as an appointed representative at an online or offline event. Representation of 53 | a project may be further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at oss@godaddy.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /lib/deployment-helper.js: -------------------------------------------------------------------------------- 1 | const { isDeepStrictEqual } = require('util') 2 | 3 | const clone = require('clone') 4 | const hash = require('object-hash') 5 | 6 | /** 7 | * Class with utilities to update deployments 8 | */ 9 | class DeploymentHelper { 10 | constructor ({ kubeClient, namespace }) { 11 | this._kubeClient = kubeClient 12 | this._namespace = namespace 13 | } 14 | 15 | /** 16 | * Gets a deployment 17 | * @param {string} name the deployment name 18 | * @returns {Promise} a promise that resolves with the deployment object 19 | */ 20 | get (name) { 21 | const kubeNamespace = this._kubeClient.apis.apps.v1.namespaces(this._namespace) 22 | return kubeNamespace.deploy(name).get() 23 | } 24 | 25 | /** 26 | * Patches a deployment in the namespace 27 | * 28 | * @param {string} name the deployment name 29 | * @param {Object} patch the object to patch the deployment with 30 | * @returns {Promise} a promise that resolves if the patch succeeded 31 | */ 32 | patch (name, patch) { 33 | const kubeNamespace = this._kubeClient.apis.apps.v1.namespaces(this._namespace) 34 | return kubeNamespace.deploy(name).patch(patch) 35 | } 36 | 37 | /** 38 | * Sets deployment to zero replicas. 39 | * @param {string} name - Kubernetes deployment name. 40 | * @returns {Promise} a promise that resolves if the deployment was 41 | * successfully killed 42 | */ 43 | kill (name) { 44 | const replicaSetZero = { 45 | body: { 46 | spec: { 47 | replicas: 0 48 | } 49 | } 50 | } 51 | return this.patch(name, replicaSetZero) 52 | } 53 | 54 | /** 55 | * Sets deployment to the specified pod spec 56 | * @param {string} name - Kubernetes deployment name. 57 | * @param {string} podSpec - Deployment pod spec. 58 | * @returns {Promise} a promise that resolves if the deployment spec was 59 | * successfully updated. 60 | */ 61 | updatePodSpec (name, podSpec) { 62 | const replacementPodSpec = clone(podSpec) 63 | replacementPodSpec.containers.push({ $patch: 'replace' }) 64 | const newDeploymentSpec = { 65 | body: { 66 | spec: { 67 | template: { 68 | spec: replacementPodSpec 69 | } 70 | } 71 | } 72 | } 73 | return this.patch(name, newDeploymentSpec) 74 | } 75 | 76 | /** 77 | * Sets deployment annotation given key and value 78 | * @param {string} name - Kubernetes deployment name. 79 | * @param {string} key - Annotation key 80 | * @param {string} value - Annotation value 81 | * @returns {Promise} a promise that resolves if the annotation was 82 | * successfully set on the deployment 83 | */ 84 | setAnnotation (name, key, value) { 85 | const spec = { 86 | body: { 87 | metadata: { 88 | annotations: { 89 | [key]: value 90 | } 91 | } 92 | } 93 | } 94 | return this.patch(name, spec) 95 | } 96 | 97 | /** 98 | * Compares the pod spec of the two deployments and returns true if they're 99 | * identical and false otherwise 100 | * 101 | * @param {Object} deployment1 a deployment object 102 | * @param {Object} deployment2 a deployment object 103 | * @returns {boolean} true if the pod specs are identical, false otherwise 104 | */ 105 | isPodSpecIdentical (deployment1, deployment2) { 106 | return isDeepStrictEqual(deployment1.spec.template.spec, deployment2.spec.template.spec) 107 | } 108 | 109 | /** 110 | * Returns the hash of the pod spec in the deployment object 111 | * 112 | * @param {Object} deployment a deployment object 113 | * @returns {string} the hash of the pod spec 114 | */ 115 | getPodSpecHash (deployment) { 116 | return hash(deployment.spec.template.spec) 117 | } 118 | } 119 | 120 | module.exports = DeploymentHelper 121 | -------------------------------------------------------------------------------- /lib/plugins/newrelic-performance.js: -------------------------------------------------------------------------------- 1 | const { DecisionResults, Plugin } = require('./plugin') 2 | const analysis = require('../analysis') 3 | const newrelic = require('../newrelic') 4 | 5 | class NewRelicPerformancePlugin extends Plugin { 6 | /** 7 | * Constructs a new NewRelicPerformancePlugin 8 | * @param {Object} options the options object 9 | * @param {Object} options.config the NewRelicPerformancePlugin config object 10 | * @param {Object} options.logger the system logger 11 | * @param {Object} options.newRelicClient client to interact with newrelic 12 | */ 13 | constructor ({ config, logger, newRelicClient, controlName, treatmentName }) { 14 | super(controlName, treatmentName, config.maxTime) 15 | this._config = config 16 | this._logger = logger 17 | this._newRelicClient = newRelicClient 18 | } 19 | /** 20 | * Creates a NewRelicPerformancePlugin from the parameters specified 21 | * in the config object 22 | * @param {Object} options the options object 23 | * @param {string} options.namespace the kubernetes namespace 24 | * @param {Object} options.kubeClient the kubernetes client 25 | * @param {Object} options.logger the system logger 26 | * @param {string} options.controlName the name of the control deployment 27 | * @param {string} options.treatmentName the name of the treatment deployment 28 | * @param {Object} options.config the config object for this plugin 29 | */ 30 | static async build ({ 31 | namespace, 32 | kubeClient, 33 | logger, 34 | controlName, 35 | treatmentName, 36 | config 37 | }) { 38 | const newRelicClient = await newrelic.getClientFromSecret({ 39 | namespace, 40 | kubeClient, 41 | config 42 | }) 43 | 44 | return new NewRelicPerformancePlugin({ config, logger, newRelicClient, controlName, treatmentName }) 45 | } 46 | 47 | /** 48 | * Polls newrelic for timing information and uses the mann-whitney u-test 49 | * to determine the significance of the results. 50 | * @returns {DecisionResults} The result of the analysis, PASS, FAIL, or WAIT 51 | */ 52 | async _poll () { 53 | const [controlResult, treatmentResult] = await Promise.all( 54 | [this._controlName, this._treatmentName].map(this._fetchPerformanceData.bind(this)) 55 | ) 56 | 57 | let analysisResult = analysis.results.notSignificant 58 | if (controlResult.samples.length && treatmentResult.samples.length) { 59 | const testResult = analysis.test({ 60 | controlSamples: controlResult.samples, 61 | treatmentSamples: treatmentResult.samples, 62 | minSamples: this._config.minSamples, 63 | harmThreshold: this._config.harmThreshold, 64 | zScoreThreshold: this._config.zScoreThreshold 65 | }) 66 | analysisResult = testResult.analysisResult 67 | } 68 | 69 | if (analysisResult === analysis.results.harm) { 70 | return DecisionResults.FAIL 71 | } else if (analysisResult === analysis.results.noHarm) { 72 | return DecisionResults.PASS 73 | } else { 74 | return DecisionResults.WAIT 75 | } 76 | } 77 | 78 | /** 79 | * Fetches performance data for the corresponding deployment 80 | * @param {string} name - the deployment name 81 | * @returns {Object} An object including average, count and samples 82 | */ 83 | async _fetchPerformanceData (name) { 84 | const newRelicArgs = { 85 | since: this._startTime, 86 | hostPrefix: name, 87 | appName: this._config.appName, 88 | pathName: this._config.testPath 89 | } 90 | const [summary, samples] = await Promise.all( 91 | [this._newRelicClient.queryAverage, this._newRelicClient.querySamples].map( 92 | queryFn => queryFn.bind(this._newRelicClient, newRelicArgs)() 93 | ) 94 | ) 95 | return { ...summary, samples } 96 | } 97 | } 98 | 99 | module.exports = NewRelicPerformancePlugin 100 | -------------------------------------------------------------------------------- /lib/custom-resource-manager.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const clone = require('clone') 4 | 5 | const SLEEP_MILLISECONDS = 1000 6 | 7 | /** Custom resource manager class. */ 8 | class CustomResourceManager { 9 | /** 10 | * Create custom resource manager. 11 | * @param {Object} kubeClient - Client for interacting with kubernetes cluster. 12 | * @param {Object} logger - Logger for logging stuff. 13 | */ 14 | constructor ({ kubeClient, logger }) { 15 | this._kubeClient = kubeClient 16 | this._logger = logger 17 | } 18 | 19 | /** 20 | * Create custom resource in kubernetes cluster. 21 | * @param {Object} customResourceManifest - Custom resource manifest. 22 | * @returns {Promise} Promise object representing operation result. 23 | */ 24 | _createResource ({ customResourceManifest }) { 25 | return this._kubeClient 26 | .apis['apiextensions.k8s.io'] 27 | .v1beta1 28 | .customresourcedefinitions 29 | .post({ body: customResourceManifest }) 30 | } 31 | 32 | /** 33 | * Get custom resource from kubernetes cluster. 34 | * @param {string} resourceName - Custom resource name. 35 | * @returns {Promise} Promise object representing custom resource. 36 | */ 37 | _getResource ({ resourceName }) { 38 | return this._kubeClient 39 | .apis['apiextensions.k8s.io'] 40 | .v1beta1 41 | .customresourcedefinitions(resourceName) 42 | .get() 43 | } 44 | 45 | /** 46 | * Update a custom resource in kubernetes cluster. 47 | * @param {Object} customResource - Custom resource. 48 | * @param {Object} customResourceManifest - Custom resource manifest. 49 | * @returns {Promise} Promise object representing operation result. 50 | */ 51 | _updateResource ({ customResource, customResourceManifest }) { 52 | const resourceVersion = customResource.body.metadata.resourceVersion 53 | const resourceName = customResource.body.metadata.name 54 | 55 | const body = clone(customResourceManifest) 56 | body.metadata.resourceVersion = resourceVersion 57 | 58 | return this._kubeClient 59 | .apis['apiextensions.k8s.io'] 60 | .v1beta1 61 | .customresourcedefinitions(resourceName) 62 | .put({ body }) 63 | } 64 | 65 | /** 66 | * Block asynchronous flow. 67 | * @param {number} milliseconds - Number of milliseconds to block flow operation. 68 | * @returns {Promise} Promise object representing block flow operation. 69 | */ 70 | _sleep ({ milliseconds }) { 71 | return new Promise(resolve => setTimeout(resolve, milliseconds)) 72 | } 73 | 74 | /** 75 | * Create or update custom resource in kubernetes cluster. 76 | * @param {Object} customResourceManifest - Custom resource manifest. 77 | * @returns {Promise} Promise object representing operation result. 78 | */ 79 | async upsertResource ({ customResourceManifest }) { 80 | const resourceName = customResourceManifest.metadata.name 81 | this._logger.info(`Upserting custom resource ${resourceName}`) 82 | 83 | this._kubeClient.addCustomResourceDefinition(customResourceManifest) 84 | 85 | // try to create the CRD at first 86 | try { 87 | return await this._createResource({ customResourceManifest }) 88 | } catch (err) { 89 | // re-throw the error if request failed for a reason 90 | // other than 409 conflict "exists" 91 | if (err.statusCode !== 409) throw err 92 | } 93 | 94 | for (let attempt = 0; attempt <= 5; attempt++) { 95 | try { 96 | const customResource = await this._getResource({ resourceName }) 97 | return await this._updateResource({ customResource, customResourceManifest }) 98 | } catch (err) { 99 | if (err.statusCode !== 409) throw err 100 | } 101 | 102 | await this._sleep({ milliseconds: SLEEP_MILLISECONDS }) 103 | } 104 | 105 | throw new Error(`Unable to update resource ${resourceName}`) 106 | } 107 | } 108 | 109 | module.exports = CustomResourceManager 110 | -------------------------------------------------------------------------------- /lib/analysis.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 'use strict' 3 | 4 | const { expect } = require('chai') 5 | const sinon = require('sinon') 6 | 7 | const mwu = require('mann-whitney-utest') 8 | 9 | const { test, results } = require('./analysis') 10 | 11 | describe('analysis', () => { 12 | afterEach(() => { 13 | sinon.restore() 14 | }) 15 | 16 | describe('test', () => { 17 | it('returns notSignificant if not enough samples in control', () => { 18 | const controlSamples = Array(49) 19 | const treatmentSamples = Array(50) 20 | const result = test({ controlSamples, treatmentSamples, minSamples: 50 }) 21 | expect(result).to.deep.equal({ u: [0, 0], analysisResult: results.notSignificant }) 22 | }) 23 | 24 | it('returns notSignificant if not enough samples in treatment', () => { 25 | const controlSamples = Array(50) 26 | const treatmentSamples = Array(49) 27 | const result = test({ controlSamples, treatmentSamples, minSamples: 50 }) 28 | expect(result).to.deep.equal({ u: [0, 0], analysisResult: results.notSignificant }) 29 | }) 30 | 31 | it('returns noHarm if criticalValue less than or equal to default zScoreThreshold', () => { 32 | const controlSamples = Array(50) 33 | const treatmentSamples = Array(50) 34 | sinon.stub(mwu, 'test').returns([1250, 1250]) 35 | sinon.stub(mwu, 'criticalValue').returns(1.96) 36 | const result = test({ controlSamples, treatmentSamples, minSamples: 50 }) 37 | expect(result).to.deep.equal({ u: [1250, 1250], analysisResult: results.noHarm }) 38 | }) 39 | 40 | it('returns noHarm if criticalValue less than or equal to custom zScoreThreshold', () => { 41 | const controlSamples = Array(50) 42 | const treatmentSamples = Array(50) 43 | sinon.stub(mwu, 'test').returns([1250, 1250]) 44 | sinon.stub(mwu, 'criticalValue').returns(2.24) 45 | const result = test({ controlSamples, treatmentSamples, minSamples: 50, zScoreThreshold: 2.5 }) 46 | expect(result).to.deep.equal({ u: [1250, 1250], analysisResult: results.noHarm }) 47 | }) 48 | 49 | it('returns noHarm if treatment U is less than or equal to default harmThreshold times control U', () => { 50 | const controlSamples = Array(50) 51 | const treatmentSamples = Array(50) 52 | sinon.stub(mwu, 'test').returns([2000, 1000]) 53 | sinon.stub(mwu, 'criticalValue').returns(2.24) 54 | const result = test({ controlSamples, treatmentSamples, minSamples: 50 }) 55 | expect(result).to.deep.equal({ u: [2000, 1000], analysisResult: results.noHarm }) 56 | }) 57 | 58 | it('returns harm if treatment U is more than default harmThreshold times control U', () => { 59 | const controlSamples = Array(50) 60 | const treatmentSamples = Array(50) 61 | sinon.stub(mwu, 'test').returns([1000, 2000]) 62 | sinon.stub(mwu, 'criticalValue').returns(2.24) 63 | const result = test({ controlSamples, treatmentSamples, minSamples: 50 }) 64 | expect(result).to.deep.equal({ u: [1000, 2000], analysisResult: results.harm }) 65 | }) 66 | 67 | it('returns noHarm if treatment U is less than or equal to custom harmThreshold times control U', () => { 68 | const controlSamples = Array(50) 69 | const treatmentSamples = Array(50) 70 | sinon.stub(mwu, 'test').returns([1000, 2000]) 71 | sinon.stub(mwu, 'criticalValue').returns(2.24) 72 | const result = test({ controlSamples, treatmentSamples, minSamples: 50, harmThreshold: 2.5 }) 73 | expect(result).to.deep.equal({ u: [1000, 2000], analysisResult: results.noHarm }) 74 | }) 75 | 76 | it('returns harm if treatment U is more than custom harmThreshold times control U', () => { 77 | const controlSamples = Array(50) 78 | const treatmentSamples = Array(50) 79 | sinon.stub(mwu, 'test').returns([1000, 3000]) 80 | sinon.stub(mwu, 'criticalValue').returns(2.7) 81 | const result = test({ controlSamples, treatmentSamples, minSamples: 50, harmThreshold: 2.5, zScoreThreshold: 2.5 }) 82 | expect(result).to.deep.equal({ u: [1000, 3000], analysisResult: results.harm }) 83 | }) 84 | }) 85 | }) 86 | -------------------------------------------------------------------------------- /lib/watchers/gated-deployment-watcher.js: -------------------------------------------------------------------------------- 1 | const DeploymentWatcher = require('./deployment-watcher') 2 | const Watcher = require('./watcher') 3 | const pluginFactory = require('../plugins/plugin-factory') 4 | 5 | class GatedDeploymentWatcher extends Watcher { 6 | /** 7 | * Creates a gated deployment watcher 8 | * 9 | * @param {Object} options the options object 10 | * @param {string} options.controllerNamespace namespace of the gated-deployment controller 11 | * @param {Object} options.kubeClient the kube client instance 12 | * @param {Object} options.logger the logger instance 13 | * @param {Number} options.pollerIntervalMilliseconds the poller interval in milliseconds 14 | */ 15 | constructor ({ controllerNamespace, kubeClient, logger, pollerIntervalMilliseconds }) { 16 | super() 17 | this._controllerNamespace = controllerNamespace 18 | this._kubeClient = kubeClient 19 | this._logger = logger 20 | this._pollerIntervalMilliseconds = pollerIntervalMilliseconds 21 | this._deploymentWatchers = {} 22 | } 23 | 24 | /** 25 | * Returns a gated deployments watch stream 26 | * 27 | * @returns {Object} the watch stream 28 | */ 29 | getStream () { 30 | return this._kubeClient.apis['kubernetes-client.io'].v1.watch.gateddeployments.getStream() 31 | } 32 | 33 | /** 34 | * Creates and starts a deployment watcher for the gated deployment 35 | * 36 | * @param {Object} gatedDeployment the gated deployment object 37 | */ 38 | async _createDeploymentWatcher (gatedDeployment) { 39 | const { id, namespace } = this._extractResourceMetadata(gatedDeployment) 40 | const deploymentDescriptor = gatedDeployment.deploymentDescriptor 41 | 42 | this._logger.info(`${id}: Creating and starting deployment watcher`) 43 | 44 | try { 45 | const plugins = await pluginFactory.buildPluginsFromConfig({ 46 | namespace: this._controllerNamespace, 47 | kubeClient: this._kubeClient, 48 | logger: this._logger, 49 | controlName: deploymentDescriptor.control.name, 50 | treatmentName: deploymentDescriptor.treatment.name, 51 | decisionPluginConfig: deploymentDescriptor.decisionPlugins 52 | }) 53 | 54 | this._deploymentWatchers[id] = new DeploymentWatcher({ 55 | kubeClient: this._kubeClient, 56 | plugins, 57 | logger: this._logger, 58 | namespace, 59 | deploymentDescriptor, 60 | pollerIntervalMilliseconds: this._pollerIntervalMilliseconds, 61 | gatedDeploymentId: id 62 | }) 63 | 64 | this._deploymentWatchers[id].start() 65 | } catch (err) { 66 | this._logger.error(`${id}: Failed to create and start deployment watcher`, err.message) 67 | } 68 | } 69 | 70 | /** 71 | * Stops and removes a deployment watcher for the gated deployment 72 | * 73 | * @param {Object} gatedDeployment the gated deployment object 74 | */ 75 | _removeDeploymentWatcher (gatedDeployment) { 76 | const { id } = this._extractResourceMetadata(gatedDeployment) 77 | 78 | this._logger.info(`${id}: Stopping and removing deployment watcher`) 79 | 80 | if (this._deploymentWatchers[id]) { 81 | this._deploymentWatchers[id].stop() 82 | delete this._deploymentWatchers[id] 83 | } else { 84 | this._logger.warn(`${id}: No deployment watcher found`) 85 | } 86 | } 87 | 88 | /** 89 | * Creates a deployment watcher 90 | * 91 | * @param {Object} gatedDeployment the gated deployment object 92 | * @returns {Promise} a promise that resolves when the deployment watcher is 93 | * created. 94 | */ 95 | onAdded (gatedDeployment) { 96 | return this._createDeploymentWatcher(gatedDeployment) 97 | } 98 | 99 | /** 100 | * Removes existing deployment watcher and creates a new deployment watcher 101 | * 102 | * @param {Object} gatedDeployment the gated deployment object 103 | * @returns {Promise} a promise that resolves when the deployment watcher is 104 | * re-created. 105 | */ 106 | onModified (gatedDeployment) { 107 | this._removeDeploymentWatcher(gatedDeployment) 108 | return this._createDeploymentWatcher(gatedDeployment) 109 | } 110 | 111 | /** 112 | * Removes the deployment watcher 113 | * 114 | * @param {Object} gatedDeployment the gated deployment object 115 | */ 116 | onDeleted (gatedDeployment) { 117 | this._removeDeploymentWatcher(gatedDeployment) 118 | } 119 | } 120 | 121 | module.exports = GatedDeploymentWatcher 122 | -------------------------------------------------------------------------------- /lib/newrelic.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 'use strict' 3 | 4 | const chai = require('chai') 5 | const nock = require('nock') 6 | const sinon = require('sinon') 7 | const sinonChai = require('sinon-chai') 8 | 9 | const { NewRelicClient, getClientFromSecret } = require('./newrelic') 10 | 11 | chai.use(sinonChai) 12 | 13 | const { expect } = chai 14 | 15 | describe('NewRelicClient', () => { 16 | let client 17 | 18 | beforeEach(() => { 19 | client = new NewRelicClient({ accountId: '1234', key: 'abc' }) 20 | }) 21 | 22 | afterEach(() => { 23 | sinon.restore() 24 | nock.cleanAll() 25 | }) 26 | 27 | describe('constructor', () => { 28 | it('sets accountId and key from constructor params', () => { 29 | expect(client.accountId).to.equal('1234') 30 | expect(client.key).to.equal('abc') 31 | }) 32 | }) 33 | 34 | describe('_query', () => { 35 | it('makes a request and returns the results', async () => { 36 | const scope = nock('https://insights-api.newrelic.com:443', { 37 | reqHeaders: { 38 | 'X-Query-Key': 'abc' 39 | } 40 | }) 41 | .get('/v1/accounts/1234/query') 42 | .query({ nrql: 'nrql query' }) 43 | .reply(200, { results: 'results' }) 44 | const queryResult = await client._query({ nrql: 'nrql query' }) 45 | expect(queryResult).to.equal('results') 46 | expect(scope.isDone()).to.equal(true) 47 | }) 48 | }) 49 | 50 | describe('queryAverage', () => { 51 | it('queries with average template query', async () => { 52 | const since = '2019-01-01T14:00:00.000Z' 53 | const hostPrefix = 'host' 54 | const appName = 'app-name' 55 | const pathName = '/path/name' 56 | const expectedQuery = 'SELECT average(duration), count(*) FROM Transaction' + 57 | " SINCE '2019-01-01 14:00:00' UNTIL now WHERE host LIKE 'host%'" + 58 | " AND appName = 'app-name' AND `request.uri` LIKE '/path/name'" 59 | client._query = sinon.stub().resolves('average result') 60 | const averageResult = await client.queryAverage({ since, hostPrefix, appName, pathName }) 61 | expect(averageResult).to.equal('average result') 62 | expect(client._query).to.have.been.calledOnceWith({ nrql: expectedQuery }) 63 | }) 64 | }) 65 | 66 | describe('querySamples', () => { 67 | it('queries with samples template query', async () => { 68 | const since = '2019-01-01T14:00:00.000Z' 69 | const hostPrefix = 'host' 70 | const appName = 'app-name' 71 | const pathName = '/path/name' 72 | const expectedQuery = "SELECT duration FROM Transaction SINCE '2019-01-01 14:00:00'" + 73 | " UNTIL now WHERE host LIKE 'host%' AND appName = 'app-name' AND" + 74 | " `request.uri` LIKE '/path/name' LIMIT 1000" 75 | client._query = sinon.stub().resolves([{ events: [{ duration: 1 }, { duration: 3 }] }]) 76 | const sampleResult = await client.querySamples({ since, hostPrefix, appName, pathName }) 77 | expect(sampleResult).to.deep.equal([1, 3]) 78 | expect(client._query).to.have.been.calledOnceWith({ nrql: expectedQuery }) 79 | }) 80 | }) 81 | }) 82 | 83 | describe('getClientFromSecret', () => { 84 | let kubeClientStub 85 | let namespaceStub 86 | let secretStub 87 | 88 | beforeEach(() => { 89 | namespaceStub = sinon.stub() 90 | secretStub = sinon.stub() 91 | kubeClientStub = sinon.stub() 92 | kubeClientStub.api = sinon.stub() 93 | kubeClientStub.api.v1 = sinon.stub() 94 | kubeClientStub.api.v1.namespaces = sinon.stub().returns(namespaceStub) 95 | namespaceStub.secrets = sinon.stub().returns(secretStub) 96 | secretStub.get = sinon.stub().resolves({ 97 | body: { 98 | data: { 99 | example: 'c2VjcmV0' // 'secret' base64 encoded 100 | } 101 | } 102 | }) 103 | }) 104 | 105 | afterEach(() => { 106 | sinon.restore() 107 | }) 108 | 109 | it('creates and returns NewRelicClient', async () => { 110 | const newRelicClient = await getClientFromSecret({ 111 | namespace: 'ns', 112 | kubeClient: kubeClientStub, 113 | config: { 114 | accountId: '123', 115 | secretName: 'newrelic-secrets', 116 | secretKey: 'example' 117 | } 118 | }) 119 | expect(kubeClientStub.api.v1.namespaces).to.have.been.calledOnceWith('ns') 120 | expect(namespaceStub.secrets).to.have.been.calledOnceWith('newrelic-secrets') 121 | expect(newRelicClient.accountId).to.equal('123') 122 | expect(newRelicClient.key).to.equal('secret') 123 | }) 124 | }) 125 | -------------------------------------------------------------------------------- /lib/deployment-helper.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | const chai = require('chai') 3 | const clone = require('clone') 4 | const hash = require('object-hash') 5 | const sinon = require('sinon') 6 | const sinonChai = require('sinon-chai') 7 | 8 | const DeploymentHelper = require('./deployment-helper') 9 | 10 | chai.use(sinonChai) 11 | 12 | const { expect } = chai 13 | 14 | describe('DeploymentHelper', () => { 15 | let kubeNamespaceMock, deployMock, kubeClientMock, deploymentHelper 16 | const deployment = { some: 'deploy' } 17 | 18 | beforeEach(() => { 19 | kubeClientMock = sinon.mock() 20 | deployMock = sinon.mock() 21 | deployMock.patch = sinon.stub().resolves() 22 | deployMock.get = sinon.stub().resolves(deployment) 23 | kubeNamespaceMock = sinon.mock() 24 | kubeNamespaceMock.deploy = sinon.stub().returns(deployMock) 25 | kubeClientMock.apis = sinon.mock() 26 | kubeClientMock.apis.apps = sinon.mock() 27 | kubeClientMock.apis.apps.v1 = sinon.mock() 28 | kubeClientMock.apis.apps.v1.namespaces = sinon.stub().returns(kubeNamespaceMock) 29 | 30 | deploymentHelper = new DeploymentHelper({ 31 | kubeClient: kubeClientMock, 32 | namespace: 'ns' 33 | }) 34 | }) 35 | 36 | describe('.get', () => { 37 | it('returns the deployment', async () => { 38 | const deploy = await deploymentHelper.get('foo') 39 | 40 | expect(deploy).to.deep.equal(deployment) 41 | expect(kubeNamespaceMock.deploy).to.have.been.calledWith('foo') 42 | }) 43 | }) 44 | 45 | describe('.patch', () => { 46 | it('patches the deployment', async () => { 47 | await deploymentHelper.patch('foo', { some: 'patch' }) 48 | 49 | expect(kubeNamespaceMock.deploy).to.have.been.calledWith('foo') 50 | expect(deployMock.patch).to.have.been.calledWith({ some: 'patch' }) 51 | }) 52 | }) 53 | 54 | describe('.kill', () => { 55 | it('patches the deployment to set replicas to 0', async () => { 56 | await deploymentHelper.kill('foo') 57 | 58 | expect(kubeNamespaceMock.deploy).to.have.been.calledWith('foo') 59 | expect(deployMock.patch).to.have.been.calledWith({ 60 | body: { 61 | spec: { 62 | replicas: 0 63 | } 64 | } 65 | }) 66 | }) 67 | }) 68 | 69 | describe('.updatePodSpec', () => { 70 | it('patches the deployment name with the given pod spec', async () => { 71 | const newSpec = { 72 | replicas: 2, 73 | containers: [ 74 | { 75 | image: 'bar:latest' 76 | } 77 | ] 78 | } 79 | 80 | await deploymentHelper.updatePodSpec('foo', newSpec) 81 | 82 | expect(kubeNamespaceMock.deploy).to.have.been.calledWith('foo') 83 | expect(deployMock.patch).to.have.been.calledWith({ 84 | body: { 85 | spec: { 86 | template: { 87 | spec: { 88 | ...newSpec, 89 | containers: [ 90 | ...newSpec.containers, 91 | { 92 | $patch: 'replace' 93 | } 94 | ] 95 | } 96 | } 97 | } 98 | } 99 | }) 100 | }) 101 | }) 102 | 103 | describe('.setAnnotation', () => { 104 | it('sets the annotation key/value pair', async () => { 105 | await deploymentHelper.setAnnotation('foo', 'gatedDeploymentStatus', 'success') 106 | 107 | expect(kubeNamespaceMock.deploy).to.have.been.calledWith('foo') 108 | expect(deployMock.patch).to.have.been.calledWith({ 109 | body: { 110 | metadata: { 111 | annotations: { 112 | gatedDeploymentStatus: 'success' 113 | } 114 | } 115 | } 116 | }) 117 | }) 118 | }) 119 | 120 | describe('.isPodSpecIdentical', () => { 121 | const podSpec = { 122 | containers: [ 123 | { name: 'pod1', image: 'image1' }, 124 | { name: 'pod2', image: 'image2' } 125 | ] 126 | } 127 | let deployment1, deployment2 128 | 129 | beforeEach(() => { 130 | deployment1 = { 131 | metadata: { 132 | name: 'deploy1' 133 | }, 134 | spec: { template: { spec: clone(podSpec) } } 135 | } 136 | deployment2 = { 137 | metadata: { 138 | name: 'deploy2' 139 | }, 140 | spec: { template: { spec: clone(podSpec) } } 141 | } 142 | }) 143 | 144 | it('returns true for identical pod specs', () => { 145 | expect(deploymentHelper.isPodSpecIdentical(deployment1, deployment2)).to.equal(true) 146 | }) 147 | 148 | it('returns false for non identical pod specs', () => { 149 | deployment2.spec.template.spec.containers[0].image = 'image11' 150 | expect(deploymentHelper.isPodSpecIdentical(deployment1, deployment2)).to.equal(false) 151 | }) 152 | }) 153 | 154 | describe('.getPodSpecHash', () => { 155 | it('returns hash of pod spec', () => { 156 | const podSpec = { 157 | containers: [ 158 | { name: 'pod1', image: 'image1' }, 159 | { name: 'pod2', image: 'image2' } 160 | ] 161 | } 162 | const deployment = { 163 | metadata: { 164 | name: 'deploy1' 165 | }, 166 | spec: { template: { spec: clone(podSpec) } } 167 | } 168 | 169 | expect(deploymentHelper.getPodSpecHash(deployment)).to.equal(hash(podSpec)) 170 | }) 171 | }) 172 | }) 173 | -------------------------------------------------------------------------------- /lib/plugins/newrelic-performance.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | const chai = require('chai') 3 | const sinon = require('sinon') 4 | const sinonChai = require('sinon-chai') 5 | 6 | const NewRelicPerformancePlugin = require('./newrelic-performance') 7 | const { DecisionResults } = require('./plugin') 8 | const analysis = require('../analysis') 9 | const newrelic = require('../newrelic') 10 | 11 | chai.use(sinonChai) 12 | 13 | const { expect } = chai 14 | 15 | describe('NewRelicPerformancePlugin', () => { 16 | let plugin 17 | const samples = [1, 2, 3] 18 | const config = { 19 | accountId: '807783', 20 | secretName: 'newrelic-secrets', 21 | secretKey: 'example-rest-service', 22 | appName: 'example-rest-service', 23 | minSamples: 50, 24 | maxTime: 600, 25 | testPath: '/shopper/products', 26 | harmThreshold: 1.25, 27 | zScoreThreshold: 1.5 28 | } 29 | 30 | beforeEach(() => { 31 | plugin = new NewRelicPerformancePlugin({ 32 | config, 33 | logger: { 34 | info: sinon.stub() 35 | }, 36 | newRelicClient: { 37 | queryAverage: sinon.stub().resolves({ 38 | count: 20, 39 | average: 30 40 | }), 41 | querySamples: sinon.stub().resolves(samples) 42 | } 43 | }) 44 | }) 45 | 46 | afterEach(() => { 47 | sinon.restore() 48 | }) 49 | 50 | describe('build', () => { 51 | it('initializes the newrelic client', async () => { 52 | const newRelicStub = sinon.stub(newrelic, 'getClientFromSecret').resolves('mockNRClient') 53 | 54 | const result = await NewRelicPerformancePlugin.build({ 55 | namespace: 'mockNamespace', 56 | kubeClient: 'mockClient', 57 | logger: 'mockLogger', 58 | config: { 59 | maxTime: 10 60 | } 61 | }) 62 | expect(result._config).to.deep.equal({ maxTime: 10 }) 63 | expect(result._logger).to.equal('mockLogger') 64 | expect(result._newRelicClient).to.equal('mockNRClient') 65 | expect(result._maxTime).to.equal(10) 66 | expect(newRelicStub).to.have.been.calledWith({ 67 | namespace: 'mockNamespace', 68 | kubeClient: 'mockClient', 69 | config: { 70 | maxTime: 10 71 | } 72 | }) 73 | }) 74 | }) 75 | 76 | describe('._poll', () => { 77 | beforeEach(() => { 78 | plugin._fetchPerformanceData = sinon.stub().resolves({ samples }) 79 | }) 80 | 81 | it('returns PASS if analysis returns no harm', async () => { 82 | const testStub = sinon.stub(analysis, 'test').returns({ analysisResult: analysis.results.noHarm }) 83 | 84 | const result = await plugin._poll() 85 | 86 | expect(result).to.equal(DecisionResults.PASS) 87 | expect(plugin._fetchPerformanceData).to.have.callCount(2) 88 | expect(testStub).to.have.been.calledWith({ 89 | controlSamples: samples, 90 | treatmentSamples: samples, 91 | minSamples: config.minSamples, 92 | harmThreshold: config.harmThreshold, 93 | zScoreThreshold: config.zScoreThreshold 94 | }) 95 | }) 96 | 97 | it('returns FAIL if analysis returns harm', async () => { 98 | const testStub = sinon.stub(analysis, 'test').returns({ analysisResult: analysis.results.harm }) 99 | 100 | const result = await plugin._poll() 101 | 102 | expect(result).to.equal(DecisionResults.FAIL) 103 | expect(plugin._fetchPerformanceData).to.have.callCount(2) 104 | expect(testStub).to.have.been.calledWith({ 105 | controlSamples: samples, 106 | treatmentSamples: samples, 107 | minSamples: config.minSamples, 108 | harmThreshold: config.harmThreshold, 109 | zScoreThreshold: config.zScoreThreshold 110 | }) 111 | }) 112 | 113 | it('return WAIT if analysis returns not significant', async () => { 114 | const testStub = sinon.stub(analysis, 'test').returns({ analysisResult: analysis.results.notSignificant }) 115 | 116 | const result = await plugin._poll() 117 | 118 | expect(result).to.equal(DecisionResults.WAIT) 119 | expect(plugin._fetchPerformanceData).to.have.callCount(2) 120 | expect(testStub).to.have.been.calledWith({ 121 | controlSamples: samples, 122 | treatmentSamples: samples, 123 | minSamples: config.minSamples, 124 | harmThreshold: config.harmThreshold, 125 | zScoreThreshold: config.zScoreThreshold 126 | }) 127 | }) 128 | 129 | it('Will not analyse if missing samples', async () => { 130 | const testStub = sinon.stub(analysis, 'test') 131 | plugin._fetchPerformanceData = sinon.stub().resolves({ samples: [] }) 132 | 133 | const result = await plugin._poll() 134 | 135 | expect(result).to.equal(DecisionResults.WAIT) 136 | expect(plugin._fetchPerformanceData).to.have.callCount(2) 137 | expect(testStub).to.have.callCount(0) 138 | }) 139 | }) 140 | 141 | describe('._fetchPerformanceData', () => { 142 | it('fetches data for a split', async () => { 143 | plugin._startTime = 15 144 | const results = await plugin._fetchPerformanceData('control') 145 | 146 | const expectedArgs = { 147 | since: 15, 148 | hostPrefix: 'control', 149 | appName: config.appName, 150 | pathName: config.testPath 151 | } 152 | expect(plugin._newRelicClient.queryAverage).to.have.been.calledWith(expectedArgs) 153 | expect(plugin._newRelicClient.querySamples).to.have.been.calledWith(expectedArgs) 154 | expect(results).to.deep.equal({ 155 | count: 20, 156 | average: 30, 157 | samples: [1, 2, 3] 158 | }) 159 | }) 160 | }) 161 | }) 162 | -------------------------------------------------------------------------------- /lib/watchers/watcher.js: -------------------------------------------------------------------------------- 1 | const AsyncLock = require('async-lock') 2 | const JSONStream = require('json-stream') 3 | 4 | const { StreamEvents, WatchEvents, WatcherStates, EVENT_LOCK } = require('./constants') 5 | 6 | /** 7 | * Watcher abstraction for watching any kubernetes resource 8 | */ 9 | class Watcher { 10 | constructor () { 11 | this._activeResources = {} 12 | this._lock = new AsyncLock() 13 | this._state = WatcherStates.READY 14 | } 15 | 16 | /** 17 | * Extracts kubernetes resource metadata 18 | * 19 | * @param {Object} resource the kubernetes resource 20 | * @returns {Object} metadata including id, namespace, and resource version 21 | */ 22 | _extractResourceMetadata (resource) { 23 | const { metadata: { name, namespace, resourceVersion } } = resource 24 | const id = `${namespace}_${name}` 25 | return { id, namespace, resourceVersion } 26 | } 27 | 28 | /** 29 | * Creates and returns a kubernetes watch stream 30 | */ 31 | getStream () { 32 | throw new Error('Must be overridden') 33 | } 34 | 35 | /** 36 | * Invoked when the watch stream emits an end event 37 | */ 38 | onEnd () { 39 | // Set an end event flag for all resources. This flag will be cleared on 40 | // all resources for which we receive an add event again when we create a 41 | // new stream. If this flag already exists for a resource, it means that it 42 | // was deleted during the period when a stream ended and restarted again. 43 | // Call delete for these resources. 44 | Object.entries(this._activeResources).forEach(([id, { resource, endEvent }]) => { 45 | if (endEvent) { 46 | this._onDeleted(resource) 47 | } else { 48 | this._activeResources[id].endEvent = true 49 | } 50 | }) 51 | 52 | // If the watcher was stopped, don't restart. Only restart if the end event 53 | // was caused by the watch stream terminating automatically. 54 | if (this._state === WatcherStates.ENDED) return 55 | 56 | this.start() 57 | } 58 | 59 | /** 60 | * Invoked when an ADDED event is received. Added events are received when 61 | * either a resource is actually added or when a stream ends a new stream 62 | * starts. Only calls onAdded if a resource was actually added. 63 | * 64 | * @param {Object} resource the kubernetes resource 65 | */ 66 | _onAdded (resource) { 67 | const { id, resourceVersion } = this._extractResourceMetadata(resource) 68 | // When a stream closes and a new stream starts, we get an added event for 69 | // all existing resources, so they could already exist. 70 | if (this._activeResources[id]) { 71 | // Clear the end event flag. 72 | delete this._activeResources[id].endEvent 73 | // Resources could be modified during the time when the stream closes and 74 | // a new stream is added. Check if resource version is the same and call 75 | // the modified event handler if they've changed. 76 | if (resourceVersion !== this._activeResources[id].resourceVersion) { 77 | this._onModified(resource) 78 | } 79 | } else { 80 | this._activeResources[id] = { resourceVersion, resource } 81 | this._lock.acquire(EVENT_LOCK, this.onAdded.bind(this, resource)) 82 | } 83 | } 84 | 85 | /** 86 | * Invoked when a resource is actually added. Must be overridden in subclass. 87 | */ 88 | onAdded () { 89 | throw new Error('Must be overridden') 90 | } 91 | 92 | /** 93 | * Invoked when a MODIFIED event is received. It updates activeResources and 94 | * calls onModified. 95 | * 96 | * @param {Object} resource the kubernetes resource 97 | */ 98 | _onModified (resource) { 99 | const { id, resourceVersion } = this._extractResourceMetadata(resource) 100 | this._activeResources[id] = { resourceVersion, resource } 101 | this._lock.acquire(EVENT_LOCK, this.onModified.bind(this, resource)) 102 | } 103 | 104 | /** 105 | * Invoked when a resource is modified. Must be overridden in subclass. 106 | */ 107 | onModified () { 108 | throw new Error('Must be overridden') 109 | } 110 | 111 | /** 112 | * Invoked when a DELETED event is received. It updates activeResources and 113 | * calls onDeleted. 114 | * 115 | * @param {Object} resource the kubernetes resource 116 | */ 117 | _onDeleted (resource) { 118 | const { id } = this._extractResourceMetadata(resource) 119 | delete this._activeResources[id] 120 | this._lock.acquire(EVENT_LOCK, this.onDeleted.bind(this, resource)) 121 | } 122 | 123 | /** 124 | * Invoked when a resource is deleted. Must be overridden in subclass. 125 | */ 126 | onDeleted () { 127 | throw new Error('Must be overridden') 128 | } 129 | 130 | /** 131 | * Creates the kubernetes watch stream and starts listening to events and 132 | * invokes the method corresponding to the event 133 | */ 134 | start () { 135 | this._state = WatcherStates.RUNNING 136 | this.stream = this.getStream() 137 | 138 | this.stream.on(StreamEvents.END, this.onEnd.bind(this)) 139 | 140 | const jsonStream = new JSONStream() 141 | this.stream.pipe(jsonStream) 142 | 143 | jsonStream.on(StreamEvents.DATA, event => { 144 | switch (event.type) { 145 | case WatchEvents.ADDED: 146 | this._onAdded(event.object) 147 | break 148 | case WatchEvents.MODIFIED: 149 | this._onModified(event.object) 150 | break 151 | case WatchEvents.DELETED: 152 | this._onDeleted(event.object) 153 | break 154 | default: 155 | break 156 | } 157 | }) 158 | } 159 | 160 | /** 161 | * Aborts the watch stream and calls _onDeleted to clean up all resources 162 | */ 163 | stop () { 164 | this._state = WatcherStates.ENDED 165 | if (this.stream) { 166 | this.stream.abort() 167 | } 168 | Object.entries(this._activeResources).forEach(([, { resource }]) => { 169 | this._onDeleted(resource) 170 | }) 171 | } 172 | } 173 | 174 | module.exports = Watcher 175 | -------------------------------------------------------------------------------- /lib/custom-resource-manager.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 'use strict' 3 | 4 | const chai = require('chai') 5 | const sinon = require('sinon') 6 | const sinonChai = require('sinon-chai') 7 | 8 | const CustomResourceManager = require('./custom-resource-manager') 9 | 10 | chai.use(sinonChai) 11 | 12 | const { expect } = chai 13 | 14 | describe('CustomResourceManager', () => { 15 | let customResourceManager 16 | let fakeCustomResource 17 | let fakeCustomResourceManifest 18 | let kubeClientMock 19 | let loggerMock 20 | let v1beta1Mock 21 | 22 | beforeEach(() => { 23 | fakeCustomResource = { 24 | body: { 25 | metadata: { 26 | name: 'fakeName', 27 | resourceVersion: 'fakeResourceVersion' 28 | } 29 | } 30 | } 31 | fakeCustomResourceManifest = { 32 | kind: 'CustomResourceDefinition', 33 | metadata: { 34 | name: 'fakeName' 35 | } 36 | } 37 | v1beta1Mock = sinon.mock() 38 | v1beta1Mock.customresourcedefinitions = sinon.stub() 39 | kubeClientMock = sinon.mock() 40 | kubeClientMock.apis = sinon.mock() 41 | kubeClientMock.apis['apiextensions.k8s.io'] = sinon.mock() 42 | kubeClientMock.apis['apiextensions.k8s.io'].v1beta1 = v1beta1Mock 43 | loggerMock = sinon.mock() 44 | customResourceManager = new CustomResourceManager({ 45 | kubeClient: kubeClientMock, 46 | logger: loggerMock 47 | }) 48 | }) 49 | 50 | describe('_createResource', () => { 51 | it('creates custom resource', async () => { 52 | v1beta1Mock.customresourcedefinitions.post = sinon.stub().resolves() 53 | 54 | await customResourceManager._createResource({ 55 | customResourceManifest: fakeCustomResourceManifest 56 | }) 57 | 58 | expect(v1beta1Mock.customresourcedefinitions.post).to.have.been.calledWith({ 59 | body: fakeCustomResourceManifest 60 | }) 61 | }) 62 | }) 63 | 64 | describe('_getResource', () => { 65 | it('gets custom resource', async () => { 66 | v1beta1Mock.customresourcedefinitions 67 | .returns(v1beta1Mock.customresourcedefinitions) 68 | v1beta1Mock.customresourcedefinitions.get = sinon.stub() 69 | .resolves(fakeCustomResource) 70 | 71 | const fakeResourceName = fakeCustomResourceManifest.metadata.name 72 | 73 | const customResource = await customResourceManager._getResource({ 74 | resourceName: fakeResourceName 75 | }) 76 | 77 | expect(customResource).deep.equals(fakeCustomResource) 78 | expect(v1beta1Mock.customresourcedefinitions).to.have.been.calledWith(fakeResourceName) 79 | expect(v1beta1Mock.customresourcedefinitions.get).to.have.callCount(1) 80 | }) 81 | }) 82 | 83 | describe('_updateResource', () => { 84 | it('updates custom resource', async () => { 85 | v1beta1Mock.customresourcedefinitions 86 | .returns(v1beta1Mock.customresourcedefinitions) 87 | v1beta1Mock.customresourcedefinitions.put = sinon.stub().resolves() 88 | 89 | const fakeResourceName = fakeCustomResourceManifest.metadata.name 90 | 91 | await customResourceManager._updateResource({ 92 | customResource: fakeCustomResource, 93 | customResourceManifest: fakeCustomResourceManifest 94 | }) 95 | 96 | // NOTE(jdaeli): no need to deep clone here since 97 | // we create the object before each test 98 | const fakeResourceVersion = fakeCustomResource.body.metadata.resourceVersion 99 | fakeCustomResourceManifest.metadata.resourceVersion = fakeResourceVersion 100 | 101 | expect(v1beta1Mock.customresourcedefinitions).to.have.been.calledWith(fakeResourceName) 102 | expect(v1beta1Mock.customresourcedefinitions.put).to.have.been.calledWith({ 103 | body: fakeCustomResourceManifest 104 | }) 105 | }) 106 | }) 107 | 108 | describe('upsertResource', () => { 109 | beforeEach(() => { 110 | sinon.stub(customResourceManager, '_createResource') 111 | sinon.stub(customResourceManager, '_getResource') 112 | sinon.stub(customResourceManager, '_updateResource') 113 | kubeClientMock.addCustomResourceDefinition = sinon.stub() 114 | loggerMock.info = sinon.stub() 115 | }) 116 | 117 | it('creates custom resource', async () => { 118 | customResourceManager._createResource.resolves() 119 | 120 | const fakeResourceName = fakeCustomResourceManifest.metadata.name 121 | const logInfoMsg = `Upserting custom resource ${fakeResourceName}` 122 | 123 | await customResourceManager.upsertResource({ 124 | customResourceManifest: fakeCustomResourceManifest 125 | }) 126 | 127 | expect(loggerMock.info.calledWith(logInfoMsg)) 128 | expect(kubeClientMock.addCustomResourceDefinition).to.have.been.calledWith(fakeCustomResourceManifest) 129 | expect(customResourceManager._createResource).to.have.been.calledWith({ 130 | customResourceManifest: fakeCustomResourceManifest 131 | }) 132 | expect(customResourceManager._getResource).to.have.callCount(0) 133 | expect(customResourceManager._updateResource).to.have.callCount(0) 134 | }) 135 | 136 | it('updates custom resource', async () => { 137 | const conflictError = new Error('fake conflict error') 138 | conflictError.statusCode = 409 139 | 140 | customResourceManager._createResource.throws(conflictError) 141 | customResourceManager._getResource.resolves(fakeCustomResource) 142 | customResourceManager._updateResource.resolves() 143 | sinon.stub(customResourceManager, '_sleep').resolves() 144 | 145 | const fakeResourceName = fakeCustomResourceManifest.metadata.name 146 | 147 | await customResourceManager.upsertResource({ 148 | customResourceManifest: fakeCustomResourceManifest 149 | }) 150 | 151 | expect(customResourceManager._getResource).to.have.been.calledWith({ 152 | resourceName: fakeResourceName 153 | }) 154 | expect(customResourceManager._updateResource).to.have.been.calledWith({ 155 | customResource: fakeCustomResource, 156 | customResourceManifest: fakeCustomResourceManifest 157 | }) 158 | expect(customResourceManager._sleep).to.have.callCount(0) 159 | }) 160 | }) 161 | }) 162 | -------------------------------------------------------------------------------- /lib/watchers/gated-deployment-watcher.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | const chai = require('chai') 3 | const sinon = require('sinon') 4 | const sinonChai = require('sinon-chai') 5 | 6 | const DeploymentWatcher = require('./deployment-watcher') 7 | const GatedDeploymentWatcher = require('./gated-deployment-watcher') 8 | const pluginFactory = require('../plugins/plugin-factory') 9 | 10 | chai.use(sinonChai) 11 | 12 | const { expect } = chai 13 | 14 | describe('GatedDeploymentWatcher', () => { 15 | describe('constructor', () => { 16 | it('sets fields on instance', () => { 17 | const watcher = new GatedDeploymentWatcher({ 18 | kubeClient: 'mockkubeclient', 19 | logger: 'mocklogger', 20 | pollerIntervalMilliseconds: 5000 21 | }) 22 | 23 | expect(watcher).to.be.an('object') 24 | expect(watcher._kubeClient).to.equal('mockkubeclient') 25 | expect(watcher._logger).to.equal('mocklogger') 26 | expect(watcher._pollerIntervalMilliseconds).to.equal(5000) 27 | expect(watcher._deploymentWatchers).to.deep.equal({}) 28 | }) 29 | }) 30 | 31 | describe('getStream', () => { 32 | it('returns gated deployment watch stream', () => { 33 | const watcher = new GatedDeploymentWatcher({ 34 | kubeClient: { 35 | apis: { 36 | 'kubernetes-client.io': { 37 | v1: { watch: { gateddeployments: { getStream: sinon.stub().returns('stream') } } } 38 | } 39 | } 40 | }, 41 | logger: 'mocklogger', 42 | pollerIntervalMilliseconds: 5000 43 | }) 44 | 45 | expect(watcher.getStream()).to.equal('stream') 46 | }) 47 | }) 48 | 49 | describe('_createDeploymentWatcher', () => { 50 | let watcher, loggerInfoStub, loggerErrorStub 51 | const gatedDeployment = { 52 | metadata: { 53 | name: 'testgd', 54 | namespace: 'testgdns' 55 | }, 56 | deploymentDescriptor: { 57 | control: { 58 | name: 'control' 59 | }, 60 | treatment: { 61 | name: 'treatment' 62 | }, 63 | decisionPlugins: 'mockPluginConfig' 64 | } 65 | } 66 | const expectedId = 'testgdns_testgd' 67 | 68 | beforeEach(() => { 69 | loggerInfoStub = sinon.stub() 70 | loggerErrorStub = sinon.stub() 71 | watcher = new GatedDeploymentWatcher({ 72 | kubeClient: 'kubeclient', 73 | logger: { 74 | info: loggerInfoStub, 75 | error: loggerErrorStub 76 | }, 77 | controllerNamespace: 'controllerNs' 78 | }) 79 | }) 80 | 81 | afterEach(() => { 82 | sinon.restore() 83 | }) 84 | 85 | it('creates, sets and starts deployment watcher', async () => { 86 | const pluginFactoryStub = sinon.stub(pluginFactory, 'buildPluginsFromConfig').resolves(['mockPlugin']) 87 | const startStub = sinon.stub(DeploymentWatcher.prototype, 'start') 88 | 89 | await watcher._createDeploymentWatcher(gatedDeployment) 90 | 91 | expect(watcher._deploymentWatchers).to.have.property(expectedId) 92 | expect(watcher._deploymentWatchers[expectedId]).to.be.an.instanceOf(DeploymentWatcher) 93 | expect(pluginFactoryStub).to.have.been.calledWith({ 94 | namespace: 'controllerNs', 95 | kubeClient: 'kubeclient', 96 | logger: watcher._logger, 97 | controlName: gatedDeployment.deploymentDescriptor.control.name, 98 | treatmentName: gatedDeployment.deploymentDescriptor.treatment.name, 99 | decisionPluginConfig: 'mockPluginConfig' 100 | }) 101 | expect(startStub).to.have.callCount(1) 102 | }) 103 | 104 | it('logs error if creating deployment watcher fails', async () => { 105 | const pluginFactoryStub = sinon.stub(pluginFactory, 'buildPluginsFromConfig').rejects() 106 | const startStub = sinon.stub(DeploymentWatcher.prototype, 'start') 107 | 108 | await watcher._createDeploymentWatcher(gatedDeployment) 109 | 110 | expect(watcher._deploymentWatchers).to.deep.equal({}) 111 | expect(pluginFactoryStub).to.have.been.calledWith({ 112 | namespace: 'controllerNs', 113 | kubeClient: 'kubeclient', 114 | logger: watcher._logger, 115 | controlName: gatedDeployment.deploymentDescriptor.control.name, 116 | treatmentName: gatedDeployment.deploymentDescriptor.treatment.name, 117 | decisionPluginConfig: 'mockPluginConfig' 118 | }) 119 | expect(loggerErrorStub).to.have.callCount(1) 120 | expect(startStub).to.have.callCount(0) 121 | }) 122 | }) 123 | 124 | describe('_removeDeploymentWatcher', () => { 125 | let watcher, loggerInfoStub, loggerWarnStub 126 | const gatedDeployment = { 127 | metadata: { 128 | name: 'testgd', 129 | namespace: 'testgdns' 130 | }, 131 | deploymentDescriptor: { 132 | newRelic: {} 133 | } 134 | } 135 | const expectedId = 'testgdns_testgd' 136 | 137 | beforeEach(() => { 138 | loggerInfoStub = sinon.stub() 139 | loggerWarnStub = sinon.stub() 140 | watcher = new GatedDeploymentWatcher({ 141 | kubeClient: 'kubeclient', 142 | logger: { 143 | info: loggerInfoStub, 144 | warn: loggerWarnStub 145 | } 146 | }) 147 | }) 148 | 149 | it('stops and removes deployment watcher', () => { 150 | const stopStub = sinon.stub() 151 | watcher._deploymentWatchers = { 152 | [expectedId]: { stop: stopStub }, 153 | anotherId: 'another watcher' 154 | } 155 | 156 | watcher._removeDeploymentWatcher(gatedDeployment) 157 | 158 | expect(watcher._deploymentWatchers).to.deep.equal({ anotherId: 'another watcher' }) 159 | expect(stopStub).to.have.callCount(1) 160 | }) 161 | 162 | it('warns if deployment watcher with id not found', () => { 163 | const stopStub = sinon.stub() 164 | watcher._deploymentWatchers = { 165 | anotherId: { stop: stopStub } 166 | } 167 | 168 | watcher._removeDeploymentWatcher(gatedDeployment) 169 | 170 | expect(watcher._deploymentWatchers).to.deep.equal({ anotherId: { stop: stopStub } }) 171 | expect(stopStub).to.have.callCount(0) 172 | expect(loggerWarnStub).to.have.callCount(1) 173 | }) 174 | }) 175 | 176 | describe('onAdded', () => { 177 | it('creates deployment watcher', () => { 178 | const watcher = new GatedDeploymentWatcher({}) 179 | const gatedDeployment = { gated: 'deploy' } 180 | watcher._createDeploymentWatcher = sinon.stub().resolves() 181 | watcher._removeDeploymentWatcher = sinon.stub() 182 | 183 | expect(watcher.onAdded(gatedDeployment)).to.be.an.instanceOf(Promise) 184 | expect(watcher._createDeploymentWatcher).to.have.been.calledWith(gatedDeployment) 185 | expect(watcher._removeDeploymentWatcher).to.have.callCount(0) 186 | }) 187 | }) 188 | 189 | describe('onModified', () => { 190 | it('removes and creates deployment watcher', () => { 191 | const watcher = new GatedDeploymentWatcher({}) 192 | const gatedDeployment = { gated: 'deploy' } 193 | watcher._createDeploymentWatcher = sinon.stub().resolves() 194 | watcher._removeDeploymentWatcher = sinon.stub() 195 | 196 | expect(watcher.onModified(gatedDeployment)).to.be.an.instanceOf(Promise) 197 | expect(watcher._createDeploymentWatcher).to.have.been.calledWith(gatedDeployment) 198 | expect(watcher._removeDeploymentWatcher).to.have.been.calledWith(gatedDeployment) 199 | }) 200 | }) 201 | 202 | describe('onDeleted', () => { 203 | it('removes deployment watcher', () => { 204 | const watcher = new GatedDeploymentWatcher({}) 205 | const gatedDeployment = { gated: 'deploy' } 206 | watcher._createDeploymentWatcher = sinon.stub() 207 | watcher._removeDeploymentWatcher = sinon.stub() 208 | 209 | watcher.onDeleted(gatedDeployment) 210 | 211 | expect(watcher._createDeploymentWatcher).to.have.callCount(0) 212 | expect(watcher._removeDeploymentWatcher).to.have.been.calledWith(gatedDeployment) 213 | }) 214 | }) 215 | }) 216 | -------------------------------------------------------------------------------- /lib/watchers/watcher.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | const AsyncLock = require('async-lock') 3 | const chai = require('chai') 4 | const sinon = require('sinon') 5 | const sinonChai = require('sinon-chai') 6 | const stream = require('stream') 7 | 8 | const { StreamEvents, WatchEvents, WatcherStates } = require('./constants') 9 | const Watcher = require('./watcher') 10 | 11 | chai.use(sinonChai) 12 | 13 | const { expect } = chai 14 | 15 | describe('Watcher', () => { 16 | describe('constructor', () => { 17 | it('sets activeResources and _state', () => { 18 | const watcher = new Watcher() 19 | 20 | expect(watcher._activeResources).to.deep.equal({}) 21 | expect(watcher._lock).to.be.an.instanceOf(AsyncLock) 22 | expect(watcher._state).to.equal(WatcherStates.READY) 23 | }) 24 | }) 25 | 26 | describe('._extractResourceMetadata', () => { 27 | it('returns metadata', () => { 28 | const resource = { 29 | metadata: { 30 | name: 'test-resource', 31 | namespace: 'test-ns', 32 | resourceVersion: 12345 33 | } 34 | } 35 | 36 | const watcher = new Watcher() 37 | const metadata = watcher._extractResourceMetadata(resource) 38 | 39 | expect(metadata).to.deep.equal({ 40 | id: 'test-ns_test-resource', 41 | namespace: 'test-ns', 42 | resourceVersion: 12345 43 | }) 44 | }) 45 | }) 46 | 47 | describe('.onEnd', () => { 48 | let watcher 49 | 50 | beforeEach(() => { 51 | watcher = new Watcher() 52 | 53 | watcher._onDeleted = sinon.stub() 54 | watcher.start = sinon.stub() 55 | }) 56 | 57 | it('restarts if no active resources exist', () => { 58 | watcher.onEnd() 59 | 60 | expect(watcher.start).to.have.callCount(1) 61 | }) 62 | 63 | it('sets endEvent for all active resources', () => { 64 | watcher._activeResources = { 65 | r1: { resource: 'res1' }, 66 | r2: { resource: 'res2' } 67 | } 68 | 69 | watcher.onEnd() 70 | 71 | expect(watcher._activeResources.r1.endEvent).to.equal(true) 72 | expect(watcher._activeResources.r2.endEvent).to.equal(true) 73 | expect(watcher.start).to.have.callCount(1) 74 | }) 75 | 76 | it('calls _onDeleted for resources with endEvent already set', () => { 77 | watcher._activeResources = { 78 | r1: { resource: 'res1', endEvent: true }, 79 | r2: { resource: 'res2' }, 80 | r3: { resource: 'res3', endEvent: true } 81 | } 82 | 83 | watcher.onEnd() 84 | 85 | expect(watcher._activeResources.r2.endEvent).to.equal(true) 86 | expect(watcher._onDeleted).to.have.callCount(2) 87 | expect(watcher._onDeleted).to.have.been.calledWith('res1') 88 | expect(watcher._onDeleted).to.have.been.calledWith('res3') 89 | expect(watcher.start).to.have.callCount(1) 90 | }) 91 | 92 | it('does not restart if watcher has already ended after calling _onDeleted', () => { 93 | watcher._state = WatcherStates.ENDED 94 | watcher._activeResources = { 95 | r1: { resource: 'res1', endEvent: true }, 96 | r2: { resource: 'res2' }, 97 | r3: { resource: 'res3', endEvent: true } 98 | } 99 | 100 | watcher.onEnd() 101 | 102 | expect(watcher._activeResources.r2.endEvent).to.equal(true) 103 | expect(watcher._onDeleted).to.have.callCount(2) 104 | expect(watcher._onDeleted).to.have.been.calledWith('res1') 105 | expect(watcher._onDeleted).to.have.been.calledWith('res3') 106 | expect(watcher.start).to.have.callCount(0) 107 | }) 108 | }) 109 | 110 | describe('._onAdded', () => { 111 | let watcher 112 | 113 | beforeEach(() => { 114 | watcher = new Watcher() 115 | 116 | watcher.onAdded = sinon.stub() 117 | watcher._onModified = sinon.stub() 118 | }) 119 | 120 | it('calls onAdded for a new resource', () => { 121 | const resource = { 122 | metadata: { 123 | name: 'test-resource', 124 | namespace: 'test-ns', 125 | resourceVersion: 12345 126 | } 127 | } 128 | 129 | watcher._onAdded(resource) 130 | 131 | expect(watcher._activeResources).to.deep.equal({ 132 | 'test-ns_test-resource': { 133 | resourceVersion: 12345, 134 | resource 135 | } 136 | }) 137 | expect(watcher.onAdded).to.have.been.calledWith(resource) 138 | expect(watcher._onModified).to.have.callCount(0) 139 | }) 140 | 141 | it('clears endEvent flag for an already active resource', () => { 142 | const resource = { 143 | metadata: { 144 | name: 'test-resource', 145 | namespace: 'test-ns', 146 | resourceVersion: 12345 147 | } 148 | } 149 | watcher._activeResources = { 150 | 'test-ns_test-resource': { 151 | resourceVersion: 12345, 152 | resource, 153 | endEvent: true 154 | } 155 | } 156 | 157 | watcher._onAdded(resource) 158 | 159 | expect(watcher._activeResources).to.deep.equal({ 160 | 'test-ns_test-resource': { 161 | resourceVersion: 12345, 162 | resource 163 | } 164 | }) 165 | expect(watcher.onAdded).to.have.callCount(0) 166 | expect(watcher._onModified).to.have.callCount(0) 167 | }) 168 | 169 | it('calls onModified for an already existing resource that is modified', () => { 170 | const resource = { 171 | metadata: { 172 | name: 'test-resource', 173 | namespace: 'test-ns', 174 | resourceVersion: 12345 175 | } 176 | } 177 | watcher._activeResources = { 178 | 'test-ns_test-resource': { 179 | resourceVersion: 12343, 180 | resource, 181 | endEvent: true 182 | } 183 | } 184 | 185 | watcher._onAdded(resource) 186 | 187 | expect(watcher._activeResources).to.deep.equal({ 188 | 'test-ns_test-resource': { 189 | resourceVersion: 12343, 190 | resource 191 | } 192 | }) 193 | expect(watcher.onAdded).to.have.callCount(0) 194 | expect(watcher._onModified).to.have.been.calledWith(resource) 195 | }) 196 | }) 197 | 198 | describe('._onModified', () => { 199 | let watcher 200 | 201 | beforeEach(() => { 202 | watcher = new Watcher() 203 | 204 | watcher.onModified = sinon.stub() 205 | }) 206 | 207 | it('updates activeResources and calls onModified', () => { 208 | const resource = { 209 | metadata: { 210 | name: 'test-resource', 211 | namespace: 'test-ns', 212 | resourceVersion: 12345 213 | } 214 | } 215 | watcher._activeResources = { 216 | 'test-ns_test-resource': { 217 | resourceVersion: 12343, 218 | resource, 219 | endEvent: true 220 | } 221 | } 222 | 223 | watcher._onModified(resource) 224 | 225 | expect(watcher._activeResources).to.deep.equal({ 226 | 'test-ns_test-resource': { 227 | resourceVersion: 12345, 228 | resource 229 | } 230 | }) 231 | expect(watcher.onModified).to.have.been.calledWith(resource) 232 | }) 233 | }) 234 | 235 | describe('._onDeleted', () => { 236 | let watcher 237 | 238 | beforeEach(() => { 239 | watcher = new Watcher() 240 | 241 | watcher.onDeleted = sinon.stub() 242 | }) 243 | 244 | it('updates activeResources and calls onDeleted', () => { 245 | const resource = { 246 | metadata: { 247 | name: 'test-resource', 248 | namespace: 'test-ns', 249 | resourceVersion: 12345 250 | } 251 | } 252 | watcher._activeResources = { 253 | 'test-ns_test-resource': { 254 | resourceVersion: 12343, 255 | resource, 256 | endEvent: true 257 | } 258 | } 259 | 260 | watcher._onDeleted(resource) 261 | 262 | expect(watcher._activeResources).to.deep.equal({}) 263 | expect(watcher.onDeleted).to.have.been.calledWith(resource) 264 | }) 265 | }) 266 | 267 | describe('.start', () => { 268 | let watcher, mockStream 269 | 270 | beforeEach(() => { 271 | watcher = new Watcher() 272 | mockStream = new stream.Readable() 273 | mockStream._read = () => {} 274 | 275 | watcher.getStream = sinon.stub().returns(mockStream) 276 | watcher._onAdded = sinon.stub() 277 | watcher._onModified = sinon.stub() 278 | watcher._onDeleted = sinon.stub() 279 | watcher.onEnd = sinon.stub() 280 | 281 | watcher.start() 282 | expect(watcher._state).to.equal(WatcherStates.RUNNING) 283 | }) 284 | 285 | it('calls onAdded on ADDED event', () => { 286 | mockStream.emit(StreamEvents.DATA, `{"type": "${WatchEvents.ADDED}", "object": {"some": "object1"}}`) 287 | 288 | expect(watcher._onAdded).to.have.been.calledWith({ some: 'object1' }) 289 | expect(watcher._onModified).to.have.callCount(0) 290 | expect(watcher._onDeleted).to.have.callCount(0) 291 | }) 292 | 293 | it('calls onModified on MODIFIED event', () => { 294 | mockStream.emit(StreamEvents.DATA, `{"type": "${WatchEvents.MODIFIED}", "object": {"some": "object2"}}`) 295 | 296 | expect(watcher._onModified).to.have.been.calledWith({ some: 'object2' }) 297 | expect(watcher._onAdded).to.have.callCount(0) 298 | expect(watcher._onDeleted).to.have.callCount(0) 299 | }) 300 | 301 | it('calls onDeleted on DELETED event', () => { 302 | mockStream.emit(StreamEvents.DATA, `{"type": "${WatchEvents.DELETED}", "object": {"some": "object3"}}`) 303 | 304 | expect(watcher._onDeleted).to.have.been.calledWith({ some: 'object3' }) 305 | expect(watcher._onAdded).to.have.callCount(0) 306 | expect(watcher._onModified).to.have.callCount(0) 307 | }) 308 | 309 | it('calls onEnd on end event', () => { 310 | mockStream.emit(StreamEvents.END) 311 | 312 | expect(watcher.onEnd).to.have.callCount(1) 313 | }) 314 | }) 315 | 316 | describe('.stop', () => { 317 | it('aborts stream and calls _onDeleted for all active resources', () => { 318 | const watcher = new Watcher() 319 | watcher._activeResources = { 320 | r1: { resource: 'res1' }, 321 | r2: { resource: 'res2', endEvent: true } 322 | } 323 | watcher.onDeleted = sinon.stub() 324 | watcher._onDeleted = sinon.stub() 325 | 326 | watcher.stream = { abort: sinon.stub() } 327 | 328 | watcher.stop() 329 | 330 | expect(watcher._state).to.equal(WatcherStates.ENDED) 331 | expect(watcher.stream.abort).to.have.callCount(1) 332 | expect(watcher._onDeleted).to.have.callCount(2) 333 | expect(watcher._onDeleted).to.have.been.calledWith('res1') 334 | expect(watcher._onDeleted).to.have.been.calledWith('res2') 335 | }) 336 | }) 337 | }) 338 | -------------------------------------------------------------------------------- /lib/watchers/deployment-watcher.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert') 2 | 3 | const moment = require('moment') 4 | 5 | const analysis = require('../analysis') 6 | const { DEPLOYMENT_ANNOTATION_NAME, EXPERIMENT_ANNOTATION_NAME } = require('./constants') 7 | const { DecisionResults } = require('../plugins/plugin') 8 | const DeploymentHelper = require('../deployment-helper') 9 | const Watcher = require('./watcher') 10 | 11 | class DeploymentWatcher extends Watcher { 12 | /** 13 | * Creates a deployment watcher 14 | * 15 | * @param {Object} options the options object 16 | * @param {Object} options.kubeClient the kube client instance 17 | * @param {Object} options.logger the logger instance 18 | * @param {Object} options.deploymentDescriptor the gated deployment resource 19 | * @param {string} options.namespace the namespace of the deployment 20 | * @param {string} options.plugins the decision plugins 21 | * @param {number} options.pollerIntervalMilliseconds the poller interval in milliseconds 22 | * @param {string} options.gatedDeploymentId the gated deployment id 23 | */ 24 | constructor ({ 25 | kubeClient, 26 | logger, 27 | deploymentDescriptor, 28 | namespace, 29 | plugins, 30 | pollerIntervalMilliseconds, 31 | gatedDeploymentId 32 | }) { 33 | super() 34 | this._kubeClient = kubeClient 35 | this._logger = logger 36 | this._deploymentDescriptor = deploymentDescriptor 37 | this._namespace = namespace 38 | this._plugins = plugins 39 | this._pollerIntervalMilliseconds = pollerIntervalMilliseconds 40 | this._gatedDeploymentId = gatedDeploymentId 41 | 42 | this._deploymentHelper = new DeploymentHelper({ kubeClient, namespace }) 43 | 44 | this._experiment = { 45 | startTime: null, 46 | pollerInterval: null, 47 | treatmentPodSpec: null 48 | } 49 | } 50 | 51 | /** 52 | * Returns a watch stream for the treatment deployment 53 | * 54 | * @returns {Object} the watch stream 55 | */ 56 | getStream () { 57 | return this._kubeClient.apis.apps.v1.watch.namespaces(this._namespace).deploy(this._deploymentDescriptor.treatment.name).getStream() 58 | } 59 | 60 | /** 61 | * Validates whether the existing annotation is valid and returns an object 62 | * containing the experiment start time and pod spec hash. 63 | * 64 | * @param {Object} treatmentDeployment the treatment deployment manifest 65 | * @returns {Object} the experiment annotation. An empty object if it doesn't 66 | * exist or is invalid 67 | */ 68 | _validateAndGetExperimentAnnotation (treatmentDeployment) { 69 | try { 70 | const annotationValue = JSON.parse(treatmentDeployment.metadata.annotations[EXPERIMENT_ANNOTATION_NAME]) 71 | assert(annotationValue && annotationValue.startTime && annotationValue.podSpecHash) 72 | annotationValue.startTime = moment.utc(annotationValue.startTime, moment.ISO_8601, true) 73 | assert(annotationValue.startTime.isValid()) 74 | return annotationValue 75 | } catch (err) { 76 | return {} 77 | } 78 | } 79 | 80 | /** 81 | * Compares control and treatment deployments and returns true if the control 82 | * and treatment pod specs differ 83 | * 84 | * @param {Object} treatmentDeployment the treatment deployment manifest 85 | * @returns {boolean} whether experiment can be started or not 86 | */ 87 | async _isEligibleForExperiment (treatmentDeployment) { 88 | const controlDeployment = (await this._deploymentHelper.get(this._deploymentDescriptor.control.name)).body 89 | 90 | return !this._deploymentHelper.isPodSpecIdentical(controlDeployment, treatmentDeployment) 91 | } 92 | 93 | /** 94 | * Checks if the treatment deployment is eligible for experiment and starts an 95 | * experiment by creating the poller 96 | * 97 | * @param {Object} treatmentDeployment the treatment deployment manifest 98 | */ 99 | async _startExperiment (treatmentDeployment) { 100 | try { 101 | const existingExperimentAnnotation = this._validateAndGetExperimentAnnotation(treatmentDeployment) 102 | const podSpecHash = this._deploymentHelper.getPodSpecHash(treatmentDeployment) 103 | let newExperimentAnnotation = null 104 | if (treatmentDeployment.spec.replicas > 0) { 105 | if (await this._isEligibleForExperiment(treatmentDeployment)) { 106 | this._logger.info(`${this._gatedDeploymentId}: Starting experiment`) 107 | // If the annotation includes an experiment and the pod spec matches, 108 | // use start time from the annotation 109 | const startTime = existingExperimentAnnotation.podSpecHash === podSpecHash ? existingExperimentAnnotation.startTime : moment.utc() 110 | newExperimentAnnotation = JSON.stringify({ startTime, podSpecHash }) 111 | this._experiment.startTime = startTime 112 | this._experiment.pollerInterval = setInterval(this._poll.bind(this), this._pollerIntervalMilliseconds) 113 | this._experiment.treatmentPodSpec = treatmentDeployment.spec.template.spec 114 | for (const plugin of this._plugins) { 115 | plugin.onExperimentStart(this._experiment.startTime) 116 | } 117 | } else { 118 | // If the treatment is the same as control, kill treatment and set no 119 | // harm annotation as we don't need an experiment to compare identical 120 | // deployments. 121 | this._logger.info(`${this._gatedDeploymentId}: Found non zero treatment replicas with same image as control. Killing treatment and setting no harm`) 122 | await this._killTreatment(analysis.results.noHarm) 123 | } 124 | } 125 | await this._deploymentHelper.setAnnotation(this._deploymentDescriptor.treatment.name, EXPERIMENT_ANNOTATION_NAME, newExperimentAnnotation) 126 | } catch (err) { 127 | this._logger.error(err, `${this._gatedDeploymentId}: Error occurred when starting experiment`) 128 | } 129 | } 130 | 131 | /** 132 | * Clears the experiment by stopping the poller 133 | */ 134 | async _clearExperiment () { 135 | if (this._experiment.pollerInterval) { 136 | this._logger.info(`${this._gatedDeploymentId}: Stopping experiment`) 137 | clearInterval(this._experiment.pollerInterval) 138 | this._experiment.startTime = null 139 | this._experiment.pollerInterval = null 140 | this._experiment.treatmentPodSpec = null 141 | for (const plugin of this._plugins) { 142 | plugin.onExperimentStop() 143 | } 144 | try { 145 | await this._deploymentHelper.setAnnotation(this._deploymentDescriptor.treatment.name, EXPERIMENT_ANNOTATION_NAME, null) 146 | } catch (err) { 147 | this._logger.error(err, `${this._gatedDeploymentId}: Error occurred when clearing experiment`) 148 | } 149 | } 150 | } 151 | 152 | /** 153 | * Kills treatment and sets the deployment annotation 154 | * 155 | * @param {string} annotationValue the annotation value 156 | */ 157 | async _killTreatment (annotationValue) { 158 | await this._deploymentHelper.kill(this._deploymentDescriptor.treatment.name) 159 | await this._deploymentHelper.setAnnotation(this._deploymentDescriptor.treatment.name, DEPLOYMENT_ANNOTATION_NAME, annotationValue) 160 | } 161 | 162 | /** 163 | * Updates control spec to treatment, sets number of replicas for treatment 164 | * to zero, sets treatment annotation to harm and stops the experiment 165 | */ 166 | async _passExperiment () { 167 | const { treatmentPodSpec } = this._experiment 168 | 169 | // NOTE: experiment must be cleared first, then deployment should be killed 170 | // before setting the annotation because they trigger a modify event, which 171 | // would try to clear any existing experiment and start a new experiment 172 | await this._clearExperiment() 173 | 174 | this._logger.info(`${this._gatedDeploymentId}: Updating control with treatment and killing treatment`) 175 | await this._deploymentHelper.updatePodSpec(this._deploymentDescriptor.control.name, treatmentPodSpec) 176 | await this._killTreatment(analysis.results.noHarm) 177 | } 178 | 179 | /** 180 | * Sets number of replicas for treatment to zero, sets treatment annotation to 181 | * harm and stops the experiment 182 | */ 183 | async _failExperiment () { 184 | await this._clearExperiment() 185 | 186 | this._logger.info(`${this._gatedDeploymentId}: Killing treatment`) 187 | await this._killTreatment(analysis.results.harm) 188 | } 189 | /** 190 | * Polls all decision plugins, and aggregates those results to decide whether to fail, pass 191 | * or let the experiment keep running 192 | */ 193 | async _poll () { 194 | const results = await Promise.all(this._plugins.map(async plugin => { 195 | try { 196 | return await plugin.onExperimentPoll(this._deploymentDescriptor.control.name, this._deploymentDescriptor.treatment.name) 197 | } catch (err) { 198 | this._logger.error(`${this._gatedDeploymentId}: Error occurred when polling plugin`, err) 199 | return DecisionResults.WAIT 200 | } 201 | })) 202 | 203 | // If any plugins return FAIL, then fail the experiment 204 | if (results.some(result => result === DecisionResults.FAIL)) { 205 | this._logger.info(`${this._gatedDeploymentId}: Experiment failed`) 206 | await this._failExperiment() 207 | } else if (results.every(result => result === DecisionResults.PASS)) { 208 | // If all plugins return PASS, then pass the experiment 209 | this._logger.info(`${this._gatedDeploymentId}: Experiment success`) 210 | await this._passExperiment() 211 | } else { 212 | this._logger.info(`${this._gatedDeploymentId}: Experiment not yet significant`) 213 | } 214 | } 215 | 216 | /** 217 | * Starts an experiment for the new treatment deployment, if eligible 218 | * 219 | * @param {Object} deployment the deployment manifest 220 | * @returns {Promise} a promise that resolves when the experiment is started 221 | * if eligible. 222 | */ 223 | onAdded (deployment) { 224 | this._previous = deployment 225 | return this._startExperiment(deployment) 226 | } 227 | 228 | /** 229 | * Clears any existing experiment and starts a new experiment for the 230 | * modified treatment deployment, if eligible 231 | * 232 | * @param {Object} deployment the deployment manifest 233 | */ 234 | async onModified (deployment) { 235 | // NOTE: Modify events are triggered multiple times when pods associated 236 | // with the deploy are affected. Only clear and start an experiment if one 237 | // does not already exist or if the replicas or spec changes from the 238 | // treatment deployment used for the experiment 239 | if (!this._previous || 240 | deployment.spec.replicas !== this._previous.spec.replicas || 241 | !this._deploymentHelper.isPodSpecIdentical(deployment, this._previous) 242 | ) { 243 | await this._clearExperiment() 244 | await this._startExperiment(deployment) 245 | } 246 | this._previous = deployment 247 | } 248 | 249 | /** 250 | * Clears any existing experiment 251 | * 252 | * @returns {Promise} a promise that resolves when the experiment is started 253 | * if eligible. 254 | */ 255 | onDeleted () { 256 | this._previous = null 257 | return this._clearExperiment() 258 | } 259 | } 260 | 261 | module.exports = DeploymentWatcher 262 | -------------------------------------------------------------------------------- /lib/watchers/watcher.integration.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | const chai = require('chai') 3 | const clone = require('clone') 4 | const sinon = require('sinon') 5 | const sinonChai = require('sinon-chai') 6 | const stream = require('stream') 7 | 8 | const { EVENT_LOCK, StreamEvents, WatchEvents } = require('./constants') 9 | const Watcher = require('./watcher') 10 | 11 | chai.use(sinonChai) 12 | 13 | const { expect } = chai 14 | 15 | describe('Watcher', () => { 16 | describe('integration tests', () => { 17 | let watcher, mockStream1, mockStream2, mockStream3, objects 18 | 19 | beforeEach(() => { 20 | watcher = new Watcher() 21 | mockStream1 = new stream.Readable() 22 | mockStream1._read = () => {} 23 | mockStream1.abort = sinon.stub().callsFake(() => { mockStream1.emit(StreamEvents.END) }) 24 | mockStream2 = new stream.Readable() 25 | mockStream2._read = () => {} 26 | mockStream2.abort = sinon.stub().callsFake(() => { mockStream2.emit(StreamEvents.END) }) 27 | mockStream3 = new stream.Readable() 28 | mockStream3._read = () => {} 29 | mockStream3.abort = sinon.stub().callsFake(() => { mockStream3.emit(StreamEvents.END) }) 30 | watcher.getStream = sinon.stub().onCall(0).returns(mockStream1) 31 | .onCall(1).returns(mockStream2) 32 | .onCall(2).returns(mockStream3) 33 | 34 | sinon.spy(watcher, '_onAdded') 35 | sinon.spy(watcher, '_onModified') 36 | sinon.spy(watcher, '_onDeleted') 37 | sinon.spy(watcher, 'onEnd') 38 | sinon.spy(watcher, 'start') 39 | watcher.onAdded = sinon.stub() 40 | watcher.onModified = sinon.stub() 41 | watcher.onDeleted = sinon.stub() 42 | watcher.start() 43 | 44 | objects = [{ 45 | metadata: { 46 | name: 'object1', 47 | namespace: 'ns1', 48 | resourceVersion: 1 49 | } 50 | }, { 51 | metadata: { 52 | name: 'object2', 53 | namespace: 'ns2', 54 | resourceVersion: 1 55 | } 56 | }] 57 | }) 58 | 59 | it('adds new resources and marks them on stream end', async () => { 60 | const expectedResources = { 61 | 'ns1_object1': { 62 | resourceVersion: 1, 63 | resource: objects[0], 64 | endEvent: true 65 | }, 66 | 'ns2_object2': { 67 | resourceVersion: 1, 68 | resource: objects[1], 69 | endEvent: true 70 | } 71 | } 72 | 73 | mockStream1.emit(StreamEvents.DATA, `{"type": "${WatchEvents.ADDED}", "object": ${JSON.stringify(objects[0])}}\n`) 74 | mockStream1.emit(StreamEvents.DATA, `{"type": "${WatchEvents.ADDED}", "object": ${JSON.stringify(objects[1])}}\n`) 75 | mockStream1.emit(StreamEvents.END) 76 | 77 | await watcher._lock.acquire(EVENT_LOCK, () => {}) 78 | 79 | expect(watcher._onAdded).to.have.been.calledWith(objects[0]) 80 | expect(watcher._onAdded).to.have.been.calledWith(objects[1]) 81 | expect(watcher.onAdded).to.have.been.calledWith(objects[0]) 82 | expect(watcher.onAdded).to.have.been.calledWith(objects[1]) 83 | expect(watcher._onModified).to.have.callCount(0) 84 | expect(watcher._onDeleted).to.have.callCount(0) 85 | expect(watcher._activeResources).to.deep.equal(expectedResources) 86 | }) 87 | 88 | it('updates existing resource for modified event', async () => { 89 | const modifiedObject = clone(objects[0]) 90 | modifiedObject.metadata.resourceVersion = 2 91 | const expectedResources = { 92 | 'ns1_object1': { 93 | resourceVersion: 2, 94 | resource: modifiedObject 95 | } 96 | } 97 | 98 | mockStream1.emit(StreamEvents.DATA, `{"type": "${WatchEvents.ADDED}", "object": ${JSON.stringify(objects[0])}}\n`) 99 | mockStream1.emit(StreamEvents.DATA, `{"type": "${WatchEvents.MODIFIED}", "object": ${JSON.stringify(modifiedObject)}}\n`) 100 | 101 | await watcher._lock.acquire(EVENT_LOCK, () => {}) 102 | 103 | expect(watcher._onAdded).to.have.been.calledWith(objects[0]) 104 | expect(watcher.onAdded).to.have.been.calledWith(objects[0]) 105 | expect(watcher._onModified).to.have.been.calledWith(modifiedObject) 106 | expect(watcher.onModified).to.have.been.calledWith(modifiedObject) 107 | expect(watcher._activeResources).to.deep.equal(expectedResources) 108 | }) 109 | 110 | it('removes existing resource for deleted event', async () => { 111 | mockStream1.emit(StreamEvents.DATA, `{"type": "${WatchEvents.ADDED}", "object": ${JSON.stringify(objects[0])}}\n`) 112 | mockStream1.emit(StreamEvents.DATA, `{"type": "${WatchEvents.DELETED}", "object": ${JSON.stringify(objects[0])}}\n`) 113 | 114 | await watcher._lock.acquire(EVENT_LOCK, () => {}) 115 | 116 | expect(watcher._onAdded).to.have.been.calledWith(objects[0]) 117 | expect(watcher.onAdded).to.have.been.calledWith(objects[0]) 118 | expect(watcher._onDeleted).to.have.been.calledWith(objects[0]) 119 | expect(watcher.onDeleted).to.have.been.calledWith(objects[0]) 120 | expect(watcher._activeResources).to.deep.equal({}) 121 | }) 122 | 123 | it('handles duplicate added events when stream ends and a new stream starts', async () => { 124 | const expectedResources = { 125 | 'ns1_object1': { 126 | resourceVersion: 1, 127 | resource: objects[0] 128 | }, 129 | 'ns2_object2': { 130 | resourceVersion: 1, 131 | resource: objects[1] 132 | } 133 | } 134 | 135 | mockStream1.emit(StreamEvents.DATA, `{"type": "${WatchEvents.ADDED}", "object": ${JSON.stringify(objects[0])}}\n`) 136 | mockStream1.emit(StreamEvents.DATA, `{"type": "${WatchEvents.ADDED}", "object": ${JSON.stringify(objects[1])}}\n`) 137 | mockStream1.emit(StreamEvents.END) 138 | mockStream2.emit(StreamEvents.DATA, `{"type": "${WatchEvents.ADDED}", "object": ${JSON.stringify(objects[0])}}\n`) 139 | mockStream2.emit(StreamEvents.DATA, `{"type": "${WatchEvents.ADDED}", "object": ${JSON.stringify(objects[1])}}\n`) 140 | 141 | await watcher._lock.acquire(EVENT_LOCK, () => {}) 142 | 143 | expect(watcher._onAdded.getCall(0)).to.have.been.calledWith(objects[0]) 144 | expect(watcher._onAdded.getCall(1)).to.have.been.calledWith(objects[1]) 145 | expect(watcher._onAdded.getCall(2)).to.have.been.calledWith(objects[0]) 146 | expect(watcher._onAdded.getCall(3)).to.have.been.calledWith(objects[1]) 147 | expect(watcher.onAdded).to.have.callCount(2) 148 | expect(watcher._onModified).to.have.callCount(0) 149 | expect(watcher._activeResources).to.deep.equal(expectedResources) 150 | }) 151 | 152 | it('handles resources that are modified after stream ends and before new stream starts', async () => { 153 | const modifiedObject = clone(objects[0]) 154 | modifiedObject.metadata.resourceVersion = 2 155 | const expectedResources = { 156 | 'ns1_object1': { 157 | resourceVersion: 2, 158 | resource: modifiedObject 159 | } 160 | } 161 | 162 | mockStream1.emit(StreamEvents.DATA, `{"type": "${WatchEvents.ADDED}", "object": ${JSON.stringify(objects[0])}}\n`) 163 | mockStream1.emit(StreamEvents.END) 164 | mockStream2.emit(StreamEvents.DATA, `{"type": "${WatchEvents.ADDED}", "object": ${JSON.stringify(modifiedObject)}}\n`) 165 | 166 | await watcher._lock.acquire(EVENT_LOCK, () => {}) 167 | 168 | expect(watcher._onAdded.getCall(0)).to.have.been.calledWith(objects[0]) 169 | expect(watcher._onAdded.getCall(1)).to.have.been.calledWith(modifiedObject) 170 | expect(watcher.onAdded).to.have.callCount(1) 171 | expect(watcher.onAdded).to.have.been.calledWith(objects[0]) 172 | expect(watcher._onModified).to.have.been.calledWith(modifiedObject) 173 | expect(watcher._activeResources).to.deep.equal(expectedResources) 174 | }) 175 | 176 | it('handles resources that are deleted after stream ends and before new stream starts', async () => { 177 | mockStream1.emit(StreamEvents.DATA, `{"type": "${WatchEvents.ADDED}", "object": ${JSON.stringify(objects[0])}}\n`) 178 | mockStream1.emit(StreamEvents.END) 179 | mockStream2.emit(StreamEvents.END) 180 | 181 | await watcher._lock.acquire(EVENT_LOCK, () => {}) 182 | 183 | expect(watcher.onEnd).to.have.callCount(2) 184 | expect(watcher._onDeleted).to.have.been.calledWith(objects[0]) 185 | expect(watcher._activeResources).to.deep.equal({}) 186 | expect(watcher.start).to.have.callCount(3) 187 | expect(watcher.getStream).to.have.callCount(3) 188 | }) 189 | 190 | it('does not restart watcher if watcher is stopped', async () => { 191 | watcher.stop() 192 | 193 | await watcher._lock.acquire(EVENT_LOCK, () => {}) 194 | 195 | expect(mockStream1.abort).to.have.been.calledWith() 196 | expect(watcher.onEnd).to.have.callCount(1) 197 | expect(watcher.start).to.have.callCount(1) 198 | expect(watcher.getStream).to.have.callCount(1) 199 | }) 200 | 201 | it('handles resources that are deleted after stream ends and before new stream starts, when watcher is stopped', async () => { 202 | mockStream1.emit(StreamEvents.DATA, `{"type": "${WatchEvents.ADDED}", "object": ${JSON.stringify(objects[0])}}\n`) 203 | mockStream1.emit(StreamEvents.END) 204 | watcher.stop() 205 | 206 | await watcher._lock.acquire(EVENT_LOCK, () => {}) 207 | 208 | expect(mockStream1.abort).to.have.callCount(0) 209 | expect(mockStream2.abort).to.have.been.calledWith() 210 | expect(watcher.onEnd).to.have.callCount(2) 211 | expect(watcher._onDeleted).to.have.been.calledWith(objects[0]) 212 | expect(watcher._activeResources).to.deep.equal({}) 213 | expect(watcher.start).to.have.callCount(2) 214 | expect(watcher.getStream).to.have.callCount(2) 215 | }) 216 | 217 | it('stops stream and cleans up activeResources if stopped', async () => { 218 | mockStream1.emit(StreamEvents.DATA, `{"type": "${WatchEvents.ADDED}", "object": ${JSON.stringify(objects[0])}}\n`) 219 | watcher.stop() 220 | 221 | await watcher._lock.acquire(EVENT_LOCK, () => {}) 222 | 223 | expect(mockStream1.abort).to.have.been.calledWith() 224 | expect(watcher._onDeleted).to.have.been.calledWith(objects[0]) 225 | expect(watcher.onDeleted).to.have.been.calledWith(objects[0]) 226 | expect(watcher._activeResources).to.deep.equal({}) 227 | }) 228 | 229 | it('handles events synchronously', async () => { 230 | const state = { val: 1 } 231 | watcher.onAdded = sinon.stub().callsFake(async () => { 232 | const stateVal = state.val 233 | await new Promise(resolve => setTimeout(resolve, 50)) 234 | state.val = stateVal * 2 235 | }) 236 | watcher.onModified = sinon.stub().callsFake(async () => { 237 | const stateVal = state.val 238 | await new Promise(resolve => setTimeout(resolve, 50)) 239 | state.val = stateVal * 2 240 | }) 241 | 242 | mockStream1.emit(StreamEvents.DATA, `{"type": "${WatchEvents.ADDED}", "object": ${JSON.stringify(objects[0])}}\n`) 243 | mockStream1.emit(StreamEvents.DATA, `{"type": "${WatchEvents.MODIFIED}", "object": ${JSON.stringify(objects[0])}}\n`) 244 | watcher.stop() 245 | 246 | await watcher._lock.acquire(EVENT_LOCK, () => {}) 247 | 248 | expect(state.val).to.equal(4) 249 | }) 250 | }) 251 | }) 252 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kubernetes Gated Deployments 2 | 3 | Kubernetes Gated Deployments facilitates A/B tests on application deployments to 4 | perform automated regression testing and canary analysis. 5 | 6 | ![Architecture](architecture.svg) 7 | 8 | Kubernetes Gated Deployments extends the Kubernetes API to add a new type of 9 | object called `GatedDeployments`, that allows engineers to specify which 10 | deployments and decision plugins to include in the A/B test. It uses a 11 | controller in the Kubernetes control plane that is responsible for implementing 12 | this behavior by retrieving and analyzing metrics from the backend specified, 13 | decisioning the A/B test, and either rolling back or continuing the deployment. 14 | 15 | ## How it works 16 | 17 | ![Gated Deployment Path](gated-deployment-path.svg) 18 | 19 | 1. A `GatedDeployment` object is added to the cluster (see the section for 20 | [Installing the controller](#installing-the-controller)) 21 | 1. The controller fetches `GatedDeployment` objects using the Kubernetes API 22 | 1. When the `treatment` deployment specified in the `GatedDeployment` object is 23 | deployed and eligible for an A/B test, i.e., it has more than zero replicas 24 | and a different pod spec than the `control` deployment, the controller will 25 | start the experiment 26 | 1. The controller will poll the decision plugins and determine if the 27 | `treatment` deployment is causing harm to the metrics measured 28 | 1. The controller will either roll back the `treatment` deployment (by setting 29 | the number of replicas to zero), or promote the `treatment` deployment by 30 | setting the `control` deployment's image to that of the `treatment` 31 | deployment, followed by scaling the `treatment` deployment down to zero 32 | replicas 33 | 34 | ## Usage 35 | 36 | ### Installing the controller 37 | 38 | #### Using `kubectl` 39 | 40 | To create the `GatedDeployment` controller on an existing Kubernetes cluster, 41 | run the following: 42 | 43 | ```sh 44 | kubectl apply -f gated-deployments.yml 45 | ``` 46 | 47 | This creates all the necessary resources and deploys the controller in the 48 | `kubernetes-gated-deployments` namespace. 49 | 50 | #### Using Helm 51 | 52 | Alternatively, Helm can be used to install and manage the resources and 53 | controller. To install, run the following: 54 | 55 | ```sh 56 | helm install helm/kubernetes-gated-deployments --name kubernetes-gated-deployments 57 | ``` 58 | 59 | See the [Developing](#developing) section for running locally during 60 | development. 61 | 62 | ### Create `control` and `treatment` deployments 63 | 64 | Create two identical deployments with different names (e.g., 65 | `example-rest-service-control` and `example-rest-service-treatment`). Initially, 66 | the number of replicas for the treatment deployment must be set to 0 and control 67 | deployment will be the only one taking production traffic. 68 | 69 | Example deployment manifests are available [here](./examples). 70 | 71 | NOTE: The names of the deployments cannot be a prefix of the other. The gated 72 | deployment controller uses the deployment name as the host prefix (since pod 73 | names are of the form `-` and if one deployment name is 74 | the prefix of the other, it will include data from all the pods. For example, 75 | using `example-rest-service` for control and `example-rest-service-treatment` 76 | for treatment will result in control including the data for treatment as well. 77 | 78 | ### Configure the gated deployment 79 | 80 | Create an 81 | [`example-rest-service-gated-deployment.yml`](./examples/example-rest-service-gated-deployment.yml) 82 | file like below: 83 | 84 | ```yml 85 | apiVersion: 'kubernetes-client.io/v1' 86 | kind: GatedDeployment 87 | metadata: 88 | name: example-rest-service 89 | deploymentDescriptor: 90 | control: 91 | name: example-rest-service-control 92 | treatment: 93 | name: example-rest-service-treatment 94 | decisionPlugins: 95 | - name: newRelicPerformance 96 | accountId: 807783 97 | secretName: newrelic-secrets 98 | secretKey: example-rest-service 99 | appName: example-rest-service 100 | minSamples: 50 101 | maxTime: 600 102 | testPath: /shopper/products 103 | ``` 104 | 105 | Save the file and run: 106 | 107 | ```sh 108 | kubectl apply -f example-rest-service-gated-deployment.yml 109 | ``` 110 | 111 | In the example above, the `GatedDeployment` object specifies that we want to 112 | gate our deployments on the performance of the `/shopper/products` path, between 113 | the `example-rest-service-control` deployment and 114 | `example-rest-service-treatment` deployment (the latter of which we deploy new 115 | changes to). In this case, we specify that we want the controller to use the 116 | `newRelicPerformance` decision plugin to analyze performance data, which will be 117 | retrieved from New Relic (which our application is instrumented with). 118 | 119 | For this plugin, you will also need to create a secret containing the NewRelic 120 | API key; an example is shown below. In this case, `newRelic.secretName` is set 121 | to `newrelic-secrets`, and `newRelic.secretKey` is set to 122 | `example-rest-service`. This means that the controller will look in its deployed 123 | namespace for a secret called `newrelic-secrets`, and look in the secret data 124 | for the value corresponding to the key `example-rest-service`. 125 | 126 | ```yml 127 | apiVersion: v1 128 | kind: Secret 129 | metadata: 130 | name: newrelic-secrets 131 | type: Opaque 132 | data: 133 | example-rest-service: aW5zaWdodHNBcGlLZXk= 134 | ``` 135 | 136 | Within the `deploymentDescriptor` section of the `GatedDeployment` object, these 137 | are the possible options to customize. All options are required unless 138 | explicitly specified as optional. 139 | 140 | |Property|Description| 141 | |---|---| 142 | |`control`|Section describing the control deployment.| 143 | |`control.name`|Name of the control deployment.| 144 | |`treatment`|Section describing the treatment deployment. This should be the one normally deployed, e.g. as part of your CICD pipeline.| 145 | |`treatment.name`|Name of the treatment deployment.| 146 | |`decisionPlugins`|Section containing the list of decision plugin config objects. See [Plugin configurations](#plugin-configurations) below for details on specific plugins.| 147 | 148 | ### Plugin configurations 149 | 150 | Each type of plugin will require its own configuration. The following parameters 151 | are common to all plugins: 152 | 153 | |Property|Description| 154 | |---|---| 155 | |`name`|The plugin name. This allows the plugin factory to find the correct plugin class.| 156 | |`maxTime` (optional)|The maximum amount of time the experiment will run, at which point the A/B test will stop and automatically roll out the treatment deployment to the control deployment. When not specified, this defaults to 600 seconds (10 minutes)| 157 | 158 | Plugins are designed to return one of three values: 159 | * `WAIT`: if the analysis cannot make a conclusion about the metric yet, e.g., 160 | it requires a minimum amount of time or if the result is not yet statistically 161 | significant 162 | * `PASS`: if the treatment version does no harm to the metric analyzed 163 | * `FAIL`: if the treatment does harm to the metric analyzed 164 | 165 | #### New Relic performance plugin 166 | 167 | |Property|Description| 168 | |---|---| 169 | |`name`|Must be `newRelicPerformance`| 170 | |`accountId`|Account ID of the New Relic account integrated with your application.| 171 | |`secretName`|Name of the secret where your New Relic API keys can be found. This should be created in the namespace where `kubernetes-gated-deployments` is deployed.| 172 | |`secretKey`|Name of the key in the secret specified in `secretName` that contains the New Relic Insights API key, used to run NRQL to collect performance data.| 173 | |`appName`|Name of the New Relic application.| 174 | |`testPath`|Path that you want to measure performance of for both deployments.| 175 | |`minSamples`|The minimum number of samples required for each deployment before testing for significance.| 176 | |`zScoreThreshold` (optional)|The Z Score threshold for Mann-Whitney U test. Defaults to 1.96, which corresponds to a p-value of 0.05| 177 | |`harmThreshold` (optional)|Maximum allowable ratio of treatment to control U values from the Mann-Whitney U Test before treatment is marked as causing harm. This defaults to 1.5.| 178 | 179 | ### Contributing plugins 180 | 181 | To contribute a new plugin, create a new plugins class in [lib/plugins](lib/plugins) that is a subclass of [`Plugin`](lib/plugins/plugin.js). At minimum, you should implement the following methods: 182 | * `build`: this should create the plugin with any necessary setup 183 | * `_poll`: this is called periodically, and it should fetch and analyze metrics to return a `DecisionResult` 184 | 185 | The following methods are implemented by default: 186 | * `onExperimentStart`: this is called when the experiment starts, and sets the experiment start time 187 | * `onExperimentStop`: this is called when an experiment ends, and clears the experiment start time 188 | * `onExperimentPoll`: this is called on every polling interval; it will check if the maximum experiment duration has been reached and return `PASS` if it has, or it will return the result from `_poll`. 189 | 190 | ### Rolling out new versions 191 | 192 | To roll out a new version, update the treatment deployment with the new image 193 | and set the number of replicas to a non zero value (depending on the percentage 194 | of traffic you want to send to the new version). 195 | 196 | Once the treatment deploy is rolled out, the gated deployment controller will 197 | start a new experiment and start polling for decisions from the decision 198 | plugins. The experiment runs until either all plugins have returned `PASS`, or 199 | any single plugin returns `FAIL`, at which point the controller will set the 200 | `gatedDeployStatus` annotation on the treatment deployment to either `noHarm` or 201 | `harm` respectively. 202 | 203 | An example command to get the value of the annotation 204 | 205 | ```sh 206 | kubectl get deploy -o jsonpath='{.metadata.annotations.gatedDeployStatus}' example-rest-service-treatment 207 | ``` 208 | 209 | This value can be periodically polled to check if the new version is causing 210 | harm or not in the CI/CD pipeline of the application. If the deployment causes 211 | no harm, the controller automatically rolls it out the new version to the 212 | control deployment. The status of the rollout can be checked using the below 213 | command. 214 | 215 | ```sh 216 | kubectl rollout status deploy/example-rest-service-control 217 | ``` 218 | 219 | ## Developing 220 | 221 | See [CONTRIBUTING.md](CONTRIBUTING.md) for how to contribute to this project. 222 | 223 | You can develop locally with 224 | [Minikube](https://kubernetes.io/docs/setup/minikube/). 225 | 226 | On Linux, the `kvm2` driver provides better performance than the default 227 | `virtualbox` driver, but either will work: 228 | 229 | ``` 230 | minikube start --vm-driver=kvm2 231 | ``` 232 | 233 | `minikube start` will configure your `kubeconfig` for your local Minikube 234 | cluster and set the current context to be for Minikube. With that configuration 235 | you can run the `kubernetes-gated-deployment` controller on your host operating 236 | system: 237 | 238 | ``` 239 | npm start 240 | ``` 241 | 242 | ## License 243 | 244 | Kubernetes Gated Deployments is [MIT licensed](LICENSE). 245 | 246 | ## Authors 247 | 248 | * Steven Fu 249 | * Satish Ravi 250 | * Jacob Brooks 251 | * Silas Boyd-Wickizer -------------------------------------------------------------------------------- /architecture.svg: -------------------------------------------------------------------------------- 1 | 2 |
Control
deployment
[Not supported by viewer]
Treatment
deployment
[Not supported by viewer]
Service
Service
Data backend
Data backend
GatedDeployment
Controller
[Not supported by viewer]
Controls deployments
Controls deployments
Pushes performance or functionality data
Pushes performance or functionality data
Retrieves configured metrics
Retrieves configured metrics
Kubernetes
Kubernetes 3 | -------------------------------------------------------------------------------- /gated-deployment-path.svg: -------------------------------------------------------------------------------- 1 | 2 |
Control
app v1.1
[Not supported by viewer]
Service
Service
Control
app v1.2
[Not supported by viewer]
Control
app v1.1
[Not supported by viewer]
Treatment
app v1.2
[Not supported by viewer]
Service
Service
Service
Service
Start gated deployment
Start gated deployment
Split test shows no harm
<div>Split test shows no harm</div>
Split test shows harm
Split test shows harm 3 | -------------------------------------------------------------------------------- /lib/watchers/deployment-watcher.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | const chai = require('chai') 3 | const clone = require('clone') 4 | const sinon = require('sinon') 5 | const sinonChai = require('sinon-chai') 6 | const moment = require('moment') 7 | 8 | const analysis = require('../analysis') 9 | const { DEPLOYMENT_ANNOTATION_NAME, EXPERIMENT_ANNOTATION_NAME } = require('./constants') 10 | const DeploymentHelper = require('../deployment-helper') 11 | const DeploymentWatcher = require('./deployment-watcher') 12 | const { DecisionResults } = require('../plugins/plugin') 13 | 14 | chai.use(sinonChai) 15 | 16 | const { expect } = chai 17 | 18 | describe('DeploymentWatcher', () => { 19 | const deploymentDescriptor = { 20 | control: { 21 | name: 'example-rest-service-control', 22 | testPath: '/shopper/products' 23 | }, 24 | treatment: { 25 | name: 'example-rest-service-treatment', 26 | testPath: '/shopper/products' 27 | }, 28 | newRelic: { 29 | appName: 'nr-app' 30 | }, 31 | experiment: { 32 | minSamples: 10, 33 | maxTime: 5 34 | } 35 | } 36 | let deploymentHelper, logger, mockPlugin 37 | 38 | beforeEach(() => { 39 | deploymentHelper = { 40 | updatePodSpec: sinon.stub().resolves(), 41 | kill: sinon.stub().resolves(), 42 | setAnnotation: sinon.stub().resolves(), 43 | isPodSpecIdentical: sinon.stub(), 44 | getPodSpecHash: sinon.stub().returns('hash') 45 | } 46 | logger = { 47 | info: sinon.stub(), 48 | warn: sinon.stub(), 49 | error: sinon.stub() 50 | } 51 | mockPlugin = { 52 | build: sinon.stub().resolves(), 53 | onExperimentStart: sinon.stub(), 54 | onExperimentStop: sinon.stub(), 55 | onExperimentPoll: sinon.stub().resolves() 56 | } 57 | }) 58 | 59 | describe('constructor', () => { 60 | it('sets fields on instance', () => { 61 | const watcher = new DeploymentWatcher({ 62 | kubeClient: 'mockkubeclient', 63 | logger: 'mocklogger', 64 | deploymentDescriptor: 'mockdd', 65 | namespace: 'ns', 66 | plugins: ['mockplugin'], 67 | pollerIntervalMilliseconds: 5000, 68 | gatedDeploymentId: 'mockgd' 69 | }) 70 | 71 | expect(watcher).to.be.an('object') 72 | expect(watcher._kubeClient).to.equal('mockkubeclient') 73 | expect(watcher._logger).to.equal('mocklogger') 74 | expect(watcher._deploymentDescriptor).to.equal('mockdd') 75 | expect(watcher._namespace).to.equal('ns') 76 | expect(watcher._plugins).to.deep.equal(['mockplugin']) 77 | expect(watcher._pollerIntervalMilliseconds).to.equal(5000) 78 | expect(watcher._gatedDeploymentId).to.equal('mockgd') 79 | expect(watcher._experiment).to.deep.equal({ 80 | startTime: null, 81 | pollerInterval: null, 82 | treatmentPodSpec: null 83 | }) 84 | expect(watcher._deploymentHelper).to.be.an.instanceOf(DeploymentHelper) 85 | }) 86 | }) 87 | 88 | describe('getStream', () => { 89 | it('returns gated deployment watch stream', () => { 90 | const watcher = new DeploymentWatcher({ 91 | kubeClient: { 92 | apis: { apps: { v1: { watch: { namespaces: sinon.stub().returns({ 93 | deploy: sinon.stub().returns({ 94 | getStream: sinon.stub().returns('stream') 95 | }) 96 | }) } } } } 97 | }, 98 | logger: 'mocklogger', 99 | deploymentDescriptor: { treatment: { name: 'treatment' } }, 100 | namespace: 'ns', 101 | pollerIntervalMilliseconds: 5000 102 | }) 103 | 104 | expect(watcher.getStream()).to.equal('stream') 105 | }) 106 | }) 107 | 108 | describe('_validateAndGetExperimentAnnotation', () => { 109 | const constructDeployment = annotationVal => ({ 110 | metadata: { 111 | annotations: { 112 | [EXPERIMENT_ANNOTATION_NAME]: annotationVal 113 | } 114 | } 115 | }) 116 | const watcher = new DeploymentWatcher({}) 117 | 118 | it('returns empty object if annotation is not set', () => { 119 | const treatmentDeployment = constructDeployment() 120 | treatmentDeployment.metadata.annotations = {} 121 | 122 | expect(watcher._validateAndGetExperimentAnnotation(treatmentDeployment)).to.deep.equal({}) 123 | }) 124 | 125 | it('returns empty object if annotation is set to null', () => { 126 | const treatmentDeployment = constructDeployment(null) 127 | 128 | expect(watcher._validateAndGetExperimentAnnotation(treatmentDeployment)).to.deep.equal({}) 129 | }) 130 | 131 | it('returns empty object if annotation is invalid', () => { 132 | const treatmentDeployment = constructDeployment('invalid') 133 | 134 | expect(watcher._validateAndGetExperimentAnnotation(treatmentDeployment)).to.deep.equal({}) 135 | }) 136 | 137 | it('returns empty object if annotation does not contain required fields', () => { 138 | const treatmentDeployment = constructDeployment(JSON.stringify({ some: 'field' })) 139 | 140 | expect(watcher._validateAndGetExperimentAnnotation(treatmentDeployment)).to.deep.equal({}) 141 | }) 142 | 143 | it('returns empty object if annotation does not have a valid time', () => { 144 | const treatmentDeployment = constructDeployment(JSON.stringify({ 145 | startTime: 'invalid', 146 | podSpecHash: 'foo' 147 | })) 148 | 149 | expect(watcher._validateAndGetExperimentAnnotation(treatmentDeployment)).to.deep.equal({}) 150 | }) 151 | 152 | it('returns annotation object on valid annotation value', () => { 153 | const startTime = moment.utc().toISOString() 154 | const podSpecHash = 'foo' 155 | const treatmentDeployment = constructDeployment(JSON.stringify({ 156 | startTime, podSpecHash 157 | })) 158 | 159 | const annotation = watcher._validateAndGetExperimentAnnotation(treatmentDeployment) 160 | 161 | expect(annotation.startTime.toISOString()).to.equal(startTime) 162 | expect(annotation.podSpecHash).to.equal(podSpecHash) 163 | }) 164 | }) 165 | 166 | describe('_isEligibleForExperiment', () => { 167 | let controlDeployment, treatmentDeployment, watcher 168 | 169 | beforeEach(() => { 170 | controlDeployment = { 171 | spec: { 172 | replicas: 2 173 | } 174 | } 175 | 176 | treatmentDeployment = clone(controlDeployment) 177 | 178 | watcher = new DeploymentWatcher({ 179 | deploymentDescriptor 180 | }) 181 | watcher._deploymentHelper = deploymentHelper 182 | watcher._deploymentHelper.get = sinon.stub().resolves({ body: controlDeployment }) 183 | }) 184 | 185 | it('returns false if replicas are positive and specs are same', async () => { 186 | watcher._deploymentHelper.isPodSpecIdentical.returns(true) 187 | 188 | expect(await watcher._isEligibleForExperiment(treatmentDeployment)).to.equal(false) 189 | expect(watcher._deploymentHelper.isPodSpecIdentical).to.have.been.calledOnceWith(controlDeployment, treatmentDeployment) 190 | }) 191 | 192 | it('returns true if replicas are positive and specs are different', async () => { 193 | watcher._deploymentHelper.isPodSpecIdentical.returns(false) 194 | 195 | expect(await watcher._isEligibleForExperiment(treatmentDeployment)).to.equal(true) 196 | expect(watcher._deploymentHelper.isPodSpecIdentical).to.have.been.calledOnceWith(controlDeployment, treatmentDeployment) 197 | }) 198 | }) 199 | 200 | describe('_startExperiment', () => { 201 | let clock, watcher 202 | 203 | const zeroReplicaDeploy = { spec: { replicas: 0, template: { spec: 'zeroPodSpec' } } } 204 | const ineligibleDeploy = { spec: { replicas: 2, template: { spec: 'ineligible' } } } 205 | const eligibleDeploy = { spec: { replicas: 2, template: { spec: 'eligible' } } } 206 | const invalidDeploy = { spec: { replicas: 2, template: { spec: 'invalid' } } } 207 | 208 | beforeEach(() => { 209 | clock = sinon.useFakeTimers() 210 | 211 | watcher = new DeploymentWatcher({ 212 | kubeClient: 'kubeclient', 213 | deploymentDescriptor, 214 | pollerIntervalMilliseconds: 5000, 215 | plugins: [mockPlugin], 216 | logger 217 | }) 218 | watcher._deploymentHelper = deploymentHelper 219 | watcher._isEligibleForExperiment = sinon.stub().resolves(false) 220 | watcher._isEligibleForExperiment.withArgs(eligibleDeploy).resolves(true) 221 | watcher._isEligibleForExperiment.withArgs(invalidDeploy).rejects('error') 222 | watcher._killTreatment = sinon.stub().resolves() 223 | watcher._poll = sinon.stub().resolves() 224 | }) 225 | 226 | afterEach(() => { 227 | watcher._clearExperiment() 228 | clock.restore() 229 | }) 230 | 231 | it('does not start experiment if treatment replicas is zero', async () => { 232 | await watcher._startExperiment(zeroReplicaDeploy) 233 | 234 | expect(watcher._isEligibleForExperiment).to.have.been.callCount(0) 235 | expect(watcher._killTreatment).to.have.been.callCount(0) 236 | expect(watcher._experiment).to.deep.equal({ 237 | startTime: null, 238 | pollerInterval: null, 239 | treatmentPodSpec: null 240 | }) 241 | expect(deploymentHelper.setAnnotation).to.have.been.calledOnceWith( 242 | deploymentDescriptor.treatment.name, EXPERIMENT_ANNOTATION_NAME, null) 243 | }) 244 | 245 | it('does not start experiment and kills treatment if treatment deployment is not eligible', async () => { 246 | await watcher._startExperiment(ineligibleDeploy) 247 | 248 | expect(watcher._isEligibleForExperiment).to.have.been.calledOnceWith(ineligibleDeploy) 249 | expect(watcher._killTreatment).to.have.been.calledOnceWith(analysis.results.noHarm) 250 | expect(watcher._experiment).to.deep.equal({ 251 | startTime: null, 252 | pollerInterval: null, 253 | treatmentPodSpec: null 254 | }) 255 | expect(deploymentHelper.setAnnotation).to.have.been.calledOnceWith( 256 | deploymentDescriptor.treatment.name, EXPERIMENT_ANNOTATION_NAME, null) 257 | }) 258 | 259 | it('starts experiment if treatment deployment is eligible and sets experiment annotation', async () => { 260 | await watcher._startExperiment(eligibleDeploy) 261 | 262 | clock.tick(10000) 263 | 264 | expect(watcher._isEligibleForExperiment).to.have.been.calledOnceWith(eligibleDeploy) 265 | expect(watcher._experiment.startTime.utcOffset()).to.equal(0) 266 | expect(watcher._experiment.treatmentPodSpec).to.equal(eligibleDeploy.spec.template.spec) 267 | expect(watcher._experiment.pollerInterval).to.be.an('object') 268 | expect(watcher._poll).to.have.callCount(2) 269 | expect(deploymentHelper.setAnnotation).to.have.been.calledOnceWith( 270 | deploymentDescriptor.treatment.name, EXPERIMENT_ANNOTATION_NAME, JSON.stringify({ 271 | startTime: watcher._experiment.startTime.toISOString(), 272 | podSpecHash: 'hash' 273 | }) 274 | ) 275 | }) 276 | 277 | it('continues existing experiment from experiment annotation if pod spec matches', async () => { 278 | watcher._validateAndGetExperimentAnnotation = sinon.stub().returns({ 279 | startTime: moment.utc('2019-07-10T22:50:18.234Z'), 280 | podSpecHash: 'hash' 281 | }) 282 | 283 | await watcher._startExperiment(eligibleDeploy) 284 | 285 | expect(watcher._isEligibleForExperiment).to.have.been.calledOnceWith(eligibleDeploy) 286 | expect(watcher._experiment.startTime.toISOString()).to.equal('2019-07-10T22:50:18.234Z') 287 | expect(watcher._experiment.treatmentPodSpec).to.equal(eligibleDeploy.spec.template.spec) 288 | expect(watcher._experiment.pollerInterval).to.be.an('object') 289 | expect(deploymentHelper.setAnnotation).to.have.been.calledOnceWith( 290 | deploymentDescriptor.treatment.name, EXPERIMENT_ANNOTATION_NAME, JSON.stringify({ 291 | startTime: '2019-07-10T22:50:18.234Z', 292 | podSpecHash: 'hash' 293 | }) 294 | ) 295 | }) 296 | 297 | it('starts new experiment if pod spec in experiment annotation does not match', async () => { 298 | watcher._validateAndGetExperimentAnnotation = sinon.stub().returns({ 299 | startTime: moment.utc('2019-07-10T22:50:18.234Z'), 300 | podSpecHash: 'hash2' 301 | }) 302 | 303 | await watcher._startExperiment(eligibleDeploy) 304 | 305 | expect(watcher._isEligibleForExperiment).to.have.been.calledOnceWith(eligibleDeploy) 306 | expect(watcher._experiment.startTime.utcOffset()).to.equal(0) 307 | expect(watcher._experiment.treatmentPodSpec).to.equal(eligibleDeploy.spec.template.spec) 308 | expect(watcher._experiment.pollerInterval).to.be.an('object') 309 | expect(deploymentHelper.setAnnotation).to.have.been.calledOnceWith( 310 | deploymentDescriptor.treatment.name, EXPERIMENT_ANNOTATION_NAME, JSON.stringify({ 311 | startTime: watcher._experiment.startTime.toISOString(), 312 | podSpecHash: 'hash' 313 | }) 314 | ) 315 | }) 316 | 317 | it('catches and logs error if eligibility check throws', async () => { 318 | await watcher._startExperiment(invalidDeploy) 319 | 320 | expect(watcher._isEligibleForExperiment).to.have.been.calledOnceWith(invalidDeploy) 321 | expect(watcher._experiment).to.deep.equal({ 322 | startTime: null, 323 | pollerInterval: null, 324 | treatmentPodSpec: null 325 | }) 326 | expect(watcher._logger.error).to.have.callCount(1) 327 | }) 328 | 329 | it('starts experiment for all plugins', async () => { 330 | await watcher._startExperiment(eligibleDeploy) 331 | 332 | expect(mockPlugin.onExperimentStart).to.have.been.calledOnceWith(watcher._experiment.startTime) 333 | }) 334 | }) 335 | 336 | describe('._clearExperiment', () => { 337 | let watcher 338 | 339 | beforeEach(() => { 340 | watcher = new DeploymentWatcher({ 341 | kubeClient: 'kubeclient', 342 | deploymentDescriptor, 343 | pollerIntervalMilliseconds: 5000, 344 | plugins: [mockPlugin], 345 | logger 346 | }) 347 | watcher._deploymentHelper = deploymentHelper 348 | }) 349 | 350 | it('does nothing if no experiment is running', async () => { 351 | await watcher._clearExperiment() 352 | 353 | expect(watcher._logger.info).to.have.callCount(0) 354 | expect(deploymentHelper.setAnnotation).to.have.been.callCount(0) 355 | }) 356 | 357 | it('clears experiment if it is running and sets experiment annotation to null', async () => { 358 | const pollerInterval = setInterval(() => {}, 1000) 359 | watcher._experiment = { 360 | startTime: moment.utc(), 361 | pollerInterval, 362 | treatmentPodSpec: 'treatment-pod-spec' 363 | } 364 | 365 | await watcher._clearExperiment() 366 | 367 | expect(watcher._experiment).to.deep.equal({ 368 | startTime: null, 369 | pollerInterval: null, 370 | treatmentPodSpec: null 371 | }) 372 | expect(pollerInterval._destroyed).to.equal(false) 373 | expect(mockPlugin.onExperimentStop).to.have.callCount(1) 374 | expect(deploymentHelper.setAnnotation).to.have.been.calledOnceWith( 375 | deploymentDescriptor.treatment.name, EXPERIMENT_ANNOTATION_NAME, null) 376 | }) 377 | 378 | it('catches and logs error if setting experiment annotation fails', async () => { 379 | const pollerInterval = setInterval(() => {}, 1000) 380 | watcher._experiment = { 381 | startTime: moment.utc(), 382 | pollerInterval, 383 | treatmentPodSpec: 'treatment-pod-spec' 384 | } 385 | deploymentHelper.setAnnotation = sinon.stub().rejects() 386 | 387 | await watcher._clearExperiment() 388 | 389 | expect(watcher._experiment).to.deep.equal({ 390 | startTime: null, 391 | pollerInterval: null, 392 | treatmentPodSpec: null 393 | }) 394 | expect(pollerInterval._destroyed).to.equal(false) 395 | expect(mockPlugin.onExperimentStop).to.have.callCount(1) 396 | expect(deploymentHelper.setAnnotation).to.have.been.calledOnceWith( 397 | deploymentDescriptor.treatment.name, EXPERIMENT_ANNOTATION_NAME, null) 398 | expect(watcher._logger.error).to.have.callCount(1) 399 | }) 400 | }) 401 | 402 | describe('._killTreatment', () => { 403 | it('kills treatment and sets status annotation', async () => { 404 | const watcher = new DeploymentWatcher({ 405 | deploymentDescriptor, 406 | logger 407 | }) 408 | watcher._deploymentHelper = deploymentHelper 409 | 410 | await watcher._killTreatment('annotation') 411 | 412 | expect(deploymentHelper.kill).to.have.been.calledOnceWith(deploymentDescriptor.treatment.name) 413 | expect(deploymentHelper.setAnnotation).to.have.been.calledOnceWith( 414 | deploymentDescriptor.treatment.name, DEPLOYMENT_ANNOTATION_NAME, 'annotation') 415 | }) 416 | }) 417 | 418 | describe('._passExperiment', () => { 419 | it('updates the control to treatment spec, kills treatment with noHarm annotation', async () => { 420 | const watcher = new DeploymentWatcher({ 421 | deploymentDescriptor, 422 | logger 423 | }) 424 | watcher._deploymentHelper = deploymentHelper 425 | watcher._killTreatment = sinon.stub() 426 | watcher._experiment = { 427 | treatmentPodSpec: 'some-spec' 428 | } 429 | watcher._clearExperiment = sinon.stub().resolves() 430 | 431 | await watcher._passExperiment() 432 | 433 | expect(deploymentHelper.updatePodSpec).to.have.been.calledOnceWith(deploymentDescriptor.control.name, 'some-spec') 434 | expect(watcher._killTreatment).to.have.been.calledOnceWith(analysis.results.noHarm) 435 | expect(watcher._clearExperiment).to.have.been.calledOnceWith() 436 | }) 437 | }) 438 | 439 | describe('._failExperiment', () => { 440 | it('kills treatment with harm annotation', async () => { 441 | const watcher = new DeploymentWatcher({ 442 | deploymentDescriptor, 443 | logger 444 | }) 445 | watcher._killTreatment = sinon.stub().resolves() 446 | watcher._clearExperiment = sinon.stub().resolves() 447 | 448 | await watcher._failExperiment() 449 | 450 | expect(watcher._killTreatment).to.have.been.calledOnceWith(analysis.results.harm) 451 | expect(watcher._clearExperiment).to.have.been.calledOnceWith() 452 | }) 453 | }) 454 | 455 | describe('._poll', () => { 456 | let watcher 457 | let anotherMockPlugin 458 | 459 | beforeEach(() => { 460 | anotherMockPlugin = { 461 | onExperimentPoll: sinon.stub().resolves() 462 | } 463 | watcher = new DeploymentWatcher({ 464 | logger, 465 | deploymentDescriptor, 466 | plugins: [mockPlugin, anotherMockPlugin] 467 | }) 468 | watcher._passExperiment = sinon.stub() 469 | watcher._failExperiment = sinon.stub() 470 | watcher._experiment = { 471 | startTime: moment.utc() 472 | } 473 | }) 474 | 475 | afterEach(() => { 476 | sinon.restore() 477 | }) 478 | 479 | it('passes experiment if all plugins return PASS', async () => { 480 | mockPlugin.onExperimentPoll.resolves(DecisionResults.PASS) 481 | anotherMockPlugin.onExperimentPoll.returns(DecisionResults.PASS) 482 | 483 | await watcher._poll() 484 | 485 | expect(watcher._passExperiment).to.have.callCount(1) 486 | expect(watcher._failExperiment).to.have.callCount(0) 487 | }) 488 | 489 | it('fails experiment if any plugins return FAIL', async () => { 490 | mockPlugin.onExperimentPoll.resolves(DecisionResults.PASS) 491 | anotherMockPlugin.onExperimentPoll.returns(DecisionResults.FAIL) 492 | 493 | await watcher._poll() 494 | 495 | expect(watcher._passExperiment).to.have.callCount(0) 496 | expect(watcher._failExperiment).to.have.callCount(1) 497 | }) 498 | 499 | it('fails experiment if plugins return FAIL/WAIT', async () => { 500 | mockPlugin.onExperimentPoll.resolves(DecisionResults.FAIL) 501 | anotherMockPlugin.onExperimentPoll.returns(DecisionResults.WAIT) 502 | 503 | await watcher._poll() 504 | 505 | expect(watcher._passExperiment).to.have.callCount(0) 506 | expect(watcher._failExperiment).to.have.callCount(1) 507 | }) 508 | 509 | it('does nothing if plugins return all WAIT', async () => { 510 | mockPlugin.onExperimentPoll.resolves(DecisionResults.WAIT) 511 | anotherMockPlugin.onExperimentPoll.resolves(DecisionResults.WAIT) 512 | 513 | await watcher._poll() 514 | 515 | expect(watcher._passExperiment).to.have.callCount(0) 516 | expect(watcher._failExperiment).to.have.callCount(0) 517 | }) 518 | 519 | it('does nothing if plugins return some WAIT', async () => { 520 | mockPlugin.onExperimentPoll.resolves(DecisionResults.PASS) 521 | anotherMockPlugin.onExperimentPoll.resolves(DecisionResults.WAIT) 522 | 523 | await watcher._poll() 524 | 525 | expect(watcher._passExperiment).to.have.callCount(0) 526 | expect(watcher._failExperiment).to.have.callCount(0) 527 | }) 528 | 529 | it('treats an error as a WAIT, with a PASS', async () => { 530 | mockPlugin.onExperimentPoll.resolves(DecisionResults.PASS) 531 | anotherMockPlugin.onExperimentPoll.rejects('Oh no!') 532 | 533 | await watcher._poll() 534 | 535 | expect(watcher._passExperiment).to.have.callCount(0) 536 | expect(watcher._failExperiment).to.have.callCount(0) 537 | expect(watcher._logger.error).to.have.callCount(1) 538 | }) 539 | 540 | it('treats an error as a WAIT, with a FAIL', async () => { 541 | mockPlugin.onExperimentPoll.resolves(DecisionResults.FAIL) 542 | anotherMockPlugin.onExperimentPoll.rejects('Oh no!') 543 | 544 | await watcher._poll() 545 | 546 | expect(watcher._passExperiment).to.have.callCount(0) 547 | expect(watcher._failExperiment).to.have.callCount(1) 548 | expect(watcher._logger.error).to.have.callCount(1) 549 | }) 550 | }) 551 | 552 | describe('.onAdded', () => { 553 | it('starts experiment', () => { 554 | const watcher = new DeploymentWatcher({}) 555 | watcher._startExperiment = sinon.stub().resolves() 556 | 557 | expect(watcher.onAdded('fake-deployment')).to.be.an.instanceOf(Promise) 558 | expect(watcher._startExperiment).to.have.been.calledOnceWith('fake-deployment') 559 | expect(watcher._previous).to.equal('fake-deployment') 560 | }) 561 | }) 562 | 563 | describe('.onModified', () => { 564 | const deployment = { 565 | annotation: '1', 566 | spec: { 567 | replicas: 4 568 | } 569 | } 570 | it('clears and starts experiment if no previous deployment', async () => { 571 | const watcher = new DeploymentWatcher({}) 572 | watcher._clearExperiment = sinon.stub().resolves() 573 | watcher._startExperiment = sinon.stub().resolves() 574 | 575 | await watcher.onModified(deployment) 576 | 577 | expect(watcher._clearExperiment).to.have.been.calledOnceWith() 578 | expect(watcher._startExperiment).to.have.been.calledOnceWith(deployment) 579 | }) 580 | 581 | it('clears and starts experiment if previous exists and treatment replicas changes', async () => { 582 | const watcher = new DeploymentWatcher({}) 583 | watcher._previous = { 584 | annotation: '1', 585 | spec: { 586 | replicas: 2 587 | } 588 | } 589 | watcher._clearExperiment = sinon.stub().resolves() 590 | watcher._startExperiment = sinon.stub().resolves() 591 | watcher._deploymentHelper = deploymentHelper 592 | watcher._deploymentHelper.isPodSpecIdentical.returns(true) 593 | 594 | await watcher.onModified(deployment) 595 | 596 | expect(watcher._clearExperiment).to.have.been.calledOnceWith() 597 | expect(watcher._startExperiment).to.have.been.calledOnceWith(deployment) 598 | expect(watcher._previous).to.equal(deployment) 599 | }) 600 | 601 | it('clears and starts experiment if previous exists and treatment pod spec changes', async () => { 602 | const watcher = new DeploymentWatcher({}) 603 | const previous = { 604 | spec: { 605 | replicas: 4 606 | } 607 | } 608 | watcher._previous = previous 609 | watcher._clearExperiment = sinon.stub().resolves() 610 | watcher._startExperiment = sinon.stub().resolves() 611 | watcher._deploymentHelper = deploymentHelper 612 | watcher._deploymentHelper.isPodSpecIdentical.returns(false) 613 | 614 | await watcher.onModified(deployment) 615 | 616 | expect(watcher._clearExperiment).to.have.been.calledOnceWith() 617 | expect(watcher._startExperiment).to.have.been.calledOnceWith(deployment) 618 | expect(watcher._deploymentHelper.isPodSpecIdentical).to.have.been.calledWith(deployment, previous) 619 | expect(watcher._previous).to.equal(deployment) 620 | }) 621 | 622 | it('does nothing if experiment exists and treatment replicas, pod spec does not change', async () => { 623 | const watcher = new DeploymentWatcher({}) 624 | const previous = { 625 | annotation: '2', 626 | spec: { 627 | replicas: 4 628 | } 629 | } 630 | watcher._previous = previous 631 | watcher._clearExperiment = sinon.stub().resolves() 632 | watcher._startExperiment = sinon.stub().resolves() 633 | watcher._deploymentHelper = deploymentHelper 634 | watcher._deploymentHelper.isPodSpecIdentical.returns(true) 635 | 636 | await watcher.onModified(deployment) 637 | 638 | expect(watcher._clearExperiment).to.have.callCount(0) 639 | expect(watcher._startExperiment).to.have.callCount(0) 640 | expect(watcher._deploymentHelper.isPodSpecIdentical).to.have.been.calledWith(deployment, previous) 641 | expect(watcher._previous).to.equal(deployment) 642 | }) 643 | }) 644 | 645 | describe('.onDeleted', () => { 646 | it('clears experiment', async () => { 647 | const watcher = new DeploymentWatcher({}) 648 | watcher._clearExperiment = sinon.stub().resolves() 649 | watcher._previous = 'previous' 650 | 651 | await watcher.onDeleted('fake-deployment') 652 | expect(watcher._clearExperiment).to.have.been.calledOnceWith() 653 | expect(watcher._previous).to.equal(null) 654 | }) 655 | }) 656 | }) 657 | --------------------------------------------------------------------------------