├── clients ├── python │ ├── requirements.txt │ ├── scrutinize │ │ ├── __init__.py │ │ ├── seed.py │ │ └── client.py │ ├── .gitignore │ ├── CONTRIBUTING.md │ ├── setup.py │ ├── README.md │ ├── LICENSE │ └── tests │ │ └── test_scrutinize_client.py ├── javascript │ ├── .gitignore │ ├── CONTRIBUTING.md │ ├── tsconfig.json │ ├── package.json │ ├── README.md │ ├── seed.ts │ ├── package-lock.json │ ├── scrutinize.ts │ └── scrutinize_test.ts └── go │ └── scrutinize.go ├── .dockerignore ├── server ├── .gitignore ├── .eslintignore ├── tsconfig.json ├── src │ ├── controller │ │ ├── reporting.ts │ │ ├── metric.ts │ │ └── experiment.ts │ ├── middleware │ │ ├── asyncRouter.ts │ │ ├── errors.ts │ │ └── logger.ts │ ├── routes │ │ ├── health.ts │ │ ├── reporting.ts │ │ ├── metric.ts │ │ └── experiment.ts │ ├── config.ts │ ├── database │ │ ├── model.ts │ │ ├── metric.ts │ │ ├── reporting.ts │ │ └── experiment.ts │ └── index.ts ├── .eslintrc.js └── package.json ├── docker ├── run.sh ├── Dockerfile.migrador ├── nginx.conf └── Dockerfile.service ├── web ├── src │ ├── react-app-env.d.ts │ ├── index.css │ ├── reportWebVitals.ts │ ├── index.tsx │ ├── App.css │ ├── components │ │ ├── experiment │ │ │ ├── PercentageSlider.tsx │ │ │ ├── MetricSelect.tsx │ │ │ ├── EndExperimentForm.tsx │ │ │ ├── StartExperimentForm.tsx │ │ │ └── ExperimentForm.tsx │ │ ├── performance │ │ │ ├── Controls.tsx │ │ │ ├── Chart.tsx │ │ │ └── InfoBar.tsx │ │ ├── Performance.tsx │ │ ├── MetricList.tsx │ │ ├── ExperimentList.tsx │ │ └── metrics │ │ │ └── MetricForm.tsx │ ├── App.tsx │ └── api │ │ └── api.ts ├── README.md ├── public │ ├── robots.txt │ ├── favicon.ico │ ├── manifest.json │ └── index.html ├── .gitignore ├── tsconfig.json └── package.json ├── .gitignore ├── docs ├── readme │ ├── reporting.png │ ├── add_metrics.png │ ├── add_experiment.png │ └── experiment_list.png └── quickstart │ ├── DEVELOPMENT.md │ └── QUICKSTART.md ├── db ├── 20201022191322_create_metric.sql ├── migrate_db.sh ├── 20201022190232_create_experiment.sql ├── 20201116112840_create_evaluation_criterion.sql ├── 20201022190233_create_run.sql ├── 20201022191341_create_measurement.sql └── 20201022190756_create_treatment.sql ├── docker-compose.yml ├── LICENSE └── README.md /clients/python/requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.7.2 2 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | **/node_modules 2 | venv 3 | **/venv 4 | -------------------------------------------------------------------------------- /clients/javascript/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | -------------------------------------------------------------------------------- /server/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | 4 | *.swo 5 | *.swp 6 | -------------------------------------------------------------------------------- /docker/run.sh: -------------------------------------------------------------------------------- 1 | nginx -g 'daemon off;' & 2 | node ./build/src/index.js 3 | -------------------------------------------------------------------------------- /web/src/react-app-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.swo 3 | 4 | venv 5 | scripts 6 | 7 | devnotes 8 | -------------------------------------------------------------------------------- /clients/python/scrutinize/__init__.py: -------------------------------------------------------------------------------- 1 | from scrutinize.client import ScrutinizeClient 2 | -------------------------------------------------------------------------------- /clients/go/scrutinize.go: -------------------------------------------------------------------------------- 1 | package scrutinize 2 | 3 | type ScrutinizeClient struct { 4 | } 5 | -------------------------------------------------------------------------------- /web/README.md: -------------------------------------------------------------------------------- 1 | # scrutinize-web 2 | 3 | This is the dashboard for the scrutinize deployment. 4 | -------------------------------------------------------------------------------- /web/public/robots.txt: -------------------------------------------------------------------------------- 1 | # https://www.robotstxt.org/robotstxt.html 2 | User-agent: * 3 | Disallow: 4 | -------------------------------------------------------------------------------- /web/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/angles-n-daemons/scrutinize/HEAD/web/public/favicon.ico -------------------------------------------------------------------------------- /docs/readme/reporting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/angles-n-daemons/scrutinize/HEAD/docs/readme/reporting.png -------------------------------------------------------------------------------- /docs/readme/add_metrics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/angles-n-daemons/scrutinize/HEAD/docs/readme/add_metrics.png -------------------------------------------------------------------------------- /docs/readme/add_experiment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/angles-n-daemons/scrutinize/HEAD/docs/readme/add_experiment.png -------------------------------------------------------------------------------- /docs/readme/experiment_list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/angles-n-daemons/scrutinize/HEAD/docs/readme/experiment_list.png -------------------------------------------------------------------------------- /server/.eslintignore: -------------------------------------------------------------------------------- 1 | # don't ever lint node_modules 2 | node_modules 3 | # don't lint build output (make sure it's set to your correct build folder name) 4 | dist 5 | # don't lint nyc coverage output 6 | coverage 7 | # don't lint myself 8 | .eslintrc.js 9 | -------------------------------------------------------------------------------- /docker/Dockerfile.migrador: -------------------------------------------------------------------------------- 1 | FROM golang:1.15 2 | 3 | WORKDIR /migrations 4 | 5 | RUN go get -u github.com/pressly/goose/cmd/goose 6 | 7 | RUN apt-get update && apt-get install netcat -y 8 | 9 | COPY ./db ./db 10 | 11 | CMD ["./migrate_db.sh"] 12 | -------------------------------------------------------------------------------- /clients/javascript/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Testing 2 | 3 | `npm run test` 4 | 5 | ## Seeding (and verifying client integration) 6 | 7 | `npm build && node build/seed.js` 8 | 9 | ## Publishing 10 | 11 | Change version in package.json then `npm publish` 12 | -------------------------------------------------------------------------------- /clients/python/.gitignore: -------------------------------------------------------------------------------- 1 | venv 2 | __pycache__ 3 | 4 | # Compiled python modules. 5 | *.pyc 6 | 7 | # Setuptools distribution folder. 8 | /dist/ 9 | /build/ 10 | 11 | # Python egg metadata, regenerated from source files by setuptools. 12 | /*.egg-info 13 | -------------------------------------------------------------------------------- /web/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "short_name": "Trial Run", 3 | "name": "Trial Run Experimentation Platform", 4 | "icons": [], 5 | "start_url": ".", 6 | "display": "standalone", 7 | "theme_color": "#000000", 8 | "background_color": "#ffffff" 9 | } 10 | -------------------------------------------------------------------------------- /clients/python/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Testing 2 | 3 | `python tests/test_scrutinize_client.py` 4 | 5 | ## Seeding (and verifying client integration) 6 | 7 | `python scrutinize/seed.py` 8 | 9 | ## Publishing 10 | 11 | Change version in setup.py then do: 12 | 13 | ``` 14 | python3 setup.py sdist bdist_wheel 15 | python3 -m twine upload dist/* 16 | ``` 17 | -------------------------------------------------------------------------------- /db/20201022191322_create_metric.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | CREATE TABLE Metric( 4 | id SERIAL PRIMARY KEY, 5 | name VARCHAR(256) UNIQUE NOT NULL, 6 | type VARCHAR(256), 7 | created_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP 8 | ); 9 | -- +goose StatementEnd 10 | 11 | -- +goose Down 12 | -- +goose StatementBegin 13 | DROP TABLE Metric; 14 | -- +goose StatementEnd 15 | -------------------------------------------------------------------------------- /web/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /web/src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 4 | 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', 5 | sans-serif; 6 | -webkit-font-smoothing: antialiased; 7 | -moz-osx-font-smoothing: grayscale; 8 | } 9 | 10 | code { 11 | font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New', 12 | monospace; 13 | } 14 | -------------------------------------------------------------------------------- /docker/nginx.conf: -------------------------------------------------------------------------------- 1 | events { 2 | worker_connections 4096; ## Default: 1024 3 | } 4 | http { 5 | server { 6 | listen 80; 7 | listen [::]:80; 8 | 9 | server_name 10.x.x.x; 10 | 11 | location / { 12 | root /usr/share/nginx/html; 13 | index index.html index.htm; 14 | try_files $uri $uri/ /index.html; 15 | } 16 | location /api { 17 | proxy_pass http://localhost:5001/api; 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /db/migrate_db.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | HOST=${SCRUTINIZE_DB_HOST:-0.0.0.0} 3 | PORT=${SCRUTINIZE_DB_PORT:-5432} 4 | DB=${SCRUTINIZE_DB_NAME:-scrutinize} 5 | USER=${SCRUTINIZE_DB_USER:-postgres} 6 | PASSWORD=${SCRUTINIZE_DB_PASSWORD:-password} 7 | 8 | while ! nc -z ${HOST} ${PORT}; do 9 | echo 'Postgres is unavailable.' 10 | sleep 1 11 | done 12 | 13 | goose -dir db postgres "host=${HOST} port=${PORT} user=${USER} dbname=${DB} password=${PASSWORD} sslmode=disable" up 14 | -------------------------------------------------------------------------------- /db/20201022190232_create_experiment.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | CREATE TABLE Experiment( 4 | id SERIAL PRIMARY KEY, 5 | name VARCHAR(256) NOT NULL UNIQUE, 6 | description TEXT, 7 | active BOOLEAN NOT NULL DEFAULT FALSE, 8 | created_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP 9 | ); 10 | -- +goose StatementEnd 11 | 12 | -- +goose Down 13 | -- +goose StatementBegin 14 | DROP TABLE Experiment; 15 | -- +goose StatementEnd 16 | -------------------------------------------------------------------------------- /clients/javascript/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "allowUnreachableCode": false, 4 | "alwaysStrict": true, 5 | "declaration": true, 6 | "esModuleInterop": true, 7 | "forceConsistentCasingInFileNames": true, 8 | "module": "commonjs", 9 | "noImplicitReturns": true, 10 | "noUnusedLocals": true, 11 | "noUnusedParameters": true, 12 | "outDir": "./build", 13 | "rootDir": "./", 14 | "target": "es6", 15 | "strict": true 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /server/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "allowUnreachableCode": false, 4 | "alwaysStrict": true, 5 | "baseUrl": "./src", 6 | "esModuleInterop": true, 7 | "forceConsistentCasingInFileNames": true, 8 | "module": "commonjs", 9 | "noImplicitReturns": true, 10 | "noUnusedLocals": true, 11 | "noUnusedParameters": true, 12 | "outDir": "./build", 13 | "rootDir": "./", 14 | "target": "es6", 15 | "strict": true 16 | }, 17 | "include": ["src"] 18 | } 19 | -------------------------------------------------------------------------------- /web/src/reportWebVitals.ts: -------------------------------------------------------------------------------- 1 | import { ReportHandler } from 'web-vitals'; 2 | 3 | const reportWebVitals = (onPerfEntry?: ReportHandler) => { 4 | if (onPerfEntry && onPerfEntry instanceof Function) { 5 | import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => { 6 | getCLS(onPerfEntry); 7 | getFID(onPerfEntry); 8 | getFCP(onPerfEntry); 9 | getLCP(onPerfEntry); 10 | getTTFB(onPerfEntry); 11 | }); 12 | } 13 | } 14 | 15 | export default reportWebVitals; 16 | -------------------------------------------------------------------------------- /db/20201116112840_create_evaluation_criterion.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | CREATE TABLE EvaluationCriterion( 4 | run_id INT NOT NULL REFERENCES Run(id), 5 | metric_id INT NOT NULL REFERENCES Metric(id), 6 | weight NUMERIC NOT NULL DEFAULT 1.0, 7 | deleted_time TIMESTAMP, 8 | PRIMARY KEY(run_id, metric_id) 9 | ); 10 | -- +goose StatementEnd 11 | 12 | -- +goose Down 13 | -- +goose StatementBegin 14 | DROP TABLE EvaluationCriterion; 15 | -- +goose StatementEnd 16 | -------------------------------------------------------------------------------- /db/20201022190233_create_run.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | CREATE TABLE Run( 4 | id SERIAL PRIMARY KEY, 5 | percentage INT NOT NULL, 6 | started_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 7 | ended_time TIMESTAMP, 8 | experiment_id INT NOT NULL REFERENCES Experiment(id) 9 | ); 10 | 11 | CREATE INDEX run_experiment_started ON Run(experiment_id, started_time); 12 | -- +goose StatementEnd 13 | 14 | -- +goose Down 15 | -- +goose StatementBegin 16 | DROP TABLE Run; 17 | -- +goose StatementEnd 18 | -------------------------------------------------------------------------------- /db/20201022191341_create_measurement.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | CREATE TABLE Measurement( 4 | metric_name VARCHAR(256) REFERENCES Metric(name), 5 | value NUMERIC NOT NULL, 6 | user_id VARCHAR(256) NOT NULL, 7 | created_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP 8 | ); 9 | 10 | CREATE INDEX idx_measurement_metric_time_user ON Measurement(metric_name, created_time, user_id); 11 | -- +goose StatementEnd 12 | 13 | -- +goose Down 14 | -- +goose StatementBegin 15 | DROP TABLE Measurement; 16 | -- +goose StatementEnd 17 | -------------------------------------------------------------------------------- /web/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import './index.css'; 4 | import App from './App'; 5 | import reportWebVitals from './reportWebVitals'; 6 | 7 | ReactDOM.render( 8 | 9 | 10 | , 11 | document.getElementById('root') 12 | ); 13 | 14 | // If you want to start measuring performance in your app, pass a function 15 | // to log results (for example: reportWebVitals(console.log)) 16 | // or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals 17 | reportWebVitals(); 18 | -------------------------------------------------------------------------------- /server/src/controller/reporting.ts: -------------------------------------------------------------------------------- 1 | import ReportingStore from 'database/reporting'; 2 | import { 3 | Details, 4 | Performance, 5 | } from 'database/model'; 6 | 7 | export default class ReportingController { 8 | constructor ( 9 | private store: ReportingStore, 10 | ) {} 11 | 12 | public async getDetails(runID: number): Promise
{ 13 | return await this.store.getDetails(runID); 14 | } 15 | 16 | public async getPerformance(runID: number, metric: string): Promise { 17 | return await this.store.getPerformance(runID, metric); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.3" 2 | services: 3 | scrutinize: 4 | image: bdillmann/scrutinize:latest 5 | ports: 6 | - "5001:80" 7 | environment: 8 | SCRUTINIZE_DB_HOST: db 9 | depends_on: 10 | - db 11 | 12 | db: 13 | expose: 14 | - "5432" 15 | image: postgres:13.1-alpine 16 | environment: 17 | POSTGRES_DB: "scrutinize" 18 | POSTGRES_PASSWORD: "password" 19 | POSTGRES_HOST_AUTH_METHOD: "trust" 20 | 21 | migrador: 22 | image: bdillmann/scrutinize-migrador:latest 23 | environment: 24 | SCRUTINIZE_DB_HOST: db 25 | depends_on: 26 | - db 27 | -------------------------------------------------------------------------------- /db/20201022190756_create_treatment.sql: -------------------------------------------------------------------------------- 1 | -- +goose Up 2 | -- +goose StatementBegin 3 | CREATE TABLE Treatment( 4 | id SERIAL PRIMARY KEY, 5 | user_id VARCHAR(256) NOT NULL, 6 | variant VARCHAR(256) NOT NULL, 7 | error VARCHAR(256), 8 | duration_ms NUMERIC, 9 | created_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 10 | run_id INT NOT NULL REFERENCES Run(id) 11 | ); 12 | 13 | CREATE INDEX idx_treatment_run_user ON Treatment(run_id, user_id); 14 | -- +goose StatementEnd 15 | 16 | -- +goose Down 17 | -- +goose StatementBegin 18 | DROP TABLE Treatment; 19 | -- +goose StatementEnd 20 | -------------------------------------------------------------------------------- /web/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": [ 5 | "dom", 6 | "dom.iterable", 7 | "esnext" 8 | ], 9 | "allowJs": true, 10 | "skipLibCheck": true, 11 | "esModuleInterop": true, 12 | "allowSyntheticDefaultImports": true, 13 | "strict": true, 14 | "forceConsistentCasingInFileNames": true, 15 | "noFallthroughCasesInSwitch": true, 16 | "module": "esnext", 17 | "isolatedModules": true, 18 | "moduleResolution": "node", 19 | "resolveJsonModule": true, 20 | "noEmit": true, 21 | "jsx": "react", 22 | "baseUrl": "src" 23 | }, 24 | "include": [ 25 | "src" 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /server/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | parser: "@typescript-eslint/parser", // Specifies the ESLint parser 3 | parserOptions: { 4 | ecmaVersion: 2020, // Allows for the parsing of modern ECMAScript features 5 | sourceType: "module" // Allows for the use of imports 6 | }, 7 | extends: [ 8 | "plugin:@typescript-eslint/recommended" // Uses the recommended rules from the @typescript-eslint/eslint-plugin 9 | ], 10 | rules: { 11 | // Place to specify ESLint rules. Can be used to overwrite rules specified from the extended configs 12 | // e.g. "@typescript-eslint/explicit-function-return-type": "off" 13 | "@typescript-eslint/ban-ts-comment": "off", 14 | } 15 | }; 16 | -------------------------------------------------------------------------------- /web/src/App.css: -------------------------------------------------------------------------------- 1 | .App { 2 | text-align: center; 3 | } 4 | 5 | .App-logo { 6 | height: 40vmin; 7 | pointer-events: none; 8 | } 9 | 10 | @media (prefers-reduced-motion: no-preference) { 11 | .App-logo { 12 | animation: App-logo-spin infinite 20s linear; 13 | } 14 | } 15 | 16 | .App-header { 17 | background-color: #282c34; 18 | min-height: 100vh; 19 | display: flex; 20 | flex-direction: column; 21 | align-items: center; 22 | justify-content: center; 23 | font-size: calc(10px + 2vmin); 24 | color: white; 25 | } 26 | 27 | .App-link { 28 | color: #61dafb; 29 | } 30 | 31 | @keyframes App-logo-spin { 32 | from { 33 | transform: rotate(0deg); 34 | } 35 | to { 36 | transform: rotate(360deg); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /web/src/components/experiment/PercentageSlider.tsx: -------------------------------------------------------------------------------- 1 | import Slider from '@material-ui/core/Slider'; 2 | import { withStyles } from '@material-ui/core/styles'; 3 | 4 | export default withStyles({ 5 | root: { 6 | color: '#52af77', 7 | height: 8, 8 | }, 9 | thumb: { 10 | height: 24, 11 | width: 24, 12 | backgroundColor: '#fff', 13 | border: '2px solid currentColor', 14 | marginTop: -8, 15 | marginLeft: -12, 16 | '&:focus, &:hover, &$active': { 17 | boxShadow: 'inherit', 18 | }, 19 | }, 20 | active: {}, 21 | valueLabel: { 22 | left: 'calc(-50% + 4px)', 23 | }, 24 | track: { 25 | height: 8, 26 | borderRadius: 4, 27 | }, 28 | rail: { 29 | height: 8, 30 | borderRadius: 4, 31 | }, 32 | })(Slider); 33 | 34 | -------------------------------------------------------------------------------- /clients/python/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | setup( 7 | name='scrutinize', 8 | version='0.0.6', 9 | description='the lightweight experimentation platform', 10 | long_description=long_description, 11 | long_description_content_type="text/markdown", 12 | url='http://github.com/angles-n-daemons/scrutinize', 13 | author='brian dillmann', 14 | author_email='dillmann.brian@gmail.com', 15 | license='MIT', 16 | packages=['scrutinize'], 17 | install_requires=[ 18 | 'aiohttp', 19 | ], 20 | classifiers=[ 21 | "Programming Language :: Python :: 3", 22 | "License :: OSI Approved :: MIT License", 23 | "Operating System :: OS Independent", 24 | ], 25 | zip_safe=False, 26 | ) 27 | -------------------------------------------------------------------------------- /server/src/controller/metric.ts: -------------------------------------------------------------------------------- 1 | import MetricStore from 'database/metric'; 2 | import { 3 | Metric, 4 | Measurement, 5 | } from 'database/model'; 6 | 7 | export default class MetricController { 8 | constructor ( 9 | private store: MetricStore, 10 | ) {} 11 | 12 | public async createMeasurement(measurement: Measurement): Promise { 13 | const { metric_name } = measurement; 14 | await this.store.upsertMetric({ 15 | name: metric_name, 16 | }); 17 | await this.store.createMeasurement(measurement); 18 | } 19 | 20 | public async getMetrics(): Promise { 21 | return await this.store.getMetrics(); 22 | } 23 | 24 | public async createMetric(metric: Metric): Promise { 25 | await this.store.createMetric(metric); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /server/src/middleware/asyncRouter.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response, Router as ExpressRouter } from 'express'; 2 | 3 | // Async routing copied from: https://stackoverflow.com/a/57099213 4 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 5 | const asyncHandler = (fn: any) => (req: Request, res: Response, next: any) => { 6 | return Promise 7 | .resolve(fn(req, res, next)) 8 | .catch(next); 9 | } 10 | 11 | export default function toAsyncRouter(router: ExpressRouter) { 12 | const methods = [ 13 | 'get', 14 | 'post', 15 | 'delete' // & etc. 16 | ]; 17 | 18 | for (const key in router) { 19 | if (methods.includes(key)) { 20 | // @ts-ignore 21 | const method = router[key]; 22 | // @ts-ignore 23 | router[key] = (path, ...callbacks) => method.call(router, path, ...callbacks.map(cb => asyncHandler(cb))); 24 | } 25 | } 26 | return router 27 | } 28 | 29 | 30 | -------------------------------------------------------------------------------- /server/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "scrutinize-server", 3 | "version": "1.0.0", 4 | "description": "", 5 | "scripts": { 6 | "start": "nodemon src/index.ts", 7 | "build": "tsc --project ./", 8 | "lint": "eslint . --ext .js,.jsx,.ts,.tsx", 9 | "test": "echo \"Error: no test specified\" && exit 1" 10 | }, 11 | "keywords": [], 12 | "author": "", 13 | "license": "ISC", 14 | "dependencies": { 15 | "@types/express": "^4.17.8", 16 | "@types/pg": "^7.14.5", 17 | "chalk": "^4.1.0", 18 | "express": "^4.17.1", 19 | "pg": "^8.4.2", 20 | "ts-postgres": "^1.1.3", 21 | "typescript": "^4.0.5", 22 | "typescript-is": "^0.16.3" 23 | }, 24 | "devDependencies": { 25 | "@typescript-eslint/eslint-plugin": "^4.6.0", 26 | "@typescript-eslint/parser": "^4.6.0", 27 | "eslint": "^7.12.1", 28 | "nodemon": "^2.0.22", 29 | "ts-node": "^9.1.1" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /server/src/routes/health.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response, Router as ExpressRouter } from 'express'; 2 | import { Pool } from 'pg'; 3 | 4 | import toAsyncRouter from '../middleware/asyncRouter' 5 | 6 | export default class HealthRouter { 7 | constructor( 8 | private pool: Pool, 9 | ) {} 10 | 11 | public routes(): ExpressRouter { 12 | const router = toAsyncRouter(ExpressRouter()); 13 | router.get('/alive', this.alive.bind(this)); 14 | router.get('/health', this.health.bind(this)); 15 | return router; 16 | } 17 | 18 | private async alive(_: Request, res: Response) { 19 | res.json({status: 'ok'}); 20 | } 21 | 22 | private async health(_: Request, res: Response) { 23 | try { 24 | await this.pool.query(`SELECT id FROM Experiment LIMIT 1`); 25 | res.json({status: 'ok'}); 26 | } catch (e) { 27 | console.log(e); 28 | res.status(500).json({status: 'not ok'}); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /clients/javascript/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "scrutinize-client", 3 | "version": "0.0.6", 4 | "description": "client for scrutinize lightweight experimentation platform", 5 | "main": "build/scrutinize.js", 6 | "types": "build/scrutinize.d.ts", 7 | "scripts": { 8 | "test": "tsc && node build/scrutinize_test.js", 9 | "prepublish": "tsc" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "git+https://github.com/angles-n-daemons/scrutinize.git" 14 | }, 15 | "keywords": [ 16 | "ab-testing", 17 | "experimentation" 18 | ], 19 | "author": "brian dillmann", 20 | "license": "MIT", 21 | "bugs": { 22 | "url": "https://github.com/angles-n-daemons/scrutinize/issues" 23 | }, 24 | "homepage": "https://github.com/angles-n-daemons/scrutinize#readme", 25 | "devDependencies": { 26 | "@types/md5": "^2.2.1", 27 | "typescript": "^4.1.2" 28 | }, 29 | "dependencies": { 30 | "axios": "^0.21.0", 31 | "md5": "^2.3.0" 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /server/src/middleware/errors.ts: -------------------------------------------------------------------------------- 1 | /*eslint no-unused-vars: ["error", { "argsIgnorePattern": "^_" }]*/ 2 | import { Request, Response } from 'express'; 3 | 4 | export function UserError(err: Error | null = null, userMsg: string = ''): Error { 5 | if (err == null) { 6 | err = new Error(); 7 | } 8 | var newErr = new (err.constructor)(); 9 | newErr = Object.assign(newErr, err); 10 | newErr.userMsg = userMsg; 11 | return newErr; 12 | } 13 | 14 | export default function errorMiddleware(err: Error, _: Request, res: Response, next: Function): void { 15 | if (res.headersSent || !res.status){ next(err); } 16 | 17 | console.error(err.stack) 18 | console.error(err.message) 19 | var userMsg = 'Unable to handle request'; 20 | console.log(err); 21 | if ('userMsg' in err) { 22 | userMsg = err['userMsg']; 23 | } 24 | res.status(500).send({ 25 | 'error': `Unknown ${err.constructor.name} thrown`, 26 | 'userMsg': userMsg, 27 | }); 28 | } 29 | -------------------------------------------------------------------------------- /server/src/routes/reporting.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response, Router as ExpressRouter } from 'express'; 2 | 3 | import ReportingController from 'controller/reporting'; 4 | import toAsyncRouter from '../middleware/asyncRouter' 5 | 6 | export default class ReportingRouter { 7 | constructor( 8 | private controller: ReportingController, 9 | ) {} 10 | 11 | public routes(): ExpressRouter { 12 | const router = toAsyncRouter(ExpressRouter()); 13 | router.get('/details/:run_id', this.getDetails.bind(this)); 14 | router.get('/performance/:run_id/:metric', this.getPerformance.bind(this)); 15 | return router; 16 | } 17 | 18 | private async getDetails(req: Request, res: Response) { 19 | res.json(await this.controller.getDetails(parseInt(req.params.run_id || '0'))); 20 | } 21 | 22 | private async getPerformance(req: Request, res: Response) { 23 | res.json(await this.controller.getPerformance( 24 | parseInt(req.params.run_id || '0'), 25 | req.params.metric || '', 26 | )); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /server/src/routes/metric.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response, Router as ExpressRouter } from 'express'; 2 | 3 | import MetricController from 'controller/metric'; 4 | import toAsyncRouter from '../middleware/asyncRouter' 5 | 6 | export default class MetricRouter { 7 | constructor( 8 | private controller: MetricController, 9 | ) {} 10 | 11 | public routes(): ExpressRouter { 12 | const router = toAsyncRouter(ExpressRouter()); 13 | router.post('/measurement', this.postMeasurement.bind(this)); 14 | router.get('/metric', this.getMetrics.bind(this)); 15 | router.post('/metric', this.postMetric.bind(this)); 16 | return router; 17 | } 18 | 19 | private async postMeasurement(req: Request, res: Response) { 20 | await this.controller.createMeasurement(req.body); 21 | res.json({status: 'ok'}); 22 | } 23 | 24 | private async getMetrics(_: Request, res: Response) { 25 | res.json(await this.controller.getMetrics()); 26 | } 27 | 28 | private async postMetric(req: Request, res: Response) { 29 | await this.controller.createMetric(req.body); 30 | res.json({status: 'ok'}); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /clients/javascript/README.md: -------------------------------------------------------------------------------- 1 | # scrutinize-client 2 | 3 | This is the javascript client for the [scrutinize experimentation platform](https://github.com/angles-n-daemons/scrutinize). 4 | 5 | # Installation 6 | 7 | To install the client, use npm: 8 | 9 | `npm i scrutinize-client` 10 | 11 | ## Usage 12 | 13 | Once installed, you can publish metrics and conduct experiments using the client API. 14 | 15 | __Publishing Metrics__ 16 | 17 | ```javascript 18 | import ScrutinizeClient from 'scrutinize'; 19 | 20 | const scrutinize = ScrutinizeClient('https://scrutinize-location'); 21 | await scrutinize.observe( 22 | 'wilma_rudolph', 23 | 'purchased_coffee', 24 | True, 25 | ) 26 | ``` 27 | 28 | __Running an experiment__ 29 | 30 | ```javascript 31 | import ScrutinizeClient from 'scrutinize'; 32 | import canUserHaveFreeCoffee from 'my_helper_lib'; 33 | 34 | const scrutinize = ScrutinizeClient('https://scrutinize-location'); 35 | const [isExperiment, gaveFreeCoffee] = await scrutinize.call( 36 | 'eng.give_user_free_coffee', 37 | 'wilma_rudolph', 38 | False, 39 | lambda: canUserHaveFreeCoffee('wilma_rudolph'), 40 | ) 41 | ``` 42 | -------------------------------------------------------------------------------- /clients/python/README.md: -------------------------------------------------------------------------------- 1 | # scrutinize 2 | 3 | This is the python client for the [scrutinize experimentation platform](https://github.com/angles-n-daemons/scrutinize). 4 | 5 | # Installation 6 | 7 | To install the client, use pip: 8 | 9 | `pip install scrutinize` 10 | 11 | ## Usage 12 | 13 | Once installed, you can publish metrics and conduct experiments using the client API. 14 | 15 | __Publishing Metrics__ 16 | 17 | ```python 18 | from scrutinize import ScrutinizeClient 19 | 20 | scrutinize = ScrutinizeClient('https://scrutinize-location') 21 | await scrutinize.observe( 22 | user_id='wilma_rudolph', 23 | metric='purchased_coffee', 24 | value=True, 25 | ) 26 | ``` 27 | 28 | __Running an experiment__ 29 | 30 | ```python 31 | from scrutinize import ScrutinizeClient 32 | from my_library import can_user_have_free_coffee 33 | 34 | scrutinize = ScrutinizeClient('https://scrutinize-location') 35 | give_free_coffee = await scrutinize.call( 36 | experiment_name='eng.give_user_free_coffee', 37 | user_id='wilma_rudolph', 38 | control=False, 39 | experiment=lambda: can_user_have_free_coffee('wilma_rudolph'), 40 | ) 41 | ``` 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Brian Dillmann 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /clients/python/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018 The Python Packaging Authority 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /web/src/components/performance/Controls.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | import Input from '@material-ui/core/Input'; 3 | import MenuItem from '@material-ui/core/MenuItem'; 4 | import ListItemText from '@material-ui/core/ListItemText'; 5 | import Select from '@material-ui/core/Select'; 6 | import Checkbox from '@material-ui/core/Checkbox'; 7 | 8 | const controlsOptions = [ 9 | 'Customer Purchase', 10 | 'Load Time (ms)', 11 | 'Customer Satisfaction', 12 | ]; 13 | 14 | export default function ExplorerControls() { 15 | const [controls, setControls] = useState([]); 16 | 17 | const handleChange = (event: any) => { 18 | setControls(event.target.value); 19 | }; 20 | 21 | return ( 22 | 35 | ); 36 | } 37 | -------------------------------------------------------------------------------- /server/src/config.ts: -------------------------------------------------------------------------------- 1 | import { Configuration } from 'ts-postgres'; 2 | 3 | const ENV_BASE='SCRUTINIZE'; 4 | 5 | export interface ProcessEnv { 6 | [key: string]: string | undefined 7 | } 8 | 9 | export default class Config { 10 | constructor( 11 | public port: number, 12 | public db_host: string, 13 | public db_port: number, 14 | public db_name: string, 15 | public db_user: string, 16 | public db_password: string, 17 | ) {} 18 | 19 | public dbOptions(): Configuration { 20 | return { 21 | host: this.db_host, 22 | port: this.db_port, 23 | database: this.db_name, 24 | user: this.db_user, 25 | password: this.db_password, 26 | } 27 | } 28 | 29 | static readFromEnvironment(env: ProcessEnv): Config { 30 | return new Config( 31 | parseInt(env[`${ENV_BASE}_PORT`] || '11771'), 32 | env[`${ENV_BASE}_DB_HOST`] || '0.0.0.0', 33 | parseInt(env[`${ENV_BASE}_DB_PORT`] || '5432'), 34 | env[`${ENV_BASE}_DB_NAME`] || 'scrutinize', 35 | env[`${ENV_BASE}_DB_USER`] || 'postgres', 36 | env[`${ENV_BASE}_DB_PASSWORD`] || 'password', 37 | ); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /docker/Dockerfile.service: -------------------------------------------------------------------------------- 1 | # ---------- Base ---------- 2 | FROM node:15-alpine AS base 3 | 4 | WORKDIR /app 5 | 6 | # ---------- build-web ---------- 7 | # Creates: 8 | # - dist: A production build compiled with Babel 9 | FROM base AS build-web 10 | 11 | COPY ./web/package*.json ./ 12 | 13 | RUN npm install 14 | 15 | COPY ./web/tsconfig.json ./ 16 | 17 | COPY ./web/public ./public 18 | 19 | COPY ./web/src ./src 20 | 21 | RUN npm run build 22 | 23 | RUN npm prune --production # Remove dev dependencies 24 | 25 | # ---------- build-server ---------- 26 | # Creates: 27 | # - build: A production build compiled with Babel 28 | FROM base AS build-server 29 | 30 | ENV NODE_ENV=development 31 | 32 | COPY ./server/package*.json ./ 33 | 34 | COPY ./server/tsconfig.json ./ 35 | 36 | COPY ./server/src ./src 37 | 38 | RUN npm install typescript -g 39 | 40 | RUN npm install 41 | 42 | ENV PATH /app/node_modules/.bin:$PATH 43 | 44 | RUN npm run build 45 | 46 | RUN npm prune --production # Remove dev dependencies 47 | 48 | # ---------- Release ---------- 49 | # Stage 2 50 | FROM base 51 | 52 | RUN apk add bash nginx vim 53 | RUN mkdir /run/nginx 54 | 55 | COPY --from=build-web /app/build /usr/share/nginx/html 56 | COPY --from=build-server /app/build ./build 57 | COPY --from=build-server /app/node_modules ./node_modules 58 | 59 | COPY ./docker/run.sh ./ 60 | COPY ./docker/nginx.conf /etc/nginx/nginx.conf 61 | 62 | EXPOSE 80 63 | 64 | CMD ["./run.sh"] 65 | -------------------------------------------------------------------------------- /web/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "scrutinize-web", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "@material-ui/core": "^4.11.0", 7 | "@material-ui/icons": "^4.9.1", 8 | "@material-ui/lab": "^4.0.0-alpha.56", 9 | "@testing-library/jest-dom": "^5.11.5", 10 | "@testing-library/react": "^11.1.0", 11 | "@testing-library/user-event": "^12.1.10", 12 | "@types/jest": "^26.0.15", 13 | "@types/node": "^12.19.1", 14 | "@types/react": "^16.9.53", 15 | "@types/react-dom": "^16.9.8", 16 | "@types/react-router-dom": "^5.1.6", 17 | "dom-helpers": "^5.2.0", 18 | "material-ui": "^0.20.2", 19 | "react": "^17.0.1", 20 | "react-dom": "^17.0.1", 21 | "react-google-charts": "^3.0.15", 22 | "react-router-dom": "^5.2.0", 23 | "react-scripts": "4.0.0", 24 | "typescript": "^4.0.3", 25 | "web-vitals": "^0.2.4" 26 | }, 27 | "scripts": { 28 | "start": "PORT=11206 react-scripts start", 29 | "build": "react-scripts build", 30 | "test": "react-scripts test", 31 | "eject": "react-scripts eject" 32 | }, 33 | "eslintConfig": { 34 | "extends": [ 35 | "react-app", 36 | "react-app/jest" 37 | ] 38 | }, 39 | "browserslist": { 40 | "production": [ 41 | ">0.2%", 42 | "not dead", 43 | "not op_mini all" 44 | ], 45 | "development": [ 46 | "last 1 chrome version", 47 | "last 1 firefox version", 48 | "last 1 safari version" 49 | ] 50 | }, 51 | "proxy": "http://localhost:11771" 52 | } 53 | -------------------------------------------------------------------------------- /server/src/routes/experiment.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response, Router as ExpressRouter } from 'express'; 2 | 3 | import ExperimentController from 'controller/experiment'; 4 | import toAsyncRouter from '../middleware/asyncRouter' 5 | 6 | export default class ExperimentRouter { 7 | constructor( 8 | private controller: ExperimentController, 9 | ) {} 10 | 11 | public routes(): ExpressRouter { 12 | const router = toAsyncRouter(ExpressRouter()); 13 | router.get('/experiment', this.getExperiment.bind(this)); 14 | router.post('/experiment', this.postExperiment.bind(this)); 15 | router.post('/experiment/start', this.postExperimentStart.bind(this)); 16 | router.post('/experiment/end', this.postExperimentEnd.bind(this)); 17 | router.post('/treatment', this.postTreatment.bind(this)); 18 | return router; 19 | } 20 | 21 | private async getExperiment(_: Request, res: Response) { 22 | res.json(await this.controller.getExperiments()); 23 | } 24 | 25 | private async postExperiment(req: Request, res: Response) { 26 | await this.controller.createExperiment(req.body); 27 | res.json({status: 'ok'}); 28 | } 29 | 30 | private async postExperimentStart(req: Request, res: Response) { 31 | await this.controller.startExperiment(req.body); 32 | res.json({status: 'ok'}); 33 | } 34 | 35 | private async postExperimentEnd(req: Request, res: Response) { 36 | await this.controller.endExperiment(req.body); 37 | res.json({status: 'ok'}); 38 | } 39 | 40 | private async postTreatment(req: Request, res: Response) { 41 | await this.controller.createTreatment(req.body); 42 | res.json({status: 'ok'}); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /server/src/database/model.ts: -------------------------------------------------------------------------------- 1 | export interface Experiment { 2 | id?: number; 3 | name: string; 4 | description: string; 5 | active: boolean; 6 | percentage?: number; 7 | run_id?: number; 8 | created_time?: Date; 9 | } 10 | 11 | export interface Run { 12 | id?: number 13 | experiment_id: number; 14 | percentage: number; 15 | metrics?: Metric[]; 16 | started_time?: Date; 17 | ended_time?: Date; 18 | } 19 | 20 | export interface Treatment { 21 | user_id: string; 22 | run_id: number; 23 | variant: 'control' | 'experiment'; 24 | error: string; 25 | duration_ms: number; 26 | } 27 | 28 | export interface Measurement { 29 | user_id: string; 30 | metric_name: string; 31 | value: number; 32 | created_time?: string; 33 | } 34 | 35 | export interface Metric { 36 | id?: number; 37 | name: string; 38 | type?: 'binomial' | 'continuous' | 'count'; 39 | } 40 | 41 | export interface Performance { 42 | control: DataPoint[]; 43 | experiment: DataPoint[]; 44 | } 45 | 46 | export interface DataPoint { 47 | metric_name: string; 48 | variant: 'control' | 'experiment'; 49 | date: string; 50 | count: number; 51 | avg: number; 52 | stddev: number; 53 | } 54 | 55 | export interface VariantDetails { 56 | variant: string; 57 | volume: string; 58 | pct_error: string; 59 | duration_ms: string; 60 | } 61 | 62 | export interface Details { 63 | name: string; 64 | percentage: number; 65 | created_time?: Date; 66 | last_active_time?: Date; 67 | variants: VariantDetails[]; 68 | evaluation_criterion: Metric[]; 69 | } 70 | -------------------------------------------------------------------------------- /server/src/controller/experiment.ts: -------------------------------------------------------------------------------- 1 | import ExperimentStore from 'database/experiment'; 2 | import { 3 | Experiment, 4 | Run, 5 | Treatment, 6 | } from 'database/model'; 7 | 8 | import { UserError } from '../middleware/errors'; 9 | 10 | export default class ExperimentController { 11 | constructor ( 12 | private store: ExperimentStore, 13 | ) {} 14 | 15 | public async getExperiments(): Promise { 16 | return await this.store.getExperiments(); 17 | } 18 | 19 | public async createExperiment(experiment: Experiment): Promise { 20 | await this.store.createExperiment(experiment); 21 | } 22 | 23 | public async startExperiment(run: Run): Promise { 24 | const experiment = await this.store.getExperiment(run.experiment_id); 25 | 26 | if (!experiment) { 27 | throw(UserError(null, `Unable to find experiment with id ${run.experiment_id}`)); 28 | } 29 | 30 | if (experiment.active) { 31 | throw(UserError(null, 'Experiment already running')); 32 | } 33 | 34 | await this.store.startExperiment(run); 35 | } 36 | 37 | public async endExperiment({ id }: Experiment): Promise { 38 | const experiment = await this.store.getExperiment(id as number); 39 | 40 | if (!experiment) { 41 | throw(UserError(null, `Unable to find experiment with id ${id}`)); 42 | } 43 | if (!experiment.active) { 44 | throw(UserError(null, 'Experiment not running')); 45 | } 46 | 47 | await this.store.endExperiment(id as number); 48 | } 49 | 50 | public async createTreatment(treatment: Treatment): Promise { 51 | if (!treatment.run_id) { 52 | return 53 | } 54 | await this.store.createTreatment(treatment); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /web/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 16 | 17 | 26 | scrutinize 27 | 28 | 29 | 30 |
31 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /web/src/components/Performance.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from 'react'; 2 | 3 | import { makeStyles } from '@material-ui/core/styles'; 4 | import PerformanceInfoBar from 'components/performance/InfoBar'; 5 | import PerformanceChart from 'components/performance/Chart'; 6 | 7 | import API, { ExperimentDetails, Metric } from 'api/api' 8 | 9 | const useStyles = makeStyles({ 10 | root: { 11 | display: 'flex', 12 | paddingRight: '8%', 13 | paddingLeft: '4%', 14 | flexDirection: 'column', 15 | }, 16 | }); 17 | 18 | export default function PerformancePage() { 19 | const classes = useStyles(); 20 | const params = new URLSearchParams(window.location.search); 21 | const runID = params.get('run_id') || ''; 22 | 23 | const [details, setDetails] = useState({ 24 | name: ' ', 25 | percentage: 0, 26 | active: false, 27 | created_time: '1970-01-01', 28 | last_active_time: '1970-01-01', 29 | variants: [], 30 | evaluation_criterion: [], 31 | }); 32 | 33 | useEffect(() => { 34 | async function getDetails() { 35 | setDetails(await API.getDetails(runID)); 36 | } 37 | getDetails(); 38 | }, [runID]); 39 | 40 | 41 | if (runID) { 42 | return (
43 | 44 | {details.evaluation_criterion.map((metric: Metric) => { 45 | const performanceChartProps = { 46 | runID: runID, 47 | metric: metric.name, 48 | }; 49 | return (
50 | {} 51 |
) 52 | })} 53 |
); 54 | } else { 55 | return (
No experiment selected
) 56 | } 57 | }; 58 | -------------------------------------------------------------------------------- /web/src/components/experiment/MetricSelect.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from 'react'; 2 | import Chip from '@material-ui/core/Chip'; 3 | import TextField from '@material-ui/core/TextField'; 4 | import Autocomplete from '@material-ui/lab/Autocomplete'; 5 | 6 | import API from 'api/api' 7 | 8 | interface MetricSelectProps { 9 | setMetrics: any; 10 | } 11 | 12 | interface AutocompleteOption { 13 | title: string; 14 | id: number; 15 | } 16 | 17 | export default function MetricSelect({ 18 | setMetrics 19 | }: MetricSelectProps) { 20 | const [options, setOptions] = useState([]); 21 | 22 | useEffect(() => { 23 | async function getMetrics() { 24 | const apiMetrics = await API.getMetrics(); 25 | setOptions(apiMetrics.map((metric) => { 26 | return { 27 | id: metric.id, 28 | title: metric.name, 29 | }; 30 | })); 31 | } 32 | getMetrics(); 33 | }, []); 34 | 35 | function handleChangeMetrics(_: any, values: any) { 36 | setMetrics(values.map(({ id, title }: AutocompleteOption) => { return { id, name: title } })); 37 | } 38 | 39 | return ( 40 | option.title} 46 | renderTags={(tagValue, getTagProps) => 47 | tagValue.map((option, index) => ( 48 | 52 | )) 53 | } 54 | style={{ width: '100%' }} 55 | renderInput={(params) => ( 56 | 57 | )} 58 | /> 59 | ); 60 | } 61 | -------------------------------------------------------------------------------- /server/src/database/metric.ts: -------------------------------------------------------------------------------- 1 | import { Pool } from 'pg'; 2 | 3 | import { 4 | Metric, 5 | Measurement, 6 | } from 'database/model'; 7 | 8 | import { UserError } from '../middleware/errors'; 9 | 10 | export default class MetricStore { 11 | constructor ( 12 | private pool: Pool, 13 | ) {} 14 | 15 | public async createMetric(metric: Metric): Promise { 16 | const { name, type } = metric; 17 | try { 18 | await this.pool.query( 19 | ` 20 | INSERT INTO Metric (name, type) 21 | VALUES ($1, $2) 22 | `, 23 | [name, type], 24 | ); 25 | } catch (e: any) { 26 | if (e instanceof Error) { 27 | if (e.message.indexOf('unique constraint')) { 28 | e = UserError(e, 'Metric name taken, please choose a different one'); 29 | } 30 | } 31 | throw(e); 32 | } 33 | } 34 | 35 | public async upsertMetric(metric: Metric): Promise { 36 | const { name } = metric; 37 | await this.pool.query( 38 | ` 39 | INSERT INTO Metric (name) 40 | VALUES ($1) 41 | ON CONFLICT DO NOTHING 42 | `, 43 | [name], 44 | ); 45 | } 46 | 47 | public async createMeasurement(measurement: Measurement): Promise { 48 | const { metric_name, value, user_id, created_time } = measurement; 49 | await this.pool.query( 50 | ` 51 | INSERT INTO Measurement( 52 | metric_name, 53 | value, 54 | user_id, 55 | created_time 56 | ) VALUES ($1, $2, $3, $4) 57 | `, 58 | [metric_name, value.toString(), user_id, created_time || new Date()], 59 | ); 60 | } 61 | 62 | public async getMetrics(): Promise { 63 | return (await this.pool.query( 64 | `SELECT id, name, type FROM Metric`, 65 | )).rows; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /docs/quickstart/DEVELOPMENT.md: -------------------------------------------------------------------------------- 1 | # Development 2 | 3 | This document details what you would need to get scrutinize running locally so that you may make changes to it. It assumes that you're able to run the web app, the server and the database on your machine so that you can make changes accordingly. 4 | 5 | If you aim to use a scrutinize client, you should be using it in another project simultaneously. 6 | 7 | ### Requirements 8 | 9 | | Requirement | Version | 10 | | ------------- | ------------- | 11 | | node | 15.0+ | 12 | | postgresql | 14.0+ | 13 | | goose | 2.7.0+ 14 | 15 | ### Setup the database 16 | 17 | Assuming you have postgresql installed, you need to create a database named `scrutinize` with the owner `postgres`. 18 | 19 | By default the application assumes you have no password for the database - if you need this or any other settings changed, you can supply them via environment variables specified in the [server's configuration file](/server/src/config.ts). 20 | 21 | You can then run the migration script to bring the schema up to date. This script uses the same env variable names as the server's config file. 22 | 23 | ```bash 24 | ./db/migrate_db.sh 25 | ``` 26 | 27 | ### Running the scrutinize server 28 | 29 | Once the database is setup, you should be able to run the server. Install the depdendencies 30 | 31 | ```bash 32 | npm i 33 | ``` 34 | 35 | and run the server 36 | 37 | ```bash 38 | npm start 39 | ``` 40 | 41 | ### Running the scrutninize web app 42 | 43 | Do the same for the web application. Install the dependencies 44 | 45 | ```bash 46 | npm i 47 | ``` 48 | 49 | and run the server 50 | 51 | ```bash 52 | npm start 53 | ``` 54 | 55 | Verify that the system is working by navigating to the web app in your browser. 56 | 57 | 58 | ```bash 59 | open http://localhost:11206 60 | ``` 61 | 62 | ### Seeding the database with mock data 63 | 64 | TBD 65 | 66 | ### Creating database migrations 67 | 68 | We use goose to create migrations. You can learn more about it [here](https://github.com/pressly/goose). Migrations should be sql only, and in the `db` directory. 69 | -------------------------------------------------------------------------------- /clients/python/scrutinize/seed.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import random 3 | import time 4 | from datetime import datetime, timedelta 5 | 6 | from scrutinize import ScrutinizeClient 7 | 8 | def mock_func(experiment: bool, sleep_ms: int, err_pct: float): 9 | def my_func(): 10 | sleep_seconds = (random.random() * sleep_ms) / 1000 11 | time.sleep(sleep_seconds) 12 | if random.random() * 100 < err_pct: 13 | raise Exception('haha mang') 14 | return experiment 15 | 16 | return my_func 17 | 18 | async def run_seed(scrutinize: ScrutinizeClient): 19 | current_time = datetime.now() - timedelta(days=15) 20 | 21 | while str(current_time.date()) < str(datetime.now().date()): 22 | print('seeding date: ', str(current_time.date())) 23 | 24 | for _ in range(100 + int(random.random() * 1000)): #00 + int(random.random() * 1500)): 25 | id = ''.join(random.sample('abcdefghijklmnopqrstuvwxyz', 3)) 26 | checkout_base = random.choice([5, 20, 100]) 27 | checkout_amount = checkout_base + (10 * random.random()) 28 | if str(current_time.date()) > '2020-10-12': 29 | checkout_amount += 10 30 | 31 | try: 32 | is_experiment, _ = await scrutinize.call( 33 | experiment_name='appeng.new_checkout_experience', 34 | user_id=id, 35 | control=mock_func(False, 20, 2), 36 | experiment=mock_func(True, 45, 3), 37 | ) 38 | except Exception as e: 39 | print(e) 40 | continue 41 | 42 | if is_experiment: 43 | checkout_amount += 5 + (10 * random.random()) 44 | 45 | await scrutinize.observe( 46 | user_id=id, 47 | metric='checkout_amount', 48 | value=checkout_amount, 49 | created_time=str(current_time.date()), 50 | ) 51 | 52 | current_time += timedelta(days=1) 53 | 54 | 55 | if __name__ == '__main__': 56 | scrutinize = ScrutinizeClient() 57 | loop = asyncio.get_event_loop() 58 | loop.run_until_complete(run_seed(scrutinize)) 59 | 60 | -------------------------------------------------------------------------------- /server/src/middleware/logger.ts: -------------------------------------------------------------------------------- 1 | import { Request, Response } from 'express'; 2 | import chalk from 'chalk'; 3 | 4 | const getDurationInMilliseconds = (start: [number, number]) => { 5 | const NS_PER_SEC = 1e9; // convert to nanoseconds 6 | const NS_TO_MS = 1e6; // convert to milliseconds 7 | const diff = process.hrtime(start); 8 | return (diff[0] * NS_PER_SEC + diff[1]) / NS_TO_MS; 9 | }; 10 | 11 | function getRequestLog( 12 | method: string, 13 | url: string, 14 | formatted_date: string, 15 | status: number, 16 | start: [number, number], 17 | ) { 18 | const durationInMilliseconds = getDurationInMilliseconds(start) 19 | const statusStr = status > 380 ? chalk.redBright(status) : chalk.greenBright(status); 20 | const durationStr = durationInMilliseconds > 1000 ? chalk.redBright(durationInMilliseconds) : chalk.greenBright(durationInMilliseconds); 21 | return `[${chalk.cyan(formatted_date)}] ${method} ${url} ${statusStr} ${durationStr}`; 22 | } 23 | 24 | export default function logMiddleware(req: Request, res: Response, next: () => void): void { 25 | const current_datetime = new Date(); 26 | const formatted_date = 27 | current_datetime.getFullYear() + 28 | "-" + 29 | (current_datetime.getMonth() + 1) + 30 | "-" + 31 | current_datetime.getDate() + 32 | " " + 33 | current_datetime.getHours() + 34 | ":" + 35 | current_datetime.getMinutes() + 36 | ":" + 37 | current_datetime.getSeconds(); 38 | const method = req.method; 39 | const url = req.url; 40 | const start = process.hrtime(); 41 | let logged = false; 42 | 43 | res.on('finish', () => { 44 | if (!logged) { 45 | const status = res.statusCode; 46 | console.log(getRequestLog(method, url, formatted_date, status, start)); 47 | logged = true; 48 | } 49 | }) 50 | 51 | res.on('close', () => { 52 | if (!logged) { 53 | const status = res.statusCode; 54 | console.log(getRequestLog(method, url, formatted_date, status, start)); 55 | logged = true; 56 | } 57 | }) 58 | 59 | next(); 60 | } 61 | -------------------------------------------------------------------------------- /server/src/index.ts: -------------------------------------------------------------------------------- 1 | import express from 'express'; 2 | import { Pool } from 'pg'; 3 | 4 | import Config from './config'; 5 | import ExperimentController from './controller/experiment'; 6 | import ExperimentRouter from './routes/experiment'; 7 | import ExperimentStore from './database/experiment'; 8 | import MetricController from './controller/metric'; 9 | import MetricRouter from './routes/metric'; 10 | import MetricStore from './database/metric'; 11 | import ReportingController from './controller/reporting'; 12 | import ReportingRouter from './routes/reporting'; 13 | import ReportingStore from './database/reporting'; 14 | import HealthRouter from './routes/health'; 15 | import logMiddleware from './middleware/logger'; 16 | import errorMiddleware from './middleware/errors'; 17 | 18 | const BASE_PATH = '/api/v1' 19 | 20 | async function main() { 21 | // Setup service components 22 | const config = Config.readFromEnvironment(process.env); 23 | const pool = new Pool(config.dbOptions()); 24 | const healthRouter = new HealthRouter(pool); 25 | const experimentRouter = new ExperimentRouter( 26 | new ExperimentController( 27 | new ExperimentStore(pool), 28 | ), 29 | ); 30 | const metricRouter = new MetricRouter( 31 | new MetricController( 32 | new MetricStore(pool), 33 | ), 34 | ); 35 | const reportingRouter = new ReportingRouter( 36 | new ReportingController( 37 | new ReportingStore(pool), 38 | ), 39 | ); 40 | 41 | // Setup application 42 | const app = express(); 43 | app.use(express.json()); 44 | app.use(logMiddleware); 45 | app.use(BASE_PATH, healthRouter.routes()); 46 | app.use(BASE_PATH, experimentRouter.routes()); 47 | app.use(BASE_PATH, metricRouter.routes()); 48 | app.use(BASE_PATH, reportingRouter.routes()); 49 | app.use(errorMiddleware); 50 | 51 | // Begin serving requests 52 | await pool.connect(); 53 | await app.listen(config.port, () => { 54 | console.log(`⚡️[server]: Server is running at https://localhost:${config.port}`); 55 | }).on('error', (err) => { 56 | pool.end(); 57 | throw(err); 58 | }); 59 | } 60 | 61 | main(); 62 | -------------------------------------------------------------------------------- /clients/javascript/seed.ts: -------------------------------------------------------------------------------- 1 | import ScrutinizeClient from './scrutinize'; 2 | 3 | function mockVariant(isExperiment: boolean, sleepMS: number, errPct: number) { 4 | return async () => { 5 | await new Promise(r => setTimeout(r, sleepMS)); 6 | if (Math.random() * 100 < errPct) { 7 | throw Error('oh boy'); 8 | } 9 | return isExperiment; 10 | }; 11 | } 12 | 13 | async function runSeed(scrutinize: ScrutinizeClient) { 14 | const bumpDate = new Date(); 15 | bumpDate.setDate(bumpDate.getDate()-6); 16 | 17 | for (var i = 20; i > 0; i--) { 18 | const currentDate = new Date(); 19 | currentDate.setDate(currentDate.getDate()-i); 20 | const dateStr = currentDate.toISOString().split('T')[0]; 21 | const numInteractions = 100 + Math.floor(Math.random() * 1000); 22 | console.log('publishing data for ', dateStr); 23 | 24 | for (var j = 0; j < numInteractions; j++) { 25 | var csat = [2, 3][Math.floor(Math.random() * 2)]; 26 | csat += (Math.random() * .5); 27 | 28 | if (currentDate > bumpDate) { 29 | csat += Math.random() * 1; 30 | } 31 | 32 | var isExperiment = false 33 | var _ = 'meh'; 34 | if (5 < 3) { 35 | console.log(_); 36 | } 37 | const id = Math.random().toString(36).substring(2, 5); 38 | try { 39 | [isExperiment, _] = await scrutinize.call( 40 | 'product_eng.give_extension', 41 | id, 42 | mockVariant('hi' as unknown as boolean, 20, 2), 43 | mockVariant('bye' as unknown as boolean, 45, 4), 44 | ); 45 | } catch (e: any) { 46 | console.log(e); 47 | continue; 48 | } 49 | 50 | if (isExperiment) { 51 | csat += Math.random() * 1; 52 | } 53 | 54 | await scrutinize.observe( 55 | id, 56 | 'csat_score', 57 | csat, 58 | dateStr, 59 | ); 60 | } 61 | } 62 | } 63 | 64 | runSeed(new ScrutinizeClient()); 65 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # scrutinize 2 | 3 | scrutinize is a lightweight experimentation framework focused on conducting and analysing online experiments (a/b tests). Some of its goals are as follows: 4 | 5 | - Simple local or remote deployment. 6 | - Pre-packaged clients for easy usage in multiple languages. 7 | - Straightforward experimentation practices. 8 | - Built-in metrics tracking and analysis. 9 | 10 | ![Analysing Experiment Performance](docs/readme/reporting.png) 11 | 12 | # Motiviation 13 | 14 | Currently the ecosystem for A/B testing tools consists of either costly SaaS providers, or incomplete open source solutions. For smaller companies that cannot afford an enterprise offering, options are limited to discontinued projects, fragmented deployments or building a platform in house. 15 | 16 | scrutinize aims to be an all in one package for designing, conducting, monitoring and analysing controlled online experiments at reasonable scale. It is a server + suite of clients that works well with the currently popular microservice ecosystem. 17 | 18 | # Getting Started 19 | 20 | Consult the [Quickstart Guide](docs/quickstart/QUICKSTART.md) for instructions on deploying the scrutinize service and conducting an experiment via a client. 21 | 22 | # Contributing 23 | 24 | Feel free to open an issue, PR or simply have a discussion about modifications to this project. In the weeks to come we'll try to flesh out a more comprehensive set of contribution guidelines. 25 | 26 | # Inspiration 27 | 28 | Scrutinize was inspired by and draws elements from the following projects: 29 | 30 | - [Unleash](https://github.com/Unleash/unleash) _system & client design_ 31 | - [Uber XP](https://eng.uber.com/xp/) _analysis system_ 32 | - [Wasabi](https://github.com/intuit/wasabi) _api design_ 33 | 34 | The following book informed a lot of the design choices in this project: 35 | 36 | [Trustworth Online Controlled Experiments: A Practical Guide to A/B Testing](https://books.google.com/books/about/Trustworthy_Online_Controlled_Experiment.html?id=bJY1yAEACAAJ) 37 | 38 | # Roadmap 39 | 40 | Some next high level goals for this project are as follows: 41 | 42 | - Calculating p-values on the evaluation metrics to better inform experiment success 43 | - Bulk upload of observation metrics via api 44 | - Client consistency with respect to experiment toggles (active killswitch) 45 | - Automated power estimation and recommendation via 1-7 day A/A dry run 46 | - Caching performance results to ensure scalibility in analysis 47 | - Population selection for experiment iterations 48 | 49 | # Credits 50 | 51 | Special thanks go out to Lisa Jiang and Punya Biswal for their feedback on both system and api design on this project. 52 | -------------------------------------------------------------------------------- /web/src/components/experiment/EndExperimentForm.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | import Button from '@material-ui/core/Button'; 3 | import Dialog from '@material-ui/core/Dialog'; 4 | import DialogActions from '@material-ui/core/DialogActions'; 5 | import DialogTitle from '@material-ui/core/DialogTitle'; 6 | import Typography from '@material-ui/core/Typography'; 7 | import { makeStyles } from '@material-ui/core/styles'; 8 | 9 | import API from 'api/api'; 10 | 11 | const useStyles = makeStyles((theme) => ({ 12 | dialog: { 13 | overflowY: 'visible', 14 | }, 15 | submit: { 16 | margin: '8px 0px 12px', 17 | }, 18 | errorField: { 19 | width: '100%', 20 | textAlign: 'center', 21 | }, 22 | dialogActions: { 23 | display: 'block', 24 | textAlign: 'center', 25 | }, 26 | })); 27 | 28 | export default function ExperimentForm({ 29 | experimentId, 30 | updateExperiments, 31 | }: { 32 | experimentId: number, 33 | updateExperiments: () => Promise, 34 | }) { 35 | const classes = useStyles(); 36 | 37 | const [open, setOpen] = useState(false); 38 | const [savingValue, setSavingValue] = useState(false); 39 | const [errorText, setErrorText] = useState(''); 40 | 41 | function resetState() { 42 | setOpen(false); 43 | setErrorText(''); 44 | } 45 | 46 | const handleClickOpen = () => { 47 | setOpen(true); 48 | }; 49 | 50 | const handleClose = () => { 51 | setOpen(false); 52 | }; 53 | 54 | async function submitForm(element: React.FormEvent) { 55 | element.preventDefault(); 56 | setSavingValue(true); 57 | try { 58 | const userError = await API.endExperiment(experimentId); 59 | 60 | if (userError) { 61 | setErrorText(userError); 62 | } else { 63 | resetState(); 64 | updateExperiments(); 65 | } 66 | } catch (e: any) { 67 | console.error(e); 68 | setErrorText('This page broke :/'); 69 | } finally { 70 | setSavingValue(false); 71 | } 72 | } 73 | 74 | return ( 75 |
76 | 79 | 80 | End Experiment? 81 | 82 |
83 | 86 | 94 |
95 |
96 | 97 | {errorText ? errorText : '\u00A0'} 98 | 99 |
100 | 101 | 102 |
103 | ); 104 | } 105 | -------------------------------------------------------------------------------- /web/src/components/MetricList.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from 'react'; 2 | import { withStyles, makeStyles } from '@material-ui/core/styles'; 3 | import Paper from '@material-ui/core/Paper'; 4 | import Table from '@material-ui/core/Table'; 5 | import TableBody from '@material-ui/core/TableBody'; 6 | import TableCell from '@material-ui/core/TableCell'; 7 | import TableContainer from '@material-ui/core/TableContainer'; 8 | import TableHead from '@material-ui/core/TableHead'; 9 | import TableRow from '@material-ui/core/TableRow'; 10 | 11 | import API from 'api/api' 12 | import MetricForm from 'components/metrics/MetricForm'; 13 | 14 | const StyledTableCell = withStyles((theme) => ({ 15 | head: { 16 | backgroundColor: theme.palette.common.black, 17 | color: theme.palette.common.white, 18 | textAlign: 'left', 19 | }, 20 | body: { 21 | fontSize: 14, 22 | textAlign: 'left', 23 | }, 24 | }))(TableCell); 25 | 26 | const StyledTableRow = withStyles((theme) => ({ 27 | root: { 28 | '&:nth-of-type(odd)': { 29 | backgroundColor: theme.palette.action.hover, 30 | }, 31 | }, 32 | }))(TableRow); 33 | 34 | 35 | const useStyles = makeStyles({ 36 | root: { 37 | paddingRight: '8%', 38 | paddingLeft: '4%', 39 | }, 40 | metricsHeader: { 41 | padding: '20px 0px 27px 0px', 42 | fontSize: '24px', 43 | fontWeight: 'bold', 44 | }, 45 | metricsHeaderText: { 46 | display: 'inline-block', 47 | }, 48 | newMetricButton: { 49 | float: 'right', 50 | }, 51 | table: { 52 | minWidth: 700, 53 | }, 54 | }); 55 | 56 | 57 | export default function MetricList() { 58 | const classes = useStyles(); 59 | 60 | const [metrics, setMetrics] = useState([]); 61 | 62 | async function getMetrics() { 63 | setMetrics(await API.getMetrics()); 64 | } 65 | useEffect(() => { 66 | getMetrics(); 67 | }, []); 68 | 69 | return ( 70 |
71 |
72 |
Metric
73 |
74 | 75 |
76 |
77 | 78 | 79 | 80 | 81 | Name 82 | Type 83 | 84 | 85 | 86 | {metrics.map((metric, idx) => { 87 | return ( 88 | 89 | {metric.name} 90 | {metric.type} 91 | 92 | ); 93 | })} 94 | 95 |
96 | 97 |
98 | ); 99 | } 100 | 101 | -------------------------------------------------------------------------------- /web/src/components/performance/Chart.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from 'react'; 2 | import { makeStyles } from '@material-ui/core/styles'; 3 | import { Chart } from 'react-google-charts'; 4 | import API, { PerformanceData, DataPoint } from 'api/api'; 5 | 6 | // coefficient for 95% confidence 7 | const Z = 1.960; 8 | 9 | interface PerformanceChartProps { 10 | runID: string; 11 | metric: string; 12 | } 13 | 14 | const useStyles = makeStyles({ 15 | root: { 16 | border: '1px solid rgba(0, 0, 0, 0.12)', 17 | marginBottom: '25px', 18 | }, 19 | }); 20 | 21 | export default function PerformanceChart ({ 22 | runID, 23 | metric, 24 | }: PerformanceChartProps) { 25 | const classes = useStyles(); 26 | 27 | const [performanceData, setPerformanceData] = useState({ 28 | control: [], 29 | experiment: [], 30 | }); 31 | const chartData = []; 32 | 33 | useEffect(() => { 34 | async function getData() { 35 | setPerformanceData(await API.getPerformance(runID, metric)); 36 | } 37 | getData(); 38 | }, [runID, metric]); 39 | 40 | const experimentDataMap: Record = {}; 41 | for (const dataPoint of performanceData.experiment) { 42 | experimentDataMap[dataPoint['date']] = dataPoint; 43 | } 44 | for (const dataPoint of performanceData.control) { 45 | const cP = dataPoint; 46 | var eP: DataPoint = {} as DataPoint; 47 | if (cP.date in experimentDataMap) { 48 | eP = experimentDataMap[cP.date]; 49 | } 50 | 51 | const controlConfidence = (Z * (cP.stddev / Math.sqrt(cP.count))); 52 | const experimentConfidence = (Z * (eP.stddev / Math.sqrt(eP.count))); 53 | chartData.push([ 54 | new Date(cP.date), 55 | cP.avg, 56 | cP.avg - controlConfidence, 57 | cP.avg + controlConfidence, 58 | eP.avg, 59 | eP.avg - experimentConfidence, 60 | eP.avg + experimentConfidence, 61 | ]); 62 | } 63 | 64 | chartData.unshift([ 65 | { type: 'date', label: 'Date' }, 66 | { type: 'number', label: 'Control' }, 67 | { id: 'i0', type: 'number', role: 'interval' }, 68 | { id: 'i0', type: 'number', role: 'interval' }, 69 | { type: 'number', label: 'Experiment' }, 70 | { id: 'i1', type: 'number', role: 'interval' }, 71 | { id: 'i1', type: 'number', role: 'interval' }, 72 | ]); 73 | 74 | return (
75 | Loading Chart
} 80 | data={chartData} 81 | options={{ 82 | title: metric, 83 | curveType: 'function', 84 | explorer: { 85 | actions: ['dragToPan', 'dragToZoom', 'rightClickToReset'], 86 | axis: 'horizontal', 87 | keepInBounds: true, 88 | maxZoomIn: 4.0, 89 | }, 90 | intervals: { color: 'series-color' }, 91 | interval: { 92 | i0: { color: '#5BE7F8', style: 'area', curveType: 'function', fillOpacity: 0.3 }, 93 | i1: { color: '#F8775B', style: 'area', curveType: 'function', fillOpacity: 0.3 }, 94 | }, 95 | legend: {position: 'bottom', textStyle: {color: 'black', fontSize: 16}} 96 | }} 97 | rootProps={{ 'data-id': `${runID}-${metric}` }} 98 | /> 99 | ); 100 | }; 101 | -------------------------------------------------------------------------------- /docs/quickstart/QUICKSTART.md: -------------------------------------------------------------------------------- 1 | # Quickstart 2 | 3 | The quickstart guide is structured so that you can quickly and easily understand the scrutinize feature set in an easy enough way. This guide will walk you through the following: 4 | 5 | 1. Running the service 6 | 2. Installing the client 7 | 3. Adding metrics to your project 8 | 4. Setting up your first experiment 9 | 5. Reviewing the results of your experiment 10 | 11 | ### Requirements 12 | 13 | | Requirement | Version | 14 | | ------------- | ------------- | 15 | | docker-compose| 1.0+ | 16 | | python | 3.6+ | 17 | 18 | ### Running the service 19 | 20 | After cloning the repository locally, you can start the server and database using docker compose: 21 | 22 | ```bash 23 | cd // 24 | docker-compose up 25 | ``` 26 | 27 | scrutinize should then be running on port 5001, verify this by navigating to the [Experiments Dashboard](http://localhost:5001). 28 | 29 | ### Installing the client 30 | 31 | Now that the server is populated you are ready to begin using the client. Install a client of your choice (currently only python) into the project you want to experiment with. 32 | 33 | ```bash 34 | pip install scrutinize 35 | ``` 36 | 37 | ### Adding a metric to your project 38 | 39 | Before you can use the client, you need to add some metrics and experiments. Experiments depend on metrics, so please create some metrics first by navigating to the [Metrics Dashboard](http://localhost:5001/metrics). Some example metrics might include "converted", "purchase_price", "load_time_ms" and "user_click". 40 | 41 | ![adding a metric](../readme/add_metrics.png) 42 | 43 | Once you've completed that, you can start recording metrics from within your application using the client. See the below code snippet for an example: 44 | 45 | ```python 46 | from scrutinize import ScrutinizeClient 47 | 48 | class CheckoutController: 49 | def __init__(self, scrutinize: ScrutinizeClient, ...): 50 | self.scrutinize = scrutinize 51 | ... 52 | 53 | def complete_purchase(self, user_id): 54 | basket = self.get_basket(user_id) 55 | 56 | # recording a metric value for reporting 57 | self.scrutinize.observe( 58 | user_id, 59 | 'checkout_amount', 60 | basket.price_total, 61 | ) 62 | ``` 63 | 64 | ### Setting up your first experiment 65 | 66 | Navigate to the [Experiments Dashboard](http://localhost:5001) and create an experiment with some of the metrics just added. 67 | 68 | ![adding an experiment](../readme/add_experiment.png) 69 | 70 | When the experiement has been saved, turn on the __Active__ toggle to ensure that traffic to the experiment can see the new behavior. 71 | 72 | You should be able to now conduct the experiment in your service using the client API. Use the below code snippet as a reference: 73 | 74 | ```python 75 | from scrutinize import ScrutinizeClient 76 | 77 | class AdsController: 78 | def __init__(self, scrutinize: ScrutinizeClient, ...): 79 | self.scrutinize = scrutinize 80 | ... 81 | 82 | def show_ads(self, user_id): 83 | return self.scrutinize.call( 84 | 'ml_eng.new_ads_model', 85 | user_id, 86 | lambda: self.ads_model.predict(user_id), # control behavior 87 | lambda: self.new_ads_model.predict(user_id), # experiment behavior 88 | ) 89 | ``` 90 | 91 | ### Reviewing the results of your experiment 92 | 93 | 94 | While your experiment is running, you can use the Performance feature in the [Experiments Dashboard](http://localhost:5001) to get RED metrics on your experiment, as well as the selected Evaluation Metrics you had defined. 95 | 96 | ![checking the experiment results](../readme/reporting.png) 97 | -------------------------------------------------------------------------------- /web/src/components/performance/InfoBar.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { makeStyles } from '@material-ui/core/styles'; 3 | import Card from '@material-ui/core/Card'; 4 | import CardContent from '@material-ui/core/CardContent'; 5 | import Paper from '@material-ui/core/Paper'; 6 | import Table from '@material-ui/core/Table'; 7 | import TableBody from '@material-ui/core/TableBody'; 8 | import TableCell from '@material-ui/core/TableCell'; 9 | import TableContainer from '@material-ui/core/TableContainer'; 10 | import TableHead from '@material-ui/core/TableHead'; 11 | import TableRow from '@material-ui/core/TableRow'; 12 | import Typography from '@material-ui/core/Typography'; 13 | 14 | import { ExperimentDetails } from 'api/api' 15 | 16 | const useStyles = makeStyles({ 17 | root: { 18 | marginTop: '30px', 19 | marginBottom: '20px', 20 | display: 'flex', 21 | }, 22 | detailsCard: { 23 | marginRight: '20px', 24 | display: 'inline-block', 25 | minWidth: '200px', 26 | }, 27 | pos: { 28 | marginBottom: 4, 29 | }, 30 | title: { 31 | fontSize: 14, 32 | }, 33 | center: { 34 | fontSize: '12px', 35 | textAlign: 'center', 36 | }, 37 | tableContainer: { 38 | display: 'inline-block', 39 | verticalAlign: 'top', 40 | border: '1px solid rgba(0, 0, 0, 0.12)', 41 | boxShadow: 'none', 42 | }, 43 | table: { 44 | minWidth: 650, 45 | }, 46 | }); 47 | 48 | interface PerformanceInfoBarProps { 49 | details: ExperimentDetails, 50 | } 51 | 52 | export default function PerformanceInfoBar({ 53 | details, 54 | }: PerformanceInfoBarProps) { 55 | const classes = useStyles(); 56 | 57 | const { name, percentage, active } = details; 58 | return ( 59 |
60 | 61 | 62 | 63 | Experiment 64 | 65 | 66 | {name} 67 | 68 | 69 | rollout: {percentage}% 70 | 71 | 72 | status: { active ? 'active' : 'disabled' } 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | Variant 81 | Unique Users 82 | Rate 83 | Error % 84 | Avg Duration (ms) 85 | 86 | 87 | 88 | {details.variants.map((variant) => { 89 | return ( 90 | 91 | {variant.variant} 92 | 93 | {variant.unique_users} 94 | {variant.volume} 95 | {(parseFloat(variant.pct_error) * 100).toFixed(2)}% 96 | {parseFloat(variant.duration_ms).toFixed(2)} 97 | ) 98 | })} 99 | 100 |
101 | 102 |
103 | ); 104 | } 105 | -------------------------------------------------------------------------------- /server/src/database/reporting.ts: -------------------------------------------------------------------------------- 1 | import { Pool } from 'pg'; 2 | 3 | import { 4 | Run, 5 | Metric, 6 | VariantDetails, 7 | Details, 8 | Performance, 9 | DataPoint, 10 | } from 'database/model'; 11 | 12 | export default class ReportingStore { 13 | constructor ( 14 | private pool: Pool, 15 | ) {} 16 | 17 | public async getDetails(runID: number): Promise
{ 18 | const details: Details = (await this.pool.query( 19 | `SELECT e.id, e.name, r.percentage, r.started_time, r.ended_time 20 | FROM Experiment e 21 | JOIN Run r ON r.experiment_id=e.id 22 | WHERE r.id=$1`, 23 | [runID], 24 | )).rows[0] as Details; 25 | 26 | details.variants = (await this.pool.query( 27 | `SELECT variant, 28 | COUNT(*) as volume, 29 | COUNT(DISTINCT user_id) AS unique_users, 30 | AVG(duration_ms) as duration_ms, 31 | 1.0 * SUM(CASE WHEN LENGTH(error) > 0 THEN 1 ELSE 0 END) / COUNT(*) as pct_error 32 | FROM Treatment 33 | WHERE run_id=$1 34 | GROUP BY variant`, 35 | [runID], 36 | )).rows as VariantDetails[]; 37 | 38 | details.evaluation_criterion = (await this.pool.query( 39 | `SELECT id, name, type 40 | FROM Metric m 41 | JOIN EvaluationCriterion ec ON ec.metric_id = m.id 42 | WHERE ec.run_id=$1`, 43 | [runID], 44 | )).rows as Metric[]; 45 | 46 | return details; 47 | } 48 | 49 | public async getPerformance(runID: number, metric: string): Promise { 50 | const run = (await this.pool.query( 51 | ` 52 | SELECT id, started_time, ended_time 53 | FROM Run 54 | WHERE id=$1 55 | `, 56 | [runID], 57 | )).rows[0] as Run; 58 | 59 | /* 60 | * With the following query, we preselect a TreatmentLookup 61 | * This lookup is used to match measurements to treatments 62 | * using the user_id associated with both. 63 | * 64 | * It is using this mechanism that we are able to partition 65 | * the measurement data between control and experiment. 66 | */ 67 | const rows = (await this.pool.query( 68 | ` 69 | WITH TreatmentLookup AS ( 70 | SELECT t.user_id, t.variant 71 | FROM Treatment t 72 | JOIN Run r ON r.id=t.run_id 73 | WHERE r.id=$1 74 | ) 75 | SELECT DATE(m.created_time), tl.variant, COUNT(*), AVG(value), STDDEV(value) 76 | FROM Measurement m 77 | JOIN TreatmentLookup tl ON tl.user_id=m.user_id 78 | WHERE m.created_time BETWEEN COALESCE($2, CURRENT_TIMESTAMP) AND COALESCE($3, CURRENT_TIMESTAMP) AND 79 | m.metric_name=$4 80 | GROUP BY 1, 2 81 | ORDER BY 1 ASC, 2 ASC 82 | `, 83 | [runID, run.started_time, run.ended_time, metric], 84 | )).rows as DataPoint[]; 85 | 86 | const performance: Performance = { 87 | control: [], 88 | experiment: [], 89 | }; 90 | 91 | for (const row of rows) { 92 | const { variant, count, avg, stddev } = row; 93 | // data transformation - node-postgres returns strings 94 | row.count = parseFloat(count as unknown as string); 95 | row.avg = parseFloat(avg as unknown as string); 96 | row.stddev = parseFloat(stddev as unknown as string); 97 | 98 | performance[variant].push(row); 99 | } 100 | return performance; 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /web/src/App.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { makeStyles } from '@material-ui/core/styles'; 3 | import Drawer from '@material-ui/core/Drawer'; 4 | import CssBaseline from '@material-ui/core/CssBaseline'; 5 | import AppBar from '@material-ui/core/AppBar'; 6 | import Toolbar from '@material-ui/core/Toolbar'; 7 | import List from '@material-ui/core/List'; 8 | import Typography from '@material-ui/core/Typography'; 9 | import Divider from '@material-ui/core/Divider'; 10 | import ListItem from '@material-ui/core/ListItem'; 11 | import ListItemIcon from '@material-ui/core/ListItemIcon'; 12 | import ListItemText from '@material-ui/core/ListItemText'; 13 | import BlurOnIcon from '@material-ui/icons/BlurOn'; 14 | import GraphicEqIcon from '@material-ui/icons/GraphicEq'; 15 | import TimelineIcon from '@material-ui/icons/Timeline'; 16 | 17 | import { 18 | BrowserRouter as Router, 19 | Switch, 20 | Route, 21 | Link 22 | } from "react-router-dom"; 23 | 24 | import ExperimentList from './components/ExperimentList'; 25 | import MetricList from './components/MetricList'; 26 | import PerformancePage from './components/Performance'; 27 | 28 | const drawerWidth = 240; 29 | 30 | const useStyles = makeStyles((theme) => ({ 31 | root: { 32 | display: 'flex', 33 | }, 34 | appBar: { 35 | width: `calc(100% - ${drawerWidth}px)`, 36 | marginLeft: drawerWidth, 37 | }, 38 | drawer: { 39 | width: drawerWidth, 40 | flexShrink: 0, 41 | }, 42 | drawerPaper: { 43 | width: drawerWidth, 44 | }, 45 | // necessary for content to be below app bar 46 | toolbar: theme.mixins.toolbar, 47 | content: { 48 | flexGrow: 1, 49 | backgroundColor: theme.palette.background.default, 50 | padding: theme.spacing(3), 51 | }, 52 | })); 53 | 54 | export default function App() { 55 | const classes = useStyles(); 56 | console.log(classes.toolbar); 57 | 58 | return ( 59 | 60 |
61 | 62 | 63 | 64 | 65 | scrutinize 66 | 67 | 68 | 69 | 77 |
78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 |
96 |
97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 |
109 |
110 | 111 | ); 112 | } 113 | -------------------------------------------------------------------------------- /web/src/components/ExperimentList.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useEffect } from 'react'; 2 | import { withStyles, makeStyles } from '@material-ui/core/styles'; 3 | import Button from '@material-ui/core/Button'; 4 | import Paper from '@material-ui/core/Paper'; 5 | import Table from '@material-ui/core/Table'; 6 | import TableBody from '@material-ui/core/TableBody'; 7 | import TableCell from '@material-ui/core/TableCell'; 8 | import TableContainer from '@material-ui/core/TableContainer'; 9 | import TableHead from '@material-ui/core/TableHead'; 10 | import TableRow from '@material-ui/core/TableRow'; 11 | 12 | import API from 'api/api'; 13 | import ExperimentForm from 'components/experiment/ExperimentForm'; 14 | import StartExperimentForm from 'components/experiment/StartExperimentForm'; 15 | import EndExperimentForm from 'components/experiment/EndExperimentForm'; 16 | 17 | const StyledTableCell = withStyles((theme) => ({ 18 | head: { 19 | backgroundColor: theme.palette.common.black, 20 | color: theme.palette.common.white, 21 | textAlign: 'left', 22 | }, 23 | body: { 24 | fontSize: 14, 25 | textAlign: 'left', 26 | }, 27 | }))(TableCell); 28 | 29 | const StyledTableRow = withStyles((theme) => ({ 30 | root: { 31 | '&:nth-of-type(odd)': { 32 | backgroundColor: theme.palette.action.hover, 33 | }, 34 | }, 35 | }))(TableRow); 36 | 37 | 38 | const useStyles = makeStyles({ 39 | root: { 40 | paddingRight: '8%', 41 | paddingLeft: '4%', 42 | }, 43 | experimentsHeader: { 44 | padding: '20px 0px 27px 0px', 45 | fontSize: '24px', 46 | fontWeight: 'bold', 47 | }, 48 | experimentsHeaderText: { 49 | display: 'inline-block', 50 | }, 51 | newExperimentButton: { 52 | float: 'right', 53 | }, 54 | table: { 55 | minWidth: 700, 56 | }, 57 | }); 58 | 59 | export default function ExperimentList() { 60 | const classes = useStyles(); 61 | 62 | const [experiments, setExperiments] = useState([]); 63 | 64 | async function getExperiments() { 65 | setExperiments(await API.getExperiments()); 66 | } 67 | useEffect(() => { 68 | getExperiments(); 69 | }, []); 70 | 71 | return ( 72 |
73 |
74 |
Experiments
75 |
76 | 77 |
78 |
79 | 80 | 81 | 82 | 83 | Name 84 | Rollout 85 | Status 86 | Activity 87 | 88 | 89 | 90 | {experiments.map((experiment, idx) => { 91 | const experimentActivityButton = experiment.active ? 92 | : 93 | ; 94 | return ( 95 | 96 | { experiment.name } 97 | 98 | { experiment.active ? `${experiment.percentage}%` : '' } 99 | 100 | 101 | { experiment.active ? "running" : "inactive" } 102 | 103 | 104 | { experimentActivityButton } 105 | 106 | 107 | ); 108 | })} 109 | 110 |
111 | 112 |
113 | ); 114 | } 115 | 116 | -------------------------------------------------------------------------------- /web/src/api/api.ts: -------------------------------------------------------------------------------- 1 | const BASE_PATH = '/api/v1'; 2 | 3 | export interface Experiment { 4 | id?: number; 5 | name: string; 6 | description: string; 7 | active?: boolean; 8 | } 9 | 10 | export interface ExperimentConfig { 11 | id?: number; 12 | experiment_id: number; 13 | percentage: number; 14 | metrics: Metric[]; 15 | } 16 | 17 | export interface Metric { 18 | id?: number; 19 | name: string; 20 | type?: 'binomial' | 'continuous' | 'count'; 21 | } 22 | 23 | export interface PerformanceData { 24 | control: Array; 25 | experiment: Array; 26 | } 27 | 28 | export interface DataPoint { 29 | date: string; 30 | count: number; 31 | avg: number; 32 | stddev: number; 33 | } 34 | 35 | export interface VariantDetails { 36 | variant: string; 37 | volume: string; 38 | unique_users: string; 39 | duration_ms: string; 40 | pct_error: string; 41 | } 42 | 43 | export interface ExperimentDetails { 44 | name: string; 45 | percentage: number; 46 | active: boolean; 47 | created_time: string; 48 | last_active_time: string; 49 | variants: VariantDetails[]; 50 | evaluation_criterion: Metric[]; 51 | } 52 | 53 | class API { 54 | async getExperiments(search: string=''): Promise { 55 | return (await (await fetch(`${BASE_PATH}/experiment`)).json()) as Experiment[]; 56 | } 57 | 58 | async saveExperiment(experiment: Experiment): Promise { 59 | const res = await fetch(`${BASE_PATH}/experiment`, { 60 | method: 'POST', 61 | headers: { 62 | 'Content-Type': 'application/json', 63 | }, 64 | body: JSON.stringify(experiment), 65 | }); 66 | if (res.status > 399) { 67 | const data = await res.json() 68 | if ('userError' in data) { 69 | return data['userError']; 70 | } else { 71 | return 'Unknown server error'; 72 | } 73 | } 74 | return ''; 75 | } 76 | 77 | async startExperiment(config: ExperimentConfig): Promise { 78 | const res = await fetch(`${BASE_PATH}/experiment/start`, { 79 | method: 'POST', 80 | headers: { 81 | 'Content-Type': 'application/json', 82 | }, 83 | body: JSON.stringify(config), 84 | }); 85 | if (res.status > 399) { 86 | const data = await res.json() 87 | if ('userError' in data) { 88 | return data['userError']; 89 | } else { 90 | return 'Unknown server error'; 91 | } 92 | } 93 | return ''; 94 | } 95 | 96 | async endExperiment(experiment_id: number): Promise { 97 | const res = await fetch(`${BASE_PATH}/experiment/end`, { 98 | method: 'POST', 99 | headers: { 100 | 'Content-Type': 'application/json', 101 | }, 102 | body: JSON.stringify({ id: experiment_id }), 103 | }); 104 | if (res.status > 399) { 105 | const data = await res.json() 106 | if ('userError' in data) { 107 | return data['userError']; 108 | } else { 109 | return 'Unknown server error'; 110 | } 111 | } 112 | return ''; 113 | } 114 | 115 | async toggleExperimentActive(experiment: Experiment) { 116 | await fetch(`${BASE_PATH}/experiment/active`, { 117 | method: 'POST', 118 | headers: { 119 | 'Content-Type': 'application/json', 120 | }, 121 | body: JSON.stringify(experiment), 122 | }); 123 | } 124 | 125 | async getMetrics(): Promise { 126 | return (await fetch(`${BASE_PATH}/metric`)).json(); 127 | } 128 | 129 | async saveMetric(metric: Metric): Promise { 130 | const res = await fetch(`${BASE_PATH}/metric`, { 131 | method: 'POST', 132 | headers: { 133 | 'Content-Type': 'application/json', 134 | }, 135 | body: JSON.stringify(metric), 136 | }); 137 | if (res.status > 399) { 138 | const data = await res.json() 139 | if ('userError' in data) { 140 | return data['userError']; 141 | } else { 142 | return 'Unknown server error'; 143 | } 144 | } 145 | return ''; 146 | } 147 | 148 | async getDetails(runID: string): Promise { 149 | return await(await fetch(`${BASE_PATH}/details/${runID}`)).json() as ExperimentDetails; 150 | } 151 | 152 | async getPerformance(runID: string, metric: string): Promise { 153 | return await(await fetch(`${BASE_PATH}/performance/${runID}/${metric}`)).json() as PerformanceData; 154 | } 155 | } 156 | 157 | export default new API(); 158 | -------------------------------------------------------------------------------- /web/src/components/metrics/MetricForm.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | import Button from '@material-ui/core/Button'; 3 | import Dialog from '@material-ui/core/Dialog'; 4 | import DialogActions from '@material-ui/core/DialogActions'; 5 | import DialogContent from '@material-ui/core/DialogContent'; 6 | import DialogTitle from '@material-ui/core/DialogTitle'; 7 | import Grid from '@material-ui/core/Grid'; 8 | import InputLabel from '@material-ui/core/InputLabel'; 9 | import MenuItem from '@material-ui/core/MenuItem'; 10 | import Select from '@material-ui/core/Select'; 11 | import TextField from '@material-ui/core/TextField'; 12 | import Typography from '@material-ui/core/Typography'; 13 | 14 | import API from 'api/api'; 15 | 16 | export default function MetricForm({ 17 | updateMetrics, 18 | }: {updateMetrics: () => Promise}) { 19 | const [open, setOpen] = useState(false); 20 | const [savingValue, setSavingValue] = useState(false); 21 | const [metricName, setMetricName] = useState(''); 22 | const [metricType, setMetricType] = useState('binomial'); 23 | const [errorText, setErrorText] = useState(''); 24 | 25 | function resetState() { 26 | setOpen(false); 27 | setSavingValue(false); 28 | setMetricName(''); 29 | setMetricType('binomial'); 30 | setErrorText(''); 31 | } 32 | 33 | const handleClickOpen = () => { 34 | setOpen(true); 35 | }; 36 | 37 | const handleClose = () => { 38 | setOpen(false); 39 | }; 40 | 41 | const handleChangeName = (e: any) => { 42 | setMetricName(e.target.value); 43 | } 44 | 45 | const handleChangeType = (e: any) => { 46 | setMetricType(e.target.value); 47 | } 48 | 49 | async function submitForm(element: React.FormEvent) { 50 | element.preventDefault(); 51 | setSavingValue(true); 52 | try { 53 | if (!metricName) { 54 | setErrorText('Must include a name for your metric'); 55 | return 56 | } 57 | 58 | const userError = await API.saveMetric({ 59 | name: metricName, 60 | type: metricType as any, 61 | }); 62 | 63 | if (userError) { 64 | setErrorText(userError); 65 | } else { 66 | resetState(); 67 | updateMetrics(); 68 | } 69 | } catch (e: any) { 70 | console.error(e); 71 | setErrorText('This page broke :/'); 72 | } finally { 73 | setSavingValue(false); 74 | } 75 | } 76 | 77 | return ( 78 |
79 | 82 | 83 | Create Metric 84 | 85 | 86 | 87 | 99 | 100 | Permitted: letters, numbers, periods and underscores 101 | 102 | 103 | 104 | Metric Type 105 | 115 | 116 | 117 | 118 | {errorText ? errorText : '\u00A0'} 119 | 120 | 121 | 122 | 123 | 124 | 127 | 136 | 137 | 138 |
139 | ); 140 | } 141 | -------------------------------------------------------------------------------- /web/src/components/experiment/StartExperimentForm.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | import Button from '@material-ui/core/Button'; 3 | import Dialog from '@material-ui/core/Dialog'; 4 | import DialogActions from '@material-ui/core/DialogActions'; 5 | import DialogContent from '@material-ui/core/DialogContent'; 6 | import DialogTitle from '@material-ui/core/DialogTitle'; 7 | import Grid from '@material-ui/core/Grid'; 8 | import TextField from '@material-ui/core/TextField'; 9 | import Typography from '@material-ui/core/Typography'; 10 | import { makeStyles } from '@material-ui/core/styles'; 11 | 12 | import MetricSelect from 'components/experiment/MetricSelect'; 13 | import PercentageSlider from 'components/experiment/PercentageSlider'; 14 | 15 | import API, { Metric } from 'api/api'; 16 | 17 | const useStyles = makeStyles((theme) => ({ 18 | dialog: { 19 | overflowY: 'visible', 20 | }, 21 | dialogContent: { 22 | width: '600px', 23 | overflowY: 'visible', 24 | }, 25 | form: { 26 | width: '100%', // Fix IE 11 issue. 27 | marginTop: theme.spacing(3), 28 | }, 29 | submit: { 30 | margin: '8px 0px 12px', 31 | }, 32 | evaluationMetricsText: { 33 | marginBottom: '4px', 34 | }, 35 | errorField: { 36 | width: '100%', 37 | textAlign: 'center', 38 | } 39 | })); 40 | 41 | export default function ExperimentFormCopy({ 42 | experimentId, 43 | updateExperiments, 44 | }: { 45 | experimentId: number, 46 | updateExperiments: () => Promise, 47 | }) { 48 | const classes = useStyles(); 49 | 50 | const [open, setOpen] = useState(false); 51 | const [savingValue, setSavingValue] = useState(false); 52 | const [rollout, setRollout] = useState(20); 53 | const [metrics, setMetrics] = useState([]); 54 | const [errorText, setErrorText] = useState(''); 55 | 56 | function resetState() { 57 | setOpen(false); 58 | setSavingValue(false); 59 | setRollout(5); 60 | setErrorText(''); 61 | } 62 | 63 | const handleClickOpen = () => { 64 | setOpen(true); 65 | }; 66 | 67 | const handleClose = () => { 68 | setOpen(false); 69 | }; 70 | 71 | async function submitForm(element: React.FormEvent) { 72 | element.preventDefault(); 73 | setSavingValue(true); 74 | try { 75 | const userError = await API.startExperiment({ 76 | experiment_id: experimentId, 77 | percentage: rollout, 78 | metrics: metrics, 79 | }); 80 | 81 | if (userError) { 82 | setErrorText(userError); 83 | } else { 84 | resetState(); 85 | updateExperiments(); 86 | } 87 | } catch (e: any) { 88 | console.error(e); 89 | setErrorText('This page broke :/'); 90 | } finally { 91 | setSavingValue(false); 92 | } 93 | } 94 | 95 | function handleChangeRollout(_: React.ChangeEvent<{}>, value: number | number[]) { 96 | setRollout(value as number); 97 | } 98 | 99 | return ( 100 |
101 | 104 | 105 | Start Experiment 106 | 107 |
108 | 109 | 110 | 111 | Rollout 112 | 113 | 119 | 120 | 121 | 122 | Evaluation Metrics 123 | 124 | 125 | 126 | 127 | 128 | {errorText ? errorText : '\u00A0'} 129 | 130 | 131 | 132 |
133 | 134 | 135 | 138 | 147 | 148 | 149 |
150 | ); 151 | } 152 | -------------------------------------------------------------------------------- /server/src/database/experiment.ts: -------------------------------------------------------------------------------- 1 | import { Pool } from 'pg'; 2 | 3 | import { 4 | Experiment, 5 | Run, 6 | Treatment, 7 | } from 'database/model'; 8 | 9 | import { UserError } from '../middleware/errors'; 10 | 11 | export default class ExperimentStore { 12 | constructor ( 13 | private pool: Pool, 14 | ) {} 15 | 16 | public async getExperiments(): Promise { 17 | return (await this.pool.query( 18 | ` 19 | SELECT e.id, e.name, e.active, r.id as run_id, r.percentage 20 | FROM Experiment e 21 | LEFT JOIN ( 22 | SELECT MAX(id) as id, experiment_id 23 | FROM Run 24 | GROUP BY experiment_id 25 | ) as cr ON cr.experiment_id = e.id AND e.active 26 | LEFT JOIN Run r ON r.id=cr.id 27 | ORDER BY e.id DESC 28 | ` 29 | )).rows; 30 | } 31 | 32 | public async createExperiment({ name, description }: Experiment): Promise { 33 | try { 34 | const id = (await this.pool.query( 35 | ` 36 | INSERT INTO Experiment(name, description) 37 | VALUES ($1, $2) 38 | RETURNING id 39 | `, 40 | [name, description] 41 | )).rows[0].id as unknown as number; 42 | return { 43 | id, 44 | name, 45 | description, 46 | active: false, 47 | }; 48 | } catch(e: any) { 49 | if (e instanceof Error) { 50 | if (e.message.indexOf('unique constraint') !== -1) { 51 | e = UserError(e, 'Experiment name taken, please choose a different one'); 52 | } 53 | } 54 | throw(e); 55 | } 56 | } 57 | 58 | public async startExperiment({ 59 | experiment_id, 60 | percentage, 61 | metrics, 62 | }: Run): Promise { 63 | const client = await this.pool.connect(); 64 | try { 65 | await client.query('BEGIN'); 66 | 67 | await client.query( 68 | ` 69 | UPDATE Experiment 70 | SET active=true 71 | WHERE id=$1 72 | `, 73 | [experiment_id], 74 | ); 75 | 76 | const res = await client.query( 77 | ` 78 | INSERT INTO Run(experiment_id, percentage) 79 | VALUES ($1, $2) 80 | RETURNING id 81 | `, 82 | [experiment_id, percentage], 83 | ); 84 | 85 | // NOTE: pg-node prepared statements don't support execution for multiple values i think 86 | for (const metric of metrics || []) { 87 | await client.query( 88 | ` 89 | INSERT INTO EvaluationCriterion (run_id, metric_id) 90 | VALUES ($1, $2) 91 | `, 92 | [res.rows[0].id, metric.id], 93 | ); 94 | } 95 | await client.query('COMMIT'); 96 | } catch (e: any) { 97 | await client.query('ROLLBACK'); 98 | throw(e); 99 | } finally { 100 | client.release(); 101 | } 102 | } 103 | 104 | public async endExperiment(experiment_id: number): Promise { 105 | const client = await this.pool.connect(); 106 | try { 107 | await client.query('BEGIN'); 108 | 109 | await client.query( 110 | ` 111 | UPDATE Experiment 112 | SET active=false 113 | WHERE id=$1 114 | `, 115 | [experiment_id], 116 | ); 117 | 118 | await client.query( 119 | ` 120 | UPDATE RUN 121 | SET ended_time=CURRENT_TIMESTAMP 122 | WHERE id=(SELECT MAX(id) FROM Run WHERE experiment_id=$1) 123 | `, 124 | [experiment_id], 125 | ); 126 | 127 | await client.query('COMMIT'); 128 | } catch (e: any) { 129 | await client.query('ROLLBACK'); 130 | throw(e); 131 | } finally { 132 | client.release(); 133 | } 134 | } 135 | 136 | public async createTreatment(t: Treatment): Promise { 137 | const { user_id, run_id, variant, error, duration_ms } = t; 138 | await this.pool.query( 139 | ` 140 | INSERT INTO Treatment( 141 | user_id, 142 | run_id, 143 | variant, 144 | error, 145 | duration_ms 146 | ) 147 | VALUES ($1, $2, $3, $4, $5) 148 | `, 149 | [user_id, run_id, variant, error, duration_ms], 150 | ); 151 | } 152 | 153 | public async getExperiment(experiment_id: number): Promise { 154 | const rows = (await this.pool.query( 155 | ` 156 | SELECT * 157 | FROM Experiment 158 | WHERE id=$1 159 | `, 160 | [experiment_id], 161 | )).rows; 162 | return rows.length ? rows[0] as unknown as Experiment : null; 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /web/src/components/experiment/ExperimentForm.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | import Button from '@material-ui/core/Button'; 3 | import Dialog from '@material-ui/core/Dialog'; 4 | import DialogActions from '@material-ui/core/DialogActions'; 5 | import DialogContent from '@material-ui/core/DialogContent'; 6 | import DialogTitle from '@material-ui/core/DialogTitle'; 7 | import Grid from '@material-ui/core/Grid'; 8 | import TextField from '@material-ui/core/TextField'; 9 | import Typography from '@material-ui/core/Typography'; 10 | import { makeStyles } from '@material-ui/core/styles'; 11 | 12 | import API from 'api/api'; 13 | 14 | const useStyles = makeStyles((theme) => ({ 15 | dialog: { 16 | overflowY: 'visible', 17 | }, 18 | dialogContent: { 19 | overflowY: 'visible', 20 | }, 21 | form: { 22 | width: '100%', // Fix IE 11 issue. 23 | marginTop: theme.spacing(3), 24 | }, 25 | submit: { 26 | margin: '8px 0px 12px', 27 | }, 28 | errorField: { 29 | width: '100%', 30 | textAlign: 'center', 31 | } 32 | })); 33 | 34 | export default function ExperimentForm({ 35 | updateExperiments, 36 | }: {updateExperiments: () => Promise}) { 37 | const classes = useStyles(); 38 | 39 | const [open, setOpen] = useState(false); 40 | const [savingValue, setSavingValue] = useState(false); 41 | const [experimentName, setExperimentName] = useState(''); 42 | const [description, setDescription] = useState(''); 43 | const [errorText, setErrorText] = useState(''); 44 | 45 | function resetState() { 46 | setOpen(false); 47 | setSavingValue(false); 48 | setExperimentName(''); 49 | setDescription(''); 50 | setErrorText(''); 51 | } 52 | 53 | const handleClickOpen = () => { 54 | setOpen(true); 55 | }; 56 | 57 | const handleClose = () => { 58 | setOpen(false); 59 | }; 60 | 61 | async function submitForm(element: React.FormEvent) { 62 | element.preventDefault(); 63 | setSavingValue(true); 64 | try { 65 | if (!experimentName) { 66 | setErrorText('Must include a name for your experiment'); 67 | return 68 | } 69 | 70 | const userError = await API.saveExperiment({ 71 | name: experimentName, 72 | description: description, 73 | }); 74 | 75 | if (userError) { 76 | setErrorText(userError); 77 | } else { 78 | resetState(); 79 | updateExperiments(); 80 | } 81 | } catch (e: any) { 82 | console.error(e); 83 | setErrorText('This page broke :/'); 84 | } finally { 85 | setSavingValue(false); 86 | } 87 | } 88 | 89 | function handleChangeName(e: React.ChangeEvent) { 90 | const newName = (e.target as HTMLInputElement).value; 91 | setExperimentName(newName.replace(/[^0-9a-z_.]/gi, '')); 92 | } 93 | 94 | function handleChangeDescription(e: React.ChangeEvent<{}>) { 95 | setDescription((e.target as HTMLInputElement).value); 96 | } 97 | 98 | return ( 99 |
100 | 103 | 104 | Create Experiment 105 | 106 |
107 | 108 | 109 | 121 | 122 | Permitted: letters, numbers, periods and underscores 123 | 124 | 125 | 126 | 137 | 138 | 139 | 140 | {errorText ? errorText : '\u00A0'} 141 | 142 | 143 | 144 |
145 | 146 | 147 | 150 | 159 | 160 | 161 |
162 | ); 163 | } 164 | -------------------------------------------------------------------------------- /clients/javascript/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "scrutinize-client", 3 | "version": "0.0.6", 4 | "lockfileVersion": 2, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "scrutinize-client", 9 | "version": "0.0.6", 10 | "license": "MIT", 11 | "dependencies": { 12 | "axios": "^0.21.0", 13 | "md5": "^2.3.0" 14 | }, 15 | "devDependencies": { 16 | "@types/md5": "^2.2.1", 17 | "typescript": "^4.1.2" 18 | } 19 | }, 20 | "node_modules/@types/md5": { 21 | "version": "2.2.1", 22 | "resolved": "https://registry.npmjs.org/@types/md5/-/md5-2.2.1.tgz", 23 | "integrity": "sha512-bZB0jqBL7JETFqvRKyuDETFceFaVcLm2MBPP5LFEEL/SZuqLnyvzF37tXmMERDncC3oeEj/fOUw88ftJeMpZaw==", 24 | "dev": true, 25 | "dependencies": { 26 | "@types/node": "*" 27 | } 28 | }, 29 | "node_modules/@types/node": { 30 | "version": "14.14.10", 31 | "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.10.tgz", 32 | "integrity": "sha512-J32dgx2hw8vXrSbu4ZlVhn1Nm3GbeCFNw2FWL8S5QKucHGY0cyNwjdQdO+KMBZ4wpmC7KhLCiNsdk1RFRIYUQQ==", 33 | "dev": true 34 | }, 35 | "node_modules/axios": { 36 | "version": "0.21.0", 37 | "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.0.tgz", 38 | "integrity": "sha512-fmkJBknJKoZwem3/IKSSLpkdNXZeBu5Q7GA/aRsr2btgrptmSCxi2oFjZHqGdK9DoTil9PIHlPIZw2EcRJXRvw==", 39 | "dependencies": { 40 | "follow-redirects": "^1.10.0" 41 | } 42 | }, 43 | "node_modules/charenc": { 44 | "version": "0.0.2", 45 | "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", 46 | "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=", 47 | "engines": { 48 | "node": "*" 49 | } 50 | }, 51 | "node_modules/crypt": { 52 | "version": "0.0.2", 53 | "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", 54 | "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=", 55 | "engines": { 56 | "node": "*" 57 | } 58 | }, 59 | "node_modules/follow-redirects": { 60 | "version": "1.13.0", 61 | "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.0.tgz", 62 | "integrity": "sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA==", 63 | "funding": [ 64 | { 65 | "type": "individual", 66 | "url": "https://github.com/sponsors/RubenVerborgh" 67 | } 68 | ], 69 | "engines": { 70 | "node": ">=4.0" 71 | } 72 | }, 73 | "node_modules/is-buffer": { 74 | "version": "1.1.6", 75 | "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", 76 | "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" 77 | }, 78 | "node_modules/md5": { 79 | "version": "2.3.0", 80 | "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", 81 | "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", 82 | "dependencies": { 83 | "charenc": "0.0.2", 84 | "crypt": "0.0.2", 85 | "is-buffer": "~1.1.6" 86 | } 87 | }, 88 | "node_modules/typescript": { 89 | "version": "4.1.2", 90 | "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.1.2.tgz", 91 | "integrity": "sha512-thGloWsGH3SOxv1SoY7QojKi0tc+8FnOmiarEGMbd/lar7QOEd3hvlx3Fp5y6FlDUGl9L+pd4n2e+oToGMmhRQ==", 92 | "dev": true, 93 | "bin": { 94 | "tsc": "bin/tsc", 95 | "tsserver": "bin/tsserver" 96 | }, 97 | "engines": { 98 | "node": ">=4.2.0" 99 | } 100 | } 101 | }, 102 | "dependencies": { 103 | "@types/md5": { 104 | "version": "2.2.1", 105 | "resolved": "https://registry.npmjs.org/@types/md5/-/md5-2.2.1.tgz", 106 | "integrity": "sha512-bZB0jqBL7JETFqvRKyuDETFceFaVcLm2MBPP5LFEEL/SZuqLnyvzF37tXmMERDncC3oeEj/fOUw88ftJeMpZaw==", 107 | "dev": true, 108 | "requires": { 109 | "@types/node": "*" 110 | } 111 | }, 112 | "@types/node": { 113 | "version": "14.14.10", 114 | "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.10.tgz", 115 | "integrity": "sha512-J32dgx2hw8vXrSbu4ZlVhn1Nm3GbeCFNw2FWL8S5QKucHGY0cyNwjdQdO+KMBZ4wpmC7KhLCiNsdk1RFRIYUQQ==", 116 | "dev": true 117 | }, 118 | "axios": { 119 | "version": "0.21.0", 120 | "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.0.tgz", 121 | "integrity": "sha512-fmkJBknJKoZwem3/IKSSLpkdNXZeBu5Q7GA/aRsr2btgrptmSCxi2oFjZHqGdK9DoTil9PIHlPIZw2EcRJXRvw==", 122 | "requires": { 123 | "follow-redirects": "^1.10.0" 124 | } 125 | }, 126 | "charenc": { 127 | "version": "0.0.2", 128 | "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", 129 | "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=" 130 | }, 131 | "crypt": { 132 | "version": "0.0.2", 133 | "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", 134 | "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=" 135 | }, 136 | "follow-redirects": { 137 | "version": "1.13.0", 138 | "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.0.tgz", 139 | "integrity": "sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA==" 140 | }, 141 | "is-buffer": { 142 | "version": "1.1.6", 143 | "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", 144 | "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" 145 | }, 146 | "md5": { 147 | "version": "2.3.0", 148 | "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", 149 | "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", 150 | "requires": { 151 | "charenc": "0.0.2", 152 | "crypt": "0.0.2", 153 | "is-buffer": "~1.1.6" 154 | } 155 | }, 156 | "typescript": { 157 | "version": "4.1.2", 158 | "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.1.2.tgz", 159 | "integrity": "sha512-thGloWsGH3SOxv1SoY7QojKi0tc+8FnOmiarEGMbd/lar7QOEd3hvlx3Fp5y6FlDUGl9L+pd4n2e+oToGMmhRQ==", 160 | "dev": true 161 | } 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /clients/javascript/scrutinize.ts: -------------------------------------------------------------------------------- 1 | import axios, { AxiosInstance } from 'axios'; 2 | import md5 from 'md5'; 3 | 4 | interface Experiment { 5 | run_id: number | null; 6 | name: string; 7 | percentage: number; 8 | active: boolean; 9 | } 10 | 11 | 12 | export default class ScrutinizeClient { 13 | public axios: AxiosInstance; 14 | public experiments: Record | null = null; 15 | public experimentsPullTime = 0; 16 | 17 | constructor( 18 | apiURL: string='http://localhost:11771', 19 | public experimentsTTL: number=300, 20 | ) { 21 | this.axios = axios.create({ 22 | baseURL: `${apiURL}/api/v1`, 23 | timeout: 3000, 24 | }); 25 | } 26 | 27 | public async call( 28 | experimentName: string, 29 | userID: string, 30 | control: any, 31 | experiment: any, 32 | getTime: Function=Date.now, 33 | ): Promise<[boolean, any]> { 34 | /* 35 | * Primary entry point for an experiment. Behaves as follows: 36 | * 37 | * 1. Given a user_id and experiment, determines whether the user should 38 | * get the control or experiment behavior. 39 | * 2. Attempts to resolve the control or variant - can be functional 40 | * or a literal value. 41 | * 3. After resolution is complete or has failed, records the treatment 42 | * event to the server. 43 | * 44 | * experimentName: name of experiment associated with the call 45 | * userID: identifying of the user who originated this request 46 | * control: function or literal value representing the control behavior 47 | * experiment: function or literal value representing the experiment behavior 48 | * getTime: method for getting the current time (overriden in testing) 49 | * 50 | * return value: [user in experiment, value from variant] 51 | */ 52 | const [isExperiment, variantStr] = await this.getVariant(experimentName, userID); 53 | const variant = isExperiment ? experiment : control; 54 | const startTime = getTime(); 55 | var err: Error | null = null; 56 | try { 57 | return [isExperiment, await this.resolve(variant)]; 58 | } catch (e: any) { 59 | err = e as Error; 60 | throw(e); 61 | } finally { 62 | const durationMS = (getTime() - startTime); 63 | await this.createTreatment( 64 | await this.getRunID(experimentName), 65 | userID, 66 | variantStr, 67 | durationMS, 68 | err === null ? '' : err.message, 69 | ); 70 | } 71 | } 72 | 73 | public async getVariant( 74 | experimentName: string, 75 | userID: string, 76 | ): Promise<[boolean, string]> { 77 | /* 78 | * Determines whether the user is in the experiment group. 79 | * 80 | * experimentName: name of experiment associated with the call 81 | * userID: identifying of the user who originated this request 82 | * 83 | * return value: [user in experiment, string representing variant assignment] 84 | */ 85 | const variantOperand = experimentName + userID; 86 | const experiments = await this.getExperiments(); 87 | const experimentConfig = experiments[experimentName]; 88 | var isExperiment = false; 89 | 90 | if (!experimentConfig) { 91 | console.log('ScrutinizeClient.getVariant: experiment not found, only control will be run') 92 | } else if (experimentConfig.active) { 93 | // convert id to a number between 0 and 99 94 | const hash = md5(variantOperand); 95 | const idInt = BigInt('0x' + hash) % BigInt(100); 96 | isExperiment = idInt < experimentConfig.percentage; 97 | } 98 | 99 | return [isExperiment, isExperiment ? 'experiment' : 'control']; 100 | } 101 | 102 | public async getRunID(experimentName: string): Promise { 103 | const experiments = await this.getExperiments(); 104 | const experimentConfig = experiments[experimentName]; 105 | return experimentConfig ? experimentConfig.run_id : null; 106 | } 107 | 108 | public async resolve( 109 | variant: any, 110 | ) { 111 | /* 112 | * Resolves an experiment behavior to a value. 113 | * 114 | * variant: a function or literal value representing the variant behavior 115 | * 116 | * return value: the resolved variant value 117 | */ 118 | 119 | // have mercy on my soul 120 | if (variant instanceof Function) { 121 | if (variant.constructor.name === 'AsyncFunction') { 122 | return await variant(); 123 | } 124 | return variant(); 125 | } 126 | return variant; 127 | } 128 | 129 | public async getExperiments(): Promise> { 130 | /* 131 | * Returns the experiment configuration stored in the server. 132 | * Has simple caching behavior. 133 | * 134 | * return value: a dictionary representing all experiment configurations 135 | */ 136 | const now = Date.now() / 1000; 137 | const experiments = this.experiments || {}; 138 | const shouldPull = !Boolean(this.experiments) || (now - this.experimentsPullTime > this.experimentsTTL); 139 | 140 | if (shouldPull) { 141 | const apiExperiments = (await this.axios.get('/experiment')).data; 142 | for (const exp of apiExperiments) { 143 | experiments[exp.name] = exp; 144 | } 145 | 146 | this.experiments = experiments; 147 | this.experimentsPullTime = now; 148 | } 149 | 150 | return experiments; 151 | } 152 | 153 | public async createTreatment( 154 | runID: number | null, 155 | userID: string, 156 | variant: string, 157 | durationMS: number, 158 | error: string, 159 | ) { 160 | /* 161 | * Publishes a treatment event to the server. 162 | * 163 | * experimentName: name of experiment associated with the call 164 | * userID: identifying of the user who originated this request 165 | * variant: string representation of the variant assignment 166 | * durationMS: duration in ms that value resolution took 167 | * error: exception raised during resolution if any 168 | */ 169 | await this.axios.post('/treatment', { 170 | 'run_id': runID, 171 | 'user_id': userID, 172 | 'variant': variant, 173 | 'error': error, 174 | 'duration_ms': durationMS, 175 | }); 176 | } 177 | 178 | public async observe( 179 | userID: string, 180 | metric: string, 181 | value: number | boolean, 182 | createdTime: string | undefined=undefined, 183 | ) { 184 | /* 185 | * Publishes a metric measurement event for a given user. 186 | * 187 | * userID: identifier of the user who originated this request. 188 | * metric: name of the metric being observed 189 | * value: numeric representation of the value measured 190 | * 191 | */ 192 | if (typeof(value) === 'boolean') { 193 | value = Number(value); 194 | } 195 | await this.axios.post('/measurement', { 196 | 'user_id': userID, 197 | 'metric_name': metric, 198 | 'value': value, 199 | 'created_time': createdTime, 200 | }); 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /clients/python/tests/test_scrutinize_client.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import time 3 | import unittest 4 | from decimal import Decimal 5 | from unittest.mock import AsyncMock, MagicMock, ANY 6 | 7 | from scrutinize import ScrutinizeClient 8 | 9 | 10 | class TestExperiment(unittest.TestCase): 11 | def test_call(self): 12 | scrutinize = ScrutinizeClient() 13 | scrutinize.get_experiments = AsyncMock(return_value={ 14 | 'testexp': {'run_id': 3, 'percentage': 50, 'active': True}, 15 | }) 16 | scrutinize.create_treatment = AsyncMock() 17 | 18 | control = MagicMock(return_value=5) 19 | variant = MagicMock(return_value=10) 20 | 21 | tests = [ 22 | ['lisaa', False, 5, control], 23 | ['garyoi', True, 10, variant], 24 | ['eyjohn', False, 5, control], 25 | ['joan', True, 10, variant], 26 | ] 27 | 28 | for i, (user_id, is_experiment, expected, var) in enumerate(tests): 29 | call_is_experiment, result = asyncio.run(scrutinize.call( 30 | 'testexp', 31 | user_id, 32 | lambda: control(i), 33 | lambda: variant(i), 34 | )) 35 | assert call_is_experiment == is_experiment 36 | assert result == expected 37 | 38 | var.assert_called_with(i) 39 | 40 | def test_call_exception(self): 41 | scrutinize = ScrutinizeClient() 42 | scrutinize.get_experiments = AsyncMock(return_value = { 43 | 'testexp': {'run_id': 3, 'percentage': 50, 'active': True}, 44 | }) 45 | scrutinize._get_variant = AsyncMock(return_value=(False, 'control')) 46 | scrutinize.create_treatment = AsyncMock() 47 | 48 | def exception_raiser(): 49 | raise(Exception('im an error')) 50 | 51 | try: 52 | asyncio.run(scrutinize.call( 53 | 'testexp', 54 | 'fake_person', 55 | exception_raiser, 56 | 2, 57 | )) 58 | except: 59 | # expect it to raise, just want to make sure treatment created 60 | pass 61 | 62 | # We only really care about the error parameter 63 | scrutinize.create_treatment.assert_called_with( 64 | 3, 65 | 'fake_person', 66 | 'control', 67 | ANY, 68 | 'im an error', 69 | ) 70 | 71 | def test_call_duration(self): 72 | scrutinize = ScrutinizeClient() 73 | scrutinize.get_experiments = AsyncMock(return_value = { 74 | 'testexp': {'run_id': 3, 'percentage': 50, 'active': True}, 75 | }) 76 | scrutinize._get_variant = AsyncMock(return_value=(False, 'control')) 77 | scrutinize.create_treatment = AsyncMock() 78 | 79 | get_time = MagicMock(side_effect=[5, 10]) 80 | asyncio.run(scrutinize.call( 81 | 'testexp', 82 | 'fake_person', 83 | 1, 84 | 2, 85 | get_time, 86 | )) 87 | 88 | # We only really care about the duration_ms parameter 89 | scrutinize.create_treatment.assert_called_with( 90 | 3, 91 | 'fake_person', 92 | 'control', 93 | 5000, 94 | '', 95 | ) 96 | 97 | def test_resolve_func(self): 98 | scrutinize = ScrutinizeClient() 99 | assert asyncio.run(scrutinize._resolve(lambda: 26)) == 26 100 | 101 | def test_resolve_async(self): 102 | scrutinize = ScrutinizeClient() 103 | async def variant(): 104 | return 25 105 | assert asyncio.run(scrutinize._resolve(variant)) == 25 106 | 107 | def test_resolve_literal(self): 108 | tests = [ 109 | 0, 110 | 1, 111 | '1', 112 | 'harry', 113 | False, 114 | Decimal(52), 115 | None, 116 | ] 117 | scrutinize = ScrutinizeClient() 118 | 119 | for expected in tests: 120 | assert asyncio.run(scrutinize._resolve(expected)) == expected 121 | 122 | def test_get_variant(self): 123 | tests = [ 124 | ['jane', True], 125 | ['tom', False], 126 | ['jim', False], 127 | ['mary', False], 128 | ['lonnie', True], 129 | ['alice', False], 130 | ['gerard', True], 131 | ['fiona', True], 132 | ['bernard', True], 133 | ['jimson', False], 134 | ['limson', False], 135 | ['fiona', True], 136 | ] 137 | scrutinize = ScrutinizeClient() 138 | scrutinize.get_experiments = AsyncMock(return_value={ 139 | 'testexp': {'run_id': 3, 'percentage': 50, 'active': True}, 140 | }) 141 | 142 | for user_id, expected in tests: 143 | treatment, _ = asyncio.run(scrutinize._get_variant('testexp', user_id)) 144 | assert treatment == expected 145 | 146 | def test_get_variant_differs_with_experiment(self): 147 | scrutinize = ScrutinizeClient() 148 | scrutinize.get_experiments = AsyncMock(return_value={ 149 | 'testexp': {'run_id': 3, 'percentage': 50, 'active': True}, 150 | 'otherexp': {'run_id': 3, 'percentage': 50, 'active': True}, 151 | }) 152 | 153 | 154 | user_id = 'sally' 155 | assert asyncio.run(scrutinize._get_variant('testexp', user_id)) != asyncio.run(scrutinize._get_variant('otherexp', user_id)) 156 | 157 | def test_get_variant_active_param(self): 158 | scrutinize = ScrutinizeClient() 159 | scrutinize.get_experiments = AsyncMock(return_value={ 160 | 'testexp': {'run_id': 3, 'percentage': 50, 'active': True}, 161 | 'otherexp': {'run_id': 3, 'percentage': 50, 'active': False}, 162 | }) 163 | 164 | # id lands in exp group for both experiments 165 | user_id='laura' 166 | assert asyncio.run(scrutinize._get_variant('testexp', user_id))[0] 167 | assert not asyncio.run(scrutinize._get_variant('otherexp', user_id))[0] 168 | 169 | 170 | def test_get_variant_experiment_doesnt_exist(self): 171 | scrutinize = ScrutinizeClient() 172 | scrutinize.get_experiments = AsyncMock(return_value={ 173 | 'SOME_OTHER_EXP': {'run_id': 3, 'percentage': 50, 'active': True}, 174 | }) 175 | assert asyncio.run(scrutinize._get_variant('testexp', 'johnny')) == (False, 'control') 176 | 177 | def test_get_experiments_use_cached(self): 178 | scrutinize = ScrutinizeClient() 179 | scrutinize.experiments = {'blah': {'name': 'blah'}} 180 | scrutinize.experiments_ttl = 1000000 181 | scrutinize.experiments_pull_time = time.time() 182 | scrutinize.get = AsyncMock(return_value=[{'name': 'hi'}]) 183 | result = asyncio.run(scrutinize.get_experiments()) 184 | 185 | assert result == {'blah': {'name': 'blah'}} 186 | assert result == scrutinize.experiments 187 | scrutinize.get.assert_not_called() 188 | 189 | def test_get_experiments_pull_on_none(self): 190 | scrutinize = ScrutinizeClient() 191 | scrutinize.experiments = None 192 | scrutinize.experiments_ttl = 1000000 193 | scrutinize.experiments_pull_time = time.time() 194 | scrutinize.get = AsyncMock(return_value=[{'name': 'hi'}]) 195 | result = asyncio.run(scrutinize.get_experiments()) 196 | 197 | assert result == {'hi': {'name': 'hi'}} 198 | assert result == scrutinize.experiments 199 | scrutinize.get.assert_called_with('/experiment') 200 | 201 | def test_get_experiments_timeout(self): 202 | scrutinize = ScrutinizeClient() 203 | scrutinize.experiments = {'blah': {'name': 'blah'}} 204 | scrutinize.experiments_ttl = 500 205 | scrutinize.experiments_pull_time = time.time() - 600 206 | scrutinize.get = AsyncMock(return_value=[{'name': 'hi'}]) 207 | result = asyncio.run(scrutinize.get_experiments()) 208 | 209 | assert result == {'hi': {'name': 'hi'}, 'blah': {'name': 'blah'}} 210 | assert result == scrutinize.experiments 211 | scrutinize.get.assert_called_with('/experiment') 212 | 213 | 214 | if __name__ == '__main__': 215 | unittest.main() 216 | -------------------------------------------------------------------------------- /clients/python/scrutinize/client.py: -------------------------------------------------------------------------------- 1 | import aiohttp 2 | import asyncio 3 | import datetime 4 | import hashlib 5 | import time 6 | 7 | from abc import ABC, abstractmethod 8 | from typing import Callable, Optional, Union 9 | 10 | 11 | class ScrutinizeClient: 12 | def __init__( 13 | self, 14 | api_url: str='http://localhost:11771', 15 | experiments_ttl: int=300, 16 | ): 17 | self.base_url = f'{api_url}/api/v1' 18 | self.experiments = None 19 | self.experiments_ttl = experiments_ttl 20 | self.experiments_pull_time = 0 21 | 22 | async def call( 23 | self, 24 | experiment_name: str, 25 | user_id: str, 26 | control: Callable, 27 | experiment: Callable, 28 | get_time: Callable=time.time, 29 | ): 30 | """ 31 | Primary entry point for an experiment. Behaves as follows: 32 | 33 | 1. Given a user_id and experiment, determines whether the user should 34 | get the control or experiment behavior. 35 | 2. Attempts to resolve the control or variant - can be functional 36 | or a literal value. 37 | 3. After resolution is complete or has failed, records the treatment 38 | event to the server. 39 | 40 | :param experiment_name: name of experiment associated with the call 41 | :param user_id: identifying of the user who originated this request 42 | :param control: function or literal value representing the control behavior 43 | :param experiment: function or literal value representing the experiment behavior 44 | :param get_time: method for getting the current time (overriden in testing) 45 | :return: whether the user was in the experiment group 46 | :return: either the control or experiment resolved value 47 | """ 48 | is_experiment, variant_str = await self._get_variant(experiment_name, user_id) 49 | variant = experiment if is_experiment else control 50 | start_time = get_time() 51 | err = None 52 | try: 53 | return is_experiment, await self._resolve(variant) 54 | except Exception as e: 55 | err = e 56 | raise e 57 | finally: 58 | duration_ms = (get_time() - start_time) * 1000 59 | await self.create_treatment( 60 | await self.get_run_id(experiment_name), 61 | user_id, 62 | variant_str, 63 | duration_ms, 64 | '' if err is None else str(err), 65 | ) 66 | 67 | async def _get_variant(self, experiment_name: str, user_id: str): 68 | """ 69 | Determines whether the user is in the experiment group. 70 | 71 | :param experiment_name: name of experiment associated with the call 72 | :param user_id: identifying of the user who originated this request 73 | :return: whether the user was in the experiment group 74 | :return: a string representing the variant assignment 75 | """ 76 | variant_operand = (experiment_name + user_id).encode('utf-8') 77 | experiments = await self.get_experiments() 78 | experiment_config = experiments.get(experiment_name, None) 79 | is_experiment = False 80 | 81 | if experiment_config is None: 82 | print('ScrutinizeClient._get_variant: experiment not found, only control will be run') 83 | elif experiment_config.get('active', False): 84 | # convert id to a number between 0 and 99 85 | id_int = int(hashlib.md5(variant_operand).hexdigest(), 16) % 100 86 | is_experiment = id_int < experiment_config['percentage'] 87 | 88 | return is_experiment, 'experiment' if is_experiment else 'control' 89 | 90 | async def get_run_id(self, experiment_name: str) -> Optional[int]: 91 | experiments = await self.get_experiments() 92 | experiment_config = experiments.get(experiment_name, None) 93 | return experiment_config.get('run_id', None) if experiment_config else None 94 | 95 | @staticmethod 96 | async def _resolve( 97 | variant: any 98 | ) -> any: 99 | """ 100 | Resolves an experiment behavior to a value. 101 | 102 | :param variant: a function or literal value representing the variant behavior 103 | :return: the resolved variant value 104 | """ 105 | if callable(variant): 106 | if asyncio.iscoroutinefunction(variant): 107 | return await variant() 108 | else: 109 | return variant() 110 | return variant 111 | 112 | async def get_experiments(self): 113 | """ 114 | Returns the experiment configuration stored in the server. 115 | Has simple caching behavior. 116 | 117 | :return: a dictionary representing all experiment configurations 118 | """ 119 | now = int(time.time()) 120 | should_pull = self.experiments is None or (now - self.experiments_pull_time > self.experiments_ttl) 121 | if should_pull: 122 | experiments = await self.get('/experiment') 123 | 124 | if self.experiments is None: 125 | self.experiments = {} 126 | for experiment in experiments: 127 | self.experiments[experiment['name']] = experiment 128 | 129 | self.experiments_pull_time = now 130 | return self.experiments 131 | 132 | async def create_treatment( 133 | self, 134 | run_id: int, 135 | user_id: str, 136 | variant: str, 137 | duration_ms: float, 138 | error: Optional[Exception], 139 | ): 140 | """ 141 | Publishes a treatment event to the server. 142 | 143 | :param experiment_name: name of experiment associated with the call 144 | :param user_id: identifying of the user who originated this request 145 | :param variant: string representation of the variant assignment 146 | :param duration_ms: duration in ms that value resolution took 147 | :param error: exception raised during resolution if any 148 | """ 149 | err_str = '' if error is None else str(error) 150 | await self.post('/treatment', { 151 | 'run_id': run_id, 152 | 'user_id': user_id, 153 | 'variant': variant, 154 | 'error': err_str, 155 | 'duration_ms': duration_ms, 156 | }) 157 | 158 | async def observe( 159 | self, 160 | user_id: str, 161 | metric: str, 162 | value: Union[int, float, bool], 163 | created_time: str=None, 164 | ): 165 | """ 166 | Publishes a metric measurement event for a given user. 167 | 168 | :param user_id: identifying of the user who originated this request 169 | :param metric: name of the metric being observed 170 | :param value: numeric representation of the value measured 171 | :param created_time: optional timestamp for the measurement 172 | """ 173 | if type(value) == bool: 174 | value = int(value) 175 | if not created_time: 176 | created_time = datetime.datetime.now().isoformat() 177 | await self.post('/measurement', { 178 | 'user_id': user_id, 179 | 'metric_name': metric, 180 | 'value': value, 181 | 'created_time': created_time, 182 | }) 183 | 184 | async def get( 185 | self, 186 | path: str, 187 | ): 188 | """ 189 | Helper for making HTTP GET requests 190 | """ 191 | async with aiohttp.ClientSession() as session: 192 | async with session.get(f'{self.base_url}{path}') as r: 193 | return await r.json() 194 | 195 | async def post( 196 | self, 197 | path: str, 198 | data: any, 199 | ): 200 | """ 201 | Helper for making HTTP POST requests 202 | """ 203 | async with aiohttp.ClientSession() as session: 204 | async with session.post(f'{self.base_url}{path}', json=data) as r: 205 | return await r.json() 206 | 207 | async def alive(self): 208 | """ 209 | Determines whether the client can communicate with the server. 210 | 211 | :return: bool if client <-> server communication is possible 212 | """ 213 | try: 214 | return (await self.get('/alive')).get('status', None) == 'ok' 215 | except: 216 | return False 217 | -------------------------------------------------------------------------------- /clients/javascript/scrutinize_test.ts: -------------------------------------------------------------------------------- 1 | import assert from 'assert'; 2 | 3 | import { AxiosInstance } from 'axios'; 4 | 5 | import ScrutinizeClient from './scrutinize'; 6 | 7 | class MockAxios{ 8 | get() { 9 | return {}; 10 | } 11 | post() { 12 | return {}; 13 | } 14 | } 15 | 16 | const mockAxios = new MockAxios() as unknown as AxiosInstance; 17 | 18 | async function testCall() { 19 | const scrutinize = new ScrutinizeClient(); 20 | scrutinize.axios = mockAxios; 21 | scrutinize.getExperiments = async () => { 22 | return {'testexp': {'run_id': 3, 'name': 'testexp', 'percentage': 50, 'active': true}} 23 | }; 24 | 25 | const control = () => 5; 26 | const variant = () => 10; 27 | 28 | const tests = [ 29 | ['lisaa', false, 5], 30 | ['garyoi', true, 10], 31 | ['eyjohn', false, 5], 32 | ['joan', true, 10], 33 | ]; 34 | 35 | for (const [userID, expectedIsExperiment, expected] of tests) { 36 | const [isExperiment, result] = await scrutinize.call( 37 | 'testexp', 38 | userID as string, 39 | control, 40 | variant, 41 | ); 42 | assert(isExperiment === expectedIsExperiment); 43 | assert(result === expected); 44 | } 45 | } 46 | 47 | async function testCallCreateTreatment() { 48 | const scrutinize = new ScrutinizeClient(); 49 | scrutinize.axios = mockAxios; 50 | scrutinize.getExperiments = async () => { 51 | return {'testexp': {'run_id': 3, 'name': 'testexp', 'percentage': 50, 'active': true}} 52 | }; 53 | 54 | const control = () => 5; 55 | const variant = () => 10; 56 | 57 | scrutinize.createTreatment = async (runID: any, userID: any, variant: any, _: any, _2: string) => { 58 | assert(runID === 3); 59 | assert(userID === 'mary'); 60 | assert(variant === 'control'); 61 | } 62 | 63 | await scrutinize.call( 64 | 'testexp', 65 | 'mary', 66 | control, 67 | variant, 68 | ); 69 | } 70 | 71 | async function testCallException() { 72 | const scrutinize = new ScrutinizeClient(); 73 | scrutinize.getExperiments = async () => { 74 | return {'testexp': {'run_id': 3, 'name': 'testexp', 'percentage': 50, 'active': true}} 75 | }; 76 | scrutinize.getVariant = async () => [false, 'control']; 77 | var errstring = ''; 78 | scrutinize.createTreatment = async (_: any, _1: any, _2: any, _3: any, err: string) => { 79 | errstring = err; 80 | } 81 | 82 | const errorFunc = () => { throw Error('im an error') }; 83 | 84 | try { 85 | await scrutinize.call( 86 | 'testexp', 87 | 'fake_person', 88 | errorFunc, 89 | 2, 90 | ); 91 | } catch(e: any) {} 92 | // verify errstring set from call failure 93 | assert(errstring == 'im an error'); 94 | } 95 | 96 | async function testCallDuration() { 97 | const scrutinize = new ScrutinizeClient(); 98 | scrutinize.getExperiments = async () => { 99 | return {'testexp': {'run_id': 3, 'name': 'testexp', 'percentage': 50, 'active': true}} 100 | }; 101 | scrutinize.getVariant = async () => [false, 'control']; 102 | 103 | var durationMSSeen = 0; 104 | scrutinize.createTreatment = async (_: any, _1: any, _2: any, durationMS: number, _4: any) => { 105 | durationMSSeen = durationMS; 106 | } 107 | 108 | var times = [5000, 10000]; 109 | const getTime = () => { return times.shift() }; 110 | 111 | try { 112 | await scrutinize.call( 113 | 'testexp', 114 | 'fake_person', 115 | 1, 116 | 2, 117 | getTime, 118 | ); 119 | } catch(e: any) {} 120 | assert(durationMSSeen == 5000); 121 | } 122 | 123 | async function testResolveFunc() { 124 | const scrutinize = new ScrutinizeClient(); 125 | assert(await scrutinize.resolve(() => 26) === 26); 126 | } 127 | 128 | async function testResolveAsync() { 129 | const scrutinize = new ScrutinizeClient(); 130 | assert(await scrutinize.resolve(async () => 25) === 25); 131 | } 132 | 133 | async function testResolveLiteral() { 134 | const tests = [ 135 | 0, 136 | 1, 137 | '1', 138 | 'harry', 139 | false, 140 | BigInt(52), 141 | null, 142 | ]; 143 | const scrutinize = new ScrutinizeClient(); 144 | 145 | for (const expected of tests) { 146 | assert(await scrutinize.resolve(expected) === expected); 147 | } 148 | } 149 | 150 | async function testGetVariant() { 151 | const tests = [ 152 | ['jane', true], 153 | ['tom', false], 154 | ['jim', false], 155 | ['mary', false], 156 | ['lonnie', true], 157 | ['alice', false], 158 | ['gerard', true], 159 | ['fiona', true], 160 | ['bernard', true], 161 | ['jimson', false], 162 | ['limson', false], 163 | ['fiona', true], 164 | ]; 165 | const scrutinize = new ScrutinizeClient(); 166 | scrutinize.getExperiments = async () => { 167 | return {'testexp': {'run_id': 3, 'name': 'testexp', 'percentage': 50, 'active': true}} 168 | }; 169 | 170 | for (const [userID, expectedIsExperiment] of tests) { 171 | const results = await scrutinize.getVariant('testexp', userID as string); 172 | assert(results[0] === expectedIsExperiment); 173 | } 174 | } 175 | 176 | async function testGetVariantDiffersByExperiment() { 177 | const scrutinize = new ScrutinizeClient(); 178 | scrutinize.getExperiments = async () => { 179 | return { 180 | 'testexp': {'run_id': 3,'name': 'testexp', 'percentage': 50, 'active': true}, 181 | 'otherexp': {'run_id': 3,'name': 'otherexp', 'percentage': 50, 'active': true}, 182 | }; 183 | }; 184 | const userID = 'sally'; 185 | assert(!(await scrutinize.getVariant('testexp', userID))[0]); 186 | assert((await scrutinize.getVariant('otherexp', userID))[0]); 187 | } 188 | 189 | async function testGetVariantActiveParam() { 190 | const scrutinize = new ScrutinizeClient(); 191 | scrutinize.getExperiments = async () => { 192 | return { 193 | 'testexp': {'run_id': 3, 'name': 'testexp', 'percentage': 50, 'active': true}, 194 | 'otherexp': {'run_id': 3, 'name': 'otherexp', 'percentage': 50, 'active': false}, 195 | }; 196 | }; 197 | const userID = 'sally'; 198 | assert(!(await scrutinize.getVariant('testexp', userID))[0]); 199 | assert(!(await scrutinize.getVariant('otherexp', userID))[0]); 200 | } 201 | 202 | async function testGetVariantExperimentDoesntExist() { 203 | const scrutinize = new ScrutinizeClient(); 204 | scrutinize.getExperiments = async () => { 205 | return { 206 | 'testexp': {'run_id': 3, 'name': 'testexp', 'percentage': 50, 'active': true}, 207 | }; 208 | }; 209 | const userID = 'sally'; 210 | assert(!(await scrutinize.getVariant('UNKNOWNEXPERIMENT', userID))[0]); 211 | } 212 | 213 | async function testGetExperimentsUseCached() { 214 | const scrutinize = new ScrutinizeClient(); 215 | scrutinize.experiments = {'blah': { 216 | run_id: 3, 217 | name: 'blah', 218 | percentage: 50, 219 | active: true, 220 | }}; 221 | scrutinize.experimentsTTL = 1000000; 222 | scrutinize.experimentsPullTime = Date.now(); 223 | scrutinize.axios = new MockAxios() as unknown as AxiosInstance; 224 | scrutinize.axios.get = async () => { 225 | return {data: [{'name': 'hi'}]} as any; 226 | };; 227 | 228 | const result = await scrutinize.getExperiments(); 229 | // make sure pull didnt execute 230 | assert(Object.keys(result).length === 1); 231 | assert(Object.keys(scrutinize.experiments as Object).length === 1); 232 | assert(result['blah']); 233 | } 234 | 235 | async function testGetExperimentsPullOnNone() { 236 | const scrutinize = new ScrutinizeClient(); 237 | scrutinize.experimentsTTL = 1000000; 238 | scrutinize.experimentsPullTime = Date.now() / 1000; 239 | scrutinize.axios = new MockAxios() as unknown as AxiosInstance; 240 | scrutinize.axios.get = async () => { 241 | return {data: [{'name': 'hi'}]} as any; 242 | }; 243 | 244 | const result = await scrutinize.getExperiments(); 245 | // make sure pull didnt execute 246 | assert(Object.keys(result).length === 1); 247 | assert(Object.keys(scrutinize.experiments as Object).length === 1); 248 | assert(result['hi']); 249 | } 250 | 251 | async function testGetExperimentsTimeout() { 252 | const scrutinize = new ScrutinizeClient(); 253 | scrutinize.experiments = {'blah': { 254 | run_id: 3, 255 | name: 'blah', 256 | percentage: 50, 257 | active: true, 258 | }}; 259 | scrutinize.experimentsTTL = 10; 260 | scrutinize.experimentsPullTime = 0; 261 | scrutinize.axios = new MockAxios() as unknown as AxiosInstance; 262 | scrutinize.axios.get = async () => { 263 | return {data: [{'name': 'hi'}]} as any; 264 | }; 265 | 266 | const result = await scrutinize.getExperiments(); 267 | // make sure pull didnt execute 268 | assert(Object.keys(result).length === 2); 269 | assert(Object.keys(scrutinize.experiments as Object).length === 2); 270 | assert(result['blah']); 271 | assert(result['hi']); 272 | } 273 | 274 | async function testScrutinize() { 275 | testCall(); 276 | testCallCreateTreatment(); 277 | testCallException(); 278 | testCallDuration(); 279 | testResolveFunc(); 280 | testResolveAsync(); 281 | testResolveLiteral(); 282 | testGetVariant(); 283 | testGetVariantDiffersByExperiment(); 284 | testGetVariantActiveParam(); 285 | testGetVariantExperimentDoesntExist(); 286 | testGetExperimentsUseCached(); 287 | testGetExperimentsPullOnNone(); 288 | testGetExperimentsTimeout(); 289 | } 290 | 291 | testScrutinize(); 292 | --------------------------------------------------------------------------------