├── .README ├── README.md ├── RECIPES.md ├── SPECIFICATION.md ├── USAGE.md └── dashboard.png ├── .babelrc ├── .editorconfig ├── .eslintignore ├── .eslintrc ├── .flowconfig ├── .gitignore ├── .npmignore ├── .travis.yml ├── LICENSE ├── README.md ├── package.json ├── src ├── Logger.js ├── assertions │ ├── assertUniqueTestIdPayloads.js │ └── index.js ├── bin │ ├── commands │ │ ├── .eslintrc │ │ ├── alert.js │ │ ├── monitor.js │ │ ├── report.js │ │ └── test.js │ └── index.js ├── errors.js ├── factories │ ├── createAlertController.js │ ├── createGraphqlClient.js │ ├── createGraphqlMiddleware.js │ ├── createIntervalRoutine.js │ ├── createLabelCollection.js │ ├── createMonitor.js │ ├── createResponseBody.js │ ├── createTestId.js │ ├── createTestIdPayload.js │ ├── createTestPointer.js │ ├── createWebpackConfiguration.js │ └── index.js ├── index.js ├── report-web-app │ ├── .babelrc │ ├── .eslintrc │ ├── components │ │ ├── FailingTestComponent.js │ │ └── TextButtonComponent.js │ ├── config.js │ ├── containers │ │ ├── Root.js │ │ └── TestFilter.js │ ├── factories │ │ ├── createLabelsObject.js │ │ └── index.js │ ├── index.js │ ├── main.scss │ ├── postcss.config.js │ ├── services.js │ ├── types.js │ └── webpack.configuration.production.js ├── resolvers │ ├── Query.js │ ├── RegisteredTest.js │ └── index.js ├── routines │ ├── evaluateRegisteredTest.js │ ├── explainTest.js │ └── index.js ├── schema.graphql ├── types.js └── utilities │ ├── importModule.js │ ├── index.js │ ├── localeCompareProperty.js │ └── resolveFilePathExpression.js └── test ├── .eslintrc └── palantir ├── assertions └── assertUniqueTestIdPayloads.js ├── factories ├── createAlertController.js ├── createIntervalRoutine.js └── createLabelCollection.js └── routines └── evaluateRegisteredTest.js /.README/README.md: -------------------------------------------------------------------------------- 1 | # Palantir 2 | 3 | [![Travis build status](http://img.shields.io/travis/gajus/palantir/master.svg?style=flat-square)](https://travis-ci.org/gajus/palantir) 4 | [![Coveralls](https://img.shields.io/coveralls/gajus/palantir.svg?style=flat-square)](https://coveralls.io/github/gajus/palantir) 5 | [![NPM version](http://img.shields.io/npm/v/palantir.svg?style=flat-square)](https://www.npmjs.org/package/palantir) 6 | [![Canonical Code Style](https://img.shields.io/badge/code%20style-canonical-blue.svg?style=flat-square)](https://github.com/gajus/canonical) 7 | [![Twitter Follow](https://img.shields.io/twitter/follow/kuizinas.svg?style=social&label=Follow)](https://twitter.com/kuizinas) 8 | 9 | Active monitoring and alerting system using user-defined Node.js scripts. 10 | 11 | ![Dashboard screenshot](./.README/dashboard.png) 12 | 13 | ## Features 14 | 15 | * Programmatic test cases (write your own checks using Node.js). (🔥 ready) 16 | * Programmatic troubleshooting (write your own troubleshooting queries for test cases). (🔥 ready) 17 | * Programmatic notifications (write your own mechanism for sending notifications). (🔥 ready) 18 | * Filter tests using MongoDB-like queries. (🔥 ready) 19 | * Track historical health of individual tests. (🗺️ in roadmap) 20 | * Create browser-session specific dashboards. (🗺️ in roadmap) 21 | * Produce charts using troubleshooting output. (🗺️ in roadmap) 22 | * Hosted Palantir instance with tests run using serverless infrastructure, persistent dashboards, integrated timeseries database and notifcations. (💵 commercial) (🗺️ in roadmap) 23 | 24 | ## Contents 25 | 26 | {"gitdown": "contents"} 27 | 28 | ## Motivation 29 | 30 | Existing monitoring software primarily focuses on enabling visual inspection of service health metrics and relies on service maintainers to detect anomalies. This approach is time consuming and allows for human-error. Even when monitoring systems allow to define alerts based on pre-defined thresholds, a point-in-time metric is not sufficient to determine service-health. The only way to establish service-health is to write thorough integration tests (scripts) and automate their execution, just like we do in software-development. 31 | 32 | Palantir continuously performs user-defined tests and only reports failing tests, i.e. if everything is working as expected, the system remains silent. This allows service developers/maintainers to focus on defining tests that warn about the errors that are about to occur and automate troubleshooting. 33 | 34 | Palantir decouples monitoring, alerting and reporting mechanisms. This method allows distributed monitoring and role-based, tag-based alerting system architecture. 35 | 36 | ### Further reading 37 | 38 | * [Ensuring good service health by automating thorough integration testing and alerting](https://medium.com/@gajus/d507572a2618) 39 | 40 | {"gitdown": "include", "file": "./USAGE.md"} 41 | {"gitdown": "include", "file": "./SPECIFICATION.md"} 42 | {"gitdown": "include", "file": "./RECIPES.md"} 43 | 44 | ## Development 45 | 46 | There are multiple components required to run the service. 47 | 48 | Run `npm run dev` to watch the project and re-build upon detecting a change. 49 | 50 | In order to observe project changes and restart all the services use a program such as [`nodemon`](https://www.npmjs.com/package/nodemon), e.g. 51 | 52 | ```bash 53 | $ NODE_ENV=development nodemon --watch dist --ext js,graphql dist/bin/index.js monitor ... 54 | $ NODE_ENV=development nodemon --watch dist --ext js,graphql dist/bin/index.js alert ... 55 | 56 | ``` 57 | 58 | Use `--watch` attribute multiple times to include Palantir project code and your configuration/ test scripts. 59 | 60 | `report` program run in `NODE_ENV=development` use `webpack-hot-middleware` to implement hot reloading. 61 | 62 | ```bash 63 | $ NODE_ENV=development babel-node src/bin/index.js report --service-port 8081 --api-url http://127.0.0.1:8080/ | roarr pretty-print 64 | 65 | ``` 66 | -------------------------------------------------------------------------------- /.README/RECIPES.md: -------------------------------------------------------------------------------- 1 | ## Recipes 2 | 3 | ### Asynchronously creating a test suite 4 | 5 | Creating a test suite might require to query an asynchronous source, e.g. when information required to create a test suite is stored in a database. In this case, a test suite factory can return a promise that resolves with a test suite, e.g. 6 | 7 | ```js 8 | const createTestSuite: TestSuiteFactoryType = async () => { 9 | const clients = await getClients(connection); 10 | 11 | return clients.map((client) => { 12 | return { 13 | assert: async () => { 14 | await request(client.url, { 15 | timeout: interval('10 seconds') 16 | }); 17 | }, 18 | interval: createIntervalCreator(interval('30 seconds')), 19 | labels: { 20 | 'client.country': client.country, 21 | 'client.id': client.id, 22 | source: 'http', 23 | type: 'liveness-check' 24 | }, 25 | name: client.url + ' responds with 200' 26 | }; 27 | }); 28 | }; 29 | 30 | ``` 31 | 32 | In the above example, `getClients` is used to asynchronously retrieve information required to construct the test suite. 33 | 34 | ### Refreshing a test suit 35 | 36 | It might be desired that the test suite itself informs the monitor about new tests, e.g. the example in the [asynchronously creating a test suite](#asynchronously-creating-a-test-suite) recipe retrieves information from an external datasource that may change over time. In this case, a test suite factory can inform the `monitor` program that it should recreate the test suite, e.g. 37 | 38 | ```js 39 | const createTestSuite: TestSuiteFactoryType = async (refreshTestSuite) => { 40 | const clients = await getClients(connection); 41 | 42 | (async () => { 43 | // Some logic used to determine when the `clients` data used 44 | // to construct the original test suite becomes stale. 45 | while (true) { 46 | await delay(interval('10 seconds')); 47 | 48 | if (JSON.stringify(clients) !== JSON.stringify(await getClients(connection))) { 49 | // Calling `refreshTestSuite` will make Palantir monitor program 50 | // recreate the test suite using `createTestSuite`. 51 | refreshTestSuite(); 52 | 53 | break; 54 | } 55 | } 56 | })(); 57 | 58 | return clients.map((client) => { 59 | return { 60 | assert: async () => { 61 | await request(client.url, { 62 | timeout: interval('10 seconds') 63 | }); 64 | }, 65 | interval: createIntervalCreator(interval('30 seconds')), 66 | labels: { 67 | 'client.country': client.country, 68 | 'client.id': client.id, 69 | source: 'http', 70 | type: 'liveness-check' 71 | }, 72 | name: client.url + ' responds with 200' 73 | }; 74 | }); 75 | }; 76 | 77 | ``` 78 | -------------------------------------------------------------------------------- /.README/SPECIFICATION.md: -------------------------------------------------------------------------------- 1 | ## Specification 2 | 3 | ### Palantir test 4 | 5 | Palantir test is an object with the following properties: 6 | 7 | ```js 8 | /** 9 | * @property assert Evaluates user defined script. The result (boolean) indicates if test is passing. 10 | * @property configuration User defined configuration accessible by the `beforeTest`. 11 | * @property explain Provides debugging information about the test. 12 | * @property interval A function that describes the time when the test needs to be re-run. 13 | * @property labels Arbitrary key=value labels used to categorise the tests. 14 | * @property name Unique name of the test. A combination of test + labels must be unique across all test suites. 15 | * @property priority A numeric value (0-100) indicating the importance of the test. Low value indicates high priority. 16 | */ 17 | type TestType = {| 18 | +assert: (context: TestContextType) => Promise, 19 | +configuration?: SerializableObjectType, 20 | +explain?: (context: TestContextType) => Promise<$ReadOnlyArray>, 21 | +interval: (consecutiveFailureCount: number) => number, 22 | +labels: LabelsType, 23 | +name: string, 24 | +priority: number 25 | |}; 26 | 27 | ``` 28 | 29 | In practice, an example of a test used to check whether HTTP resource is available could look like this: 30 | 31 | ```js 32 | { 33 | assert: async () => { 34 | await request('https://applaudience.com/', { 35 | timeout: interval('10 seconds') 36 | }); 37 | }, 38 | interval: () => { 39 | return interval('30 seconds'); 40 | }, 41 | labels: { 42 | project: 'applaudience', 43 | source: 'http', 44 | type: 'liveness-check' 45 | }, 46 | name: 'https://applaudience.com/ responds with 200' 47 | } 48 | 49 | ``` 50 | 51 | ### Palantir test suite 52 | 53 | `monitor` program requires a list of file paths as an input. Every input file must export a function that creates a `TestSuiteType` object: 54 | 55 | ```js 56 | type TestSuiteType = {| 57 | +tests: $ReadOnlyArray 58 | |}; 59 | 60 | ``` 61 | 62 | Example: 63 | 64 | ```js 65 | // @flow 66 | 67 | import request from 'axios'; 68 | import interval from 'human-interval'; 69 | import type { 70 | TestSuiteFactoryType 71 | } from 'palantir'; 72 | 73 | const createIntervalCreator = (intervalTime) => { 74 | return () => { 75 | return intervalTime; 76 | }; 77 | }; 78 | 79 | const createTestSuite: TestSuiteFactoryType = () => { 80 | return { 81 | tests: [ 82 | { 83 | assert: async () => { 84 | await request('https://applaudience.com/', { 85 | timeout: interval('10 seconds') 86 | }); 87 | }, 88 | interval: createIntervalCreator(interval('30 seconds')), 89 | labels: { 90 | project: 'applaudience', 91 | scope: 'http' 92 | }, 93 | name: 'https://applaudience.com/ responds with 200' 94 | } 95 | ] 96 | } 97 | }; 98 | 99 | export default createTestSuite; 100 | 101 | ``` 102 | 103 | Note that the test suite factory may return a promise. Refer to [asynchronously creating a test suite](#asynchronously-creating-a-test-suite) for a use case example. 104 | 105 | ### Monitor configuration 106 | 107 | Palantir `monitor` program accepts `configuration` configuration (a path to a script). 108 | 109 | ```js 110 | /** 111 | * @property after Called when shutting down the monitor. 112 | * @property afterTest Called after every test. 113 | * @property before Called when starting the monitor. 114 | * @property beforeTest Called before every test. 115 | */ 116 | type ConfigurationType = {| 117 | +after: () => Promise, 118 | +afterTest?: (test: RegisteredTestType, context?: TestContextType) => Promise, 119 | +before: () => Promise, 120 | +beforeTest?: (test: RegisteredTestType) => Promise 121 | |}; 122 | 123 | ``` 124 | 125 | The configuration script allows to setup hooks for different stages of the program execution. 126 | 127 | In practice, this can be used to configure the database connection, e.g. 128 | 129 | ```js 130 | import { 131 | createPool 132 | } from 'slonik'; 133 | 134 | let pool; 135 | 136 | export default { 137 | afterTest: async (test, context) => { 138 | await context.connection.release(); 139 | }, 140 | before: async () => { 141 | pool = await createPool('postgres://'); 142 | }, 143 | beforeTest: async () => { 144 | const connection = await pool.connect(); 145 | 146 | return { 147 | connection 148 | }; 149 | } 150 | }; 151 | 152 | ``` 153 | 154 | Note that in the above example, unless you are using database connection for all the tests, you do not want to allocate a connection for every test. You can restrict allocation of connection using test configuration, e.g. 155 | 156 | Test that requires connection to the database: 157 | 158 | ```js 159 | { 160 | assert: (context) => { 161 | return context.connection.any('SELECT 1'); 162 | }, 163 | configuration: { 164 | database: true 165 | }, 166 | interval: () => { 167 | return interval('30 seconds'); 168 | }, 169 | labels: { 170 | scope: 'database' 171 | }, 172 | name: 'connects to the database' 173 | } 174 | 175 | ``` 176 | 177 | Monitor configuration that is aware of the `configuration.database` configuration. 178 | 179 | ```js 180 | import { 181 | createPool 182 | } from 'slonik'; 183 | 184 | let pool; 185 | 186 | export default { 187 | afterTest: async (test, context) => { 188 | if (!test.configuration.database) { 189 | return; 190 | } 191 | 192 | await context.connection.release(); 193 | }, 194 | before: async () => { 195 | pool = await createPool('postgres://'); 196 | }, 197 | beforeTest: async (test) => { 198 | if (!test.configuration.database) { 199 | return {}; 200 | } 201 | 202 | const connection = await pool.connect(); 203 | 204 | return { 205 | connection 206 | }; 207 | } 208 | }; 209 | 210 | ``` 211 | 212 | ### Alert configuration 213 | 214 | Palantir `alert` program accepts `configuration` configuration (a path to a script). 215 | 216 | ```js 217 | /** 218 | * @property onNewFailingTest Called when a new test fails. 219 | * @property onRecoveredTest Called when a previously failing test is no longer failing. 220 | */ 221 | type AlertConfigurationType = {| 222 | +onNewFailingTest?: (registeredTest: RegisteredTestType) => void, 223 | +onRecoveredTest?: (registeredTest: RegisteredTestType) => void 224 | |}; 225 | 226 | ``` 227 | 228 | The alert configuration script allows to setup event handlers used to observe when tests fail and recover. 229 | 230 | In practice, this can be used to configure a system that notifies other systems about the failing tests, e.g. 231 | 232 | ```js 233 | /** 234 | * @file Using https://www.twilio.com/ to send a text message when tests fail and when tests recover. 235 | */ 236 | import createTwilio from 'twilio'; 237 | 238 | const twilio = createTwilio('ACCOUNT SID', 'AUTH TOKEN'); 239 | 240 | const sendMessage = (message) => { 241 | twilio.messages.create({ 242 | body: message, 243 | to: '+12345678901', 244 | from: '+12345678901' 245 | }); 246 | }; 247 | 248 | export default { 249 | onNewFailingTest: (test) => { 250 | sendMessage('FAILURE ' + test.name + ' failed'); 251 | }, 252 | onRecoveredTest: (test) => { 253 | sendMessage('RECOVERY ' + test.name + ' recovered'); 254 | } 255 | }; 256 | 257 | ``` 258 | 259 | The above example will send a message for every failure and recovery, every time failure/ recovery occurs. In practise, it is desired that the alerting system includes a mechanism to filter out temporarily failures. To address this requirement, Palantir implements an [alert controller](#alert-controller). 260 | 261 | ### Alert controller 262 | 263 | Palantir alert controller abstracts logic used to filter temporarily failures. 264 | 265 | `palantir` module exports a factory method `createAlertController` used to create an Palantir alert controller. 266 | 267 | ```js 268 | /** 269 | * @property delayFailure Returns test-specific number of milliseconds to wait before considering the test to be failing. 270 | * @property delayRecovery Returns test-specific number of milliseconds to wait before considering the test to be recovered. 271 | * @property onFailure Called when test is considered to be failing. 272 | * @property onRecovery Called when test is considered to be recovered. 273 | */ 274 | type ConfigurationType = {| 275 | +delayFailure: (test: RegisteredTestType) => number, 276 | +delayRecovery: (test: RegisteredTestType) => number, 277 | +onFailure: (test: RegisteredTestType) => void, 278 | +onRecovery: (test: RegisteredTestType) => void 279 | |}; 280 | 281 | type AlertControllerType = {| 282 | +getDelayedFailingTests: () => $ReadOnlyArray, 283 | +getDelayedRecoveringTests: () => $ReadOnlyArray, 284 | +registerTestFailure: (test: RegisteredTestType) => void, 285 | +registerTestRecovery: (test: RegisteredTestType) => void 286 | |}; 287 | 288 | createAlertController(configuration: ConfigurationType) => AlertControllerType; 289 | 290 | ``` 291 | 292 | Use `createAlertController` to implement alert throttling, e.g. 293 | 294 | ```js 295 | import interval from 'human-interval'; 296 | import createTwilio from 'twilio'; 297 | import { 298 | createAlertController 299 | } from 'palantir'; 300 | 301 | const twilio = createTwilio('ACCOUNT SID', 'AUTH TOKEN'); 302 | 303 | const sendMessage = (message) => { 304 | twilio.messages.create({ 305 | body: message, 306 | to: '+12345678901', 307 | from: '+12345678901' 308 | }); 309 | }; 310 | 311 | const controller = createAlertController({ 312 | delayFailure: (test) => { 313 | if (test.labels.scope === 'database') { 314 | return 0; 315 | } 316 | 317 | return interval('5 minute'); 318 | }, 319 | delayRecovery: () => { 320 | return interval('1 minute'); 321 | }, 322 | onFailure: (test) => { 323 | sendMessage('FAILURE ' + test.description + ' failed'); 324 | }, 325 | onRecovery: () => { 326 | sendMessage('RECOVERY ' + test.description + ' recovered'); 327 | } 328 | }); 329 | 330 | export default { 331 | onNewFailingTest: (test) => { 332 | controller.registerTestFailure(test); 333 | }, 334 | onRecoveredTest: (test) => { 335 | controller.registerTestRecovery(test); 336 | } 337 | }; 338 | 339 | ``` 340 | 341 | ### Palantir HTTP API 342 | 343 | Palantir `monitor` program creates HTTP GraphQL API server. The API exposes information about the user-registered tests and the failing tests. 344 | 345 | Refer to the [schema.graphql](./src/schema.graphql) or [introspect the API](https://graphql.org/learn/introspection/) to learn more. 346 | -------------------------------------------------------------------------------- /.README/USAGE.md: -------------------------------------------------------------------------------- 1 | ## Usage 2 | 3 | ### `monitor` program 4 | 5 | `monitor` program continuously performs user-defined tests and exposes the current state via [Palantir HTTP API](#palantir-http-api). 6 | 7 | ```bash 8 | $ palantir monitor --service-port 8080 --configuration ./monitor-configuration.js ./tests/**/* 9 | 10 | ``` 11 | 12 | Every test file must export a function that creates a `TestSuiteType` (see [Palantir test suite](#palantir-test-suite)). 13 | 14 | * Refer to [Palantir test](#palantir-test) specification. 15 | * Refer to [Monitor configuration](#monitor-configuration) specification. 16 | 17 | ### `alert` program 18 | 19 | `alert` program subscribes to [Palantir HTTP API](#palantir-http-api) and alerts other systems using user-defined configuration. 20 | 21 | ```bash 22 | $ palantir alert --configuration ./alert-configuration.js --api-url http://127.0.0.1:8080/ 23 | 24 | ``` 25 | 26 | * Refer to [Alert configuration](#alert-configuration) specification. 27 | 28 | ### `report` program 29 | 30 | `report` program creates a web UI for the [Palantir HTTP API](#palantir-http-api). 31 | 32 | ```bash 33 | $ palantir report --service-port 8081 --api-url http://127.0.0.1:8080/ 34 | 35 | ``` 36 | 37 | ### `test` program 38 | 39 | `test` program runs tests once. 40 | 41 | ```bash 42 | $ palantir test --configuration ./monitor-configuration.js ./tests/**/* 43 | 44 | ``` 45 | 46 | `test` program is used for test development. It allows to filter tests by description (`--match-description`) and by the test tags (`--match-tag`), e.g. 47 | 48 | ```bash 49 | $ palantir test --match-description 'event count is greater' --configuration ./monitor-configuration.js ./tests/**/* 50 | $ palantir test --match-tag 'database' --configuration ./monitor-configuration.js ./tests/**/* 51 | 52 | ``` 53 | -------------------------------------------------------------------------------- /.README/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gajus/palantir/7850545bd21c45173d2f9dcf0ac7a7421e827e07/.README/dashboard.png -------------------------------------------------------------------------------- /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "development": { 4 | "plugins": [ 5 | "react-hot-loader/babel" 6 | ] 7 | }, 8 | "test": { 9 | "plugins": [ 10 | "istanbul" 11 | ] 12 | } 13 | }, 14 | "plugins": [ 15 | "@babel/transform-flow-strip-types" 16 | ], 17 | "presets": [ 18 | [ 19 | "@babel/env", 20 | { 21 | "targets": { 22 | "node": "current" 23 | } 24 | } 25 | ] 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | indent_size = 2 7 | indent_style = space 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gajus/palantir/7850545bd21c45173d2f9dcf0ac7a7421e827e07/.eslintignore -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "canonical", 4 | "canonical/flowtype" 5 | ], 6 | "root": true, 7 | "rules": { 8 | "no-restricted-syntax": 0 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /.flowconfig: -------------------------------------------------------------------------------- 1 | [ignore] 2 | .*/node_modules/.*/test/.* 3 | /dist/.* 4 | 5 | [options] 6 | module.name_mapper='^.*\.scss$' -> 'css-module-flow' 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | coverage 2 | dist 3 | node_modules 4 | *.log 5 | .* 6 | !.babelrc 7 | !.editorconfig 8 | !.eslintignore 9 | !.eslintrc 10 | !.flowconfig 11 | !.gitignore 12 | !.npmignore 13 | !.README 14 | !.travis.yml 15 | /package-lock.json 16 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | src 2 | test 3 | coverage 4 | .* 5 | *.log 6 | !.static 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 10 4 | - node 5 | script: 6 | - npm run lint 7 | - npm run test 8 | - nyc --silent npm run test 9 | - nyc report --reporter=text-lcov | coveralls 10 | # - nyc check-coverage --lines 80 11 | after_success: 12 | - NODE_ENV=production npm run build 13 | - semantic-release 14 | notifications: 15 | email: false 16 | sudo: false 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018, Gajus Kuizinas (http://gajus.com/) 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above copyright 9 | notice, this list of conditions and the following disclaimer in the 10 | documentation and/or other materials provided with the distribution. 11 | * Neither the name of the Gajus Kuizinas (http://gajus.com/) nor the 12 | names of its contributors may be used to endorse or promote products 13 | derived from this software without specific prior written permission. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 18 | DISCLAIMED. IN NO EVENT SHALL ANUARY BE LIABLE FOR ANY 19 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 22 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # Palantir 3 | 4 | [![Travis build status](http://img.shields.io/travis/gajus/palantir/master.svg?style=flat-square)](https://travis-ci.org/gajus/palantir) 5 | [![Coveralls](https://img.shields.io/coveralls/gajus/palantir.svg?style=flat-square)](https://coveralls.io/github/gajus/palantir) 6 | [![NPM version](http://img.shields.io/npm/v/palantir.svg?style=flat-square)](https://www.npmjs.org/package/palantir) 7 | [![Canonical Code Style](https://img.shields.io/badge/code%20style-canonical-blue.svg?style=flat-square)](https://github.com/gajus/canonical) 8 | [![Twitter Follow](https://img.shields.io/twitter/follow/kuizinas.svg?style=social&label=Follow)](https://twitter.com/kuizinas) 9 | 10 | Active monitoring and alerting system using user-defined Node.js scripts. 11 | 12 | ![Dashboard screenshot](./.README/dashboard.png) 13 | 14 | 15 | ## Features 16 | 17 | * Programmatic test cases (write your own checks using Node.js). (🔥 ready) 18 | * Programmatic troubleshooting (write your own troubleshooting queries for test cases). (🔥 ready) 19 | * Programmatic notifications (write your own mechanism for sending notifications). (🔥 ready) 20 | * Filter tests using MongoDB-like queries. (🔥 ready) 21 | * Track historical health of individual tests. (🗺️ in roadmap) 22 | * Create browser-session specific dashboards. (🗺️ in roadmap) 23 | * Produce charts using troubleshooting output. (🗺️ in roadmap) 24 | * Hosted Palantir instance with tests run using serverless infrastructure, persistent dashboards, integrated timeseries database and notifcations. (💵 commercial) (🗺️ in roadmap) 25 | 26 | 27 | ## Contents 28 | 29 | * [Palantir](#palantir) 30 | * [Features](#palantir-features) 31 | * [Contents](#palantir-contents) 32 | * [Motivation](#palantir-motivation) 33 | * [Further reading](#palantir-motivation-further-reading) 34 | * [Usage](#palantir-usage) 35 | * [`monitor` program](#palantir-usage-monitor-program) 36 | * [`alert` program](#palantir-usage-alert-program) 37 | * [`report` program](#palantir-usage-report-program) 38 | * [`test` program](#palantir-usage-test-program) 39 | * [Specification](#palantir-specification) 40 | * [Palantir test](#palantir-specification-palantir-test) 41 | * [Palantir test suite](#palantir-specification-palantir-test-suite) 42 | * [Monitor configuration](#palantir-specification-monitor-configuration) 43 | * [Alert configuration](#palantir-specification-alert-configuration) 44 | * [Alert controller](#palantir-specification-alert-controller) 45 | * [Palantir HTTP API](#palantir-specification-palantir-http-api) 46 | * [Recipes](#palantir-recipes) 47 | * [Asynchronously creating a test suite](#palantir-recipes-asynchronously-creating-a-test-suite) 48 | * [Refreshing a test suit](#palantir-recipes-refreshing-a-test-suit) 49 | * [Development](#palantir-development) 50 | 51 | 52 | 53 | ## Motivation 54 | 55 | Existing monitoring software primarily focuses on enabling visual inspection of service health metrics and relies on service maintainers to detect anomalies. This approach is time consuming and allows for human-error. Even when monitoring systems allow to define alerts based on pre-defined thresholds, a point-in-time metric is not sufficient to determine service-health. The only way to establish service-health is to write thorough integration tests (scripts) and automate their execution, just like we do in software-development. 56 | 57 | Palantir continuously performs user-defined tests and only reports failing tests, i.e. if everything is working as expected, the system remains silent. This allows service developers/maintainers to focus on defining tests that warn about the errors that are about to occur and automate troubleshooting. 58 | 59 | Palantir decouples monitoring, alerting and reporting mechanisms. This method allows distributed monitoring and role-based, tag-based alerting system architecture. 60 | 61 | 62 | ### Further reading 63 | 64 | * [Ensuring good service health by automating thorough integration testing and alerting](https://medium.com/@gajus/d507572a2618) 65 | 66 | 67 | ## Usage 68 | 69 | 70 | ### monitor program 71 | 72 | `monitor` program continuously performs user-defined tests and exposes the current state via [Palantir HTTP API](#palantir-http-api). 73 | 74 | ```bash 75 | $ palantir monitor --service-port 8080 --configuration ./monitor-configuration.js ./tests/**/* 76 | 77 | ``` 78 | 79 | Every test file must export a function that creates a `TestSuiteType` (see [Palantir test suite](#palantir-test-suite)). 80 | 81 | * Refer to [Palantir test](#palantir-test) specification. 82 | * Refer to [Monitor configuration](#monitor-configuration) specification. 83 | 84 | 85 | ### alert program 86 | 87 | `alert` program subscribes to [Palantir HTTP API](#palantir-http-api) and alerts other systems using user-defined configuration. 88 | 89 | ```bash 90 | $ palantir alert --configuration ./alert-configuration.js --api-url http://127.0.0.1:8080/ 91 | 92 | ``` 93 | 94 | * Refer to [Alert configuration](#alert-configuration) specification. 95 | 96 | 97 | ### report program 98 | 99 | `report` program creates a web UI for the [Palantir HTTP API](#palantir-http-api). 100 | 101 | ```bash 102 | $ palantir report --service-port 8081 --api-url http://127.0.0.1:8080/ 103 | 104 | ``` 105 | 106 | 107 | ### test program 108 | 109 | `test` program runs tests once. 110 | 111 | ```bash 112 | $ palantir test --configuration ./monitor-configuration.js ./tests/**/* 113 | 114 | ``` 115 | 116 | `test` program is used for test development. It allows to filter tests by description (`--match-description`) and by the test tags (`--match-tag`), e.g. 117 | 118 | ```bash 119 | $ palantir test --match-description 'event count is greater' --configuration ./monitor-configuration.js ./tests/**/* 120 | $ palantir test --match-tag 'database' --configuration ./monitor-configuration.js ./tests/**/* 121 | 122 | ``` 123 | 124 | 125 | ## Specification 126 | 127 | 128 | ### Palantir test 129 | 130 | Palantir test is an object with the following properties: 131 | 132 | ```js 133 | /** 134 | * @property assert Evaluates user defined script. The result (boolean) indicates if test is passing. 135 | * @property configuration User defined configuration accessible by the `beforeTest`. 136 | * @property explain Provides debugging information about the test. 137 | * @property interval A function that describes the time when the test needs to be re-run. 138 | * @property labels Arbitrary key=value labels used to categorise the tests. 139 | * @property name Unique name of the test. A combination of test + labels must be unique across all test suites. 140 | * @property priority A numeric value (0-100) indicating the importance of the test. Low value indicates high priority. 141 | */ 142 | type TestType = {| 143 | +assert: (context: TestContextType) => Promise, 144 | +configuration?: SerializableObjectType, 145 | +explain?: (context: TestContextType) => Promise<$ReadOnlyArray>, 146 | +interval: (consecutiveFailureCount: number) => number, 147 | +labels: LabelsType, 148 | +name: string, 149 | +priority: number 150 | |}; 151 | 152 | ``` 153 | 154 | In practice, an example of a test used to check whether HTTP resource is available could look like this: 155 | 156 | ```js 157 | { 158 | assert: async () => { 159 | await request('https://applaudience.com/', { 160 | timeout: interval('10 seconds') 161 | }); 162 | }, 163 | interval: () => { 164 | return interval('30 seconds'); 165 | }, 166 | labels: { 167 | project: 'applaudience', 168 | source: 'http', 169 | type: 'liveness-check' 170 | }, 171 | name: 'https://applaudience.com/ responds with 200' 172 | } 173 | 174 | ``` 175 | 176 | 177 | ### Palantir test suite 178 | 179 | `monitor` program requires a list of file paths as an input. Every input file must export a function that creates a `TestSuiteType` object: 180 | 181 | ```js 182 | type TestSuiteType = {| 183 | +tests: $ReadOnlyArray 184 | |}; 185 | 186 | ``` 187 | 188 | Example: 189 | 190 | ```js 191 | // @flow 192 | 193 | import request from 'axios'; 194 | import interval from 'human-interval'; 195 | import type { 196 | TestSuiteFactoryType 197 | } from 'palantir'; 198 | 199 | const createIntervalCreator = (intervalTime) => { 200 | return () => { 201 | return intervalTime; 202 | }; 203 | }; 204 | 205 | const createTestSuite: TestSuiteFactoryType = () => { 206 | return { 207 | tests: [ 208 | { 209 | assert: async () => { 210 | await request('https://applaudience.com/', { 211 | timeout: interval('10 seconds') 212 | }); 213 | }, 214 | interval: createIntervalCreator(interval('30 seconds')), 215 | labels: { 216 | project: 'applaudience', 217 | scope: 'http' 218 | }, 219 | name: 'https://applaudience.com/ responds with 200' 220 | } 221 | ] 222 | } 223 | }; 224 | 225 | export default createTestSuite; 226 | 227 | ``` 228 | 229 | Note that the test suite factory may return a promise. Refer to [asynchronously creating a test suite](#asynchronously-creating-a-test-suite) for a use case example. 230 | 231 | 232 | ### Monitor configuration 233 | 234 | Palantir `monitor` program accepts `configuration` configuration (a path to a script). 235 | 236 | ```js 237 | /** 238 | * @property after Called when shutting down the monitor. 239 | * @property afterTest Called after every test. 240 | * @property before Called when starting the monitor. 241 | * @property beforeTest Called before every test. 242 | */ 243 | type ConfigurationType = {| 244 | +after: () => Promise, 245 | +afterTest?: (test: RegisteredTestType, context?: TestContextType) => Promise, 246 | +before: () => Promise, 247 | +beforeTest?: (test: RegisteredTestType) => Promise 248 | |}; 249 | 250 | ``` 251 | 252 | The configuration script allows to setup hooks for different stages of the program execution. 253 | 254 | In practice, this can be used to configure the database connection, e.g. 255 | 256 | ```js 257 | import { 258 | createPool 259 | } from 'slonik'; 260 | 261 | let pool; 262 | 263 | export default { 264 | afterTest: async (test, context) => { 265 | await context.connection.release(); 266 | }, 267 | before: async () => { 268 | pool = await createPool('postgres://'); 269 | }, 270 | beforeTest: async () => { 271 | const connection = await pool.connect(); 272 | 273 | return { 274 | connection 275 | }; 276 | } 277 | }; 278 | 279 | ``` 280 | 281 | Note that in the above example, unless you are using database connection for all the tests, you do not want to allocate a connection for every test. You can restrict allocation of connection using test configuration, e.g. 282 | 283 | Test that requires connection to the database: 284 | 285 | ```js 286 | { 287 | assert: (context) => { 288 | return context.connection.any('SELECT 1'); 289 | }, 290 | configuration: { 291 | database: true 292 | }, 293 | interval: () => { 294 | return interval('30 seconds'); 295 | }, 296 | labels: { 297 | scope: 'database' 298 | }, 299 | name: 'connects to the database' 300 | } 301 | 302 | ``` 303 | 304 | Monitor configuration that is aware of the `configuration.database` configuration. 305 | 306 | ```js 307 | import { 308 | createPool 309 | } from 'slonik'; 310 | 311 | let pool; 312 | 313 | export default { 314 | afterTest: async (test, context) => { 315 | if (!test.configuration.database) { 316 | return; 317 | } 318 | 319 | await context.connection.release(); 320 | }, 321 | before: async () => { 322 | pool = await createPool('postgres://'); 323 | }, 324 | beforeTest: async (test) => { 325 | if (!test.configuration.database) { 326 | return {}; 327 | } 328 | 329 | const connection = await pool.connect(); 330 | 331 | return { 332 | connection 333 | }; 334 | } 335 | }; 336 | 337 | ``` 338 | 339 | 340 | ### Alert configuration 341 | 342 | Palantir `alert` program accepts `configuration` configuration (a path to a script). 343 | 344 | ```js 345 | /** 346 | * @property onNewFailingTest Called when a new test fails. 347 | * @property onRecoveredTest Called when a previously failing test is no longer failing. 348 | */ 349 | type AlertConfigurationType = {| 350 | +onNewFailingTest?: (registeredTest: RegisteredTestType) => void, 351 | +onRecoveredTest?: (registeredTest: RegisteredTestType) => void 352 | |}; 353 | 354 | ``` 355 | 356 | The alert configuration script allows to setup event handlers used to observe when tests fail and recover. 357 | 358 | In practice, this can be used to configure a system that notifies other systems about the failing tests, e.g. 359 | 360 | ```js 361 | /** 362 | * @file Using https://www.twilio.com/ to send a text message when tests fail and when tests recover. 363 | */ 364 | import createTwilio from 'twilio'; 365 | 366 | const twilio = createTwilio('ACCOUNT SID', 'AUTH TOKEN'); 367 | 368 | const sendMessage = (message) => { 369 | twilio.messages.create({ 370 | body: message, 371 | to: '+12345678901', 372 | from: '+12345678901' 373 | }); 374 | }; 375 | 376 | export default { 377 | onNewFailingTest: (test) => { 378 | sendMessage('FAILURE ' + test.name + ' failed'); 379 | }, 380 | onRecoveredTest: (test) => { 381 | sendMessage('RECOVERY ' + test.name + ' recovered'); 382 | } 383 | }; 384 | 385 | ``` 386 | 387 | The above example will send a message for every failure and recovery, every time failure/ recovery occurs. In practise, it is desired that the alerting system includes a mechanism to filter out temporarily failures. To address this requirement, Palantir implements an [alert controller](#alert-controller). 388 | 389 | 390 | ### Alert controller 391 | 392 | Palantir alert controller abstracts logic used to filter temporarily failures. 393 | 394 | `palantir` module exports a factory method `createAlertController` used to create an Palantir alert controller. 395 | 396 | ```js 397 | /** 398 | * @property delayFailure Returns test-specific number of milliseconds to wait before considering the test to be failing. 399 | * @property delayRecovery Returns test-specific number of milliseconds to wait before considering the test to be recovered. 400 | * @property onFailure Called when test is considered to be failing. 401 | * @property onRecovery Called when test is considered to be recovered. 402 | */ 403 | type ConfigurationType = {| 404 | +delayFailure: (test: RegisteredTestType) => number, 405 | +delayRecovery: (test: RegisteredTestType) => number, 406 | +onFailure: (test: RegisteredTestType) => void, 407 | +onRecovery: (test: RegisteredTestType) => void 408 | |}; 409 | 410 | type AlertControllerType = {| 411 | +getDelayedFailingTests: () => $ReadOnlyArray, 412 | +getDelayedRecoveringTests: () => $ReadOnlyArray, 413 | +registerTestFailure: (test: RegisteredTestType) => void, 414 | +registerTestRecovery: (test: RegisteredTestType) => void 415 | |}; 416 | 417 | createAlertController(configuration: ConfigurationType) => AlertControllerType; 418 | 419 | ``` 420 | 421 | Use `createAlertController` to implement alert throttling, e.g. 422 | 423 | ```js 424 | import interval from 'human-interval'; 425 | import createTwilio from 'twilio'; 426 | import { 427 | createAlertController 428 | } from 'palantir'; 429 | 430 | const twilio = createTwilio('ACCOUNT SID', 'AUTH TOKEN'); 431 | 432 | const sendMessage = (message) => { 433 | twilio.messages.create({ 434 | body: message, 435 | to: '+12345678901', 436 | from: '+12345678901' 437 | }); 438 | }; 439 | 440 | const controller = createAlertController({ 441 | delayFailure: (test) => { 442 | if (test.labels.scope === 'database') { 443 | return 0; 444 | } 445 | 446 | return interval('5 minute'); 447 | }, 448 | delayRecovery: () => { 449 | return interval('1 minute'); 450 | }, 451 | onFailure: (test) => { 452 | sendMessage('FAILURE ' + test.description + ' failed'); 453 | }, 454 | onRecovery: () => { 455 | sendMessage('RECOVERY ' + test.description + ' recovered'); 456 | } 457 | }); 458 | 459 | export default { 460 | onNewFailingTest: (test) => { 461 | controller.registerTestFailure(test); 462 | }, 463 | onRecoveredTest: (test) => { 464 | controller.registerTestRecovery(test); 465 | } 466 | }; 467 | 468 | ``` 469 | 470 | 471 | ### Palantir HTTP API 472 | 473 | Palantir `monitor` program creates HTTP GraphQL API server. The API exposes information about the user-registered tests and the failing tests. 474 | 475 | Refer to the [schema.graphql](./src/schema.graphql) or [introspect the API](https://graphql.org/learn/introspection/) to learn more. 476 | 477 | 478 | ## Recipes 479 | 480 | 481 | ### Asynchronously creating a test suite 482 | 483 | Creating a test suite might require to query an asynchronous source, e.g. when information required to create a test suite is stored in a database. In this case, a test suite factory can return a promise that resolves with a test suite, e.g. 484 | 485 | ```js 486 | const createTestSuite: TestSuiteFactoryType = async () => { 487 | const clients = await getClients(connection); 488 | 489 | return clients.map((client) => { 490 | return { 491 | assert: async () => { 492 | await request(client.url, { 493 | timeout: interval('10 seconds') 494 | }); 495 | }, 496 | interval: createIntervalCreator(interval('30 seconds')), 497 | labels: { 498 | 'client.country': client.country, 499 | 'client.id': client.id, 500 | source: 'http', 501 | type: 'liveness-check' 502 | }, 503 | name: client.url + ' responds with 200' 504 | }; 505 | }); 506 | }; 507 | 508 | ``` 509 | 510 | In the above example, `getClients` is used to asynchronously retrieve information required to construct the test suite. 511 | 512 | 513 | ### Refreshing a test suit 514 | 515 | It might be desired that the test suite itself informs the monitor about new tests, e.g. the example in the [asynchronously creating a test suite](#asynchronously-creating-a-test-suite) recipe retrieves information from an external datasource that may change over time. In this case, a test suite factory can inform the `monitor` program that it should recreate the test suite, e.g. 516 | 517 | ```js 518 | const createTestSuite: TestSuiteFactoryType = async (refreshTestSuite) => { 519 | const clients = await getClients(connection); 520 | 521 | (async () => { 522 | // Some logic used to determine when the `clients` data used 523 | // to construct the original test suite becomes stale. 524 | while (true) { 525 | await delay(interval('10 seconds')); 526 | 527 | if (JSON.stringify(clients) !== JSON.stringify(await getClients(connection))) { 528 | // Calling `refreshTestSuite` will make Palantir monitor program 529 | // recreate the test suite using `createTestSuite`. 530 | refreshTestSuite(); 531 | 532 | break; 533 | } 534 | } 535 | })(); 536 | 537 | return clients.map((client) => { 538 | return { 539 | assert: async () => { 540 | await request(client.url, { 541 | timeout: interval('10 seconds') 542 | }); 543 | }, 544 | interval: createIntervalCreator(interval('30 seconds')), 545 | labels: { 546 | 'client.country': client.country, 547 | 'client.id': client.id, 548 | source: 'http', 549 | type: 'liveness-check' 550 | }, 551 | name: client.url + ' responds with 200' 552 | }; 553 | }); 554 | }; 555 | 556 | ``` 557 | 558 | 559 | 560 | ## Development 561 | 562 | There are multiple components required to run the service. 563 | 564 | Run `npm run dev` to watch the project and re-build upon detecting a change. 565 | 566 | In order to observe project changes and restart all the services use a program such as [`nodemon`](https://www.npmjs.com/package/nodemon), e.g. 567 | 568 | ```bash 569 | $ NODE_ENV=development nodemon --watch dist --ext js,graphql dist/bin/index.js monitor ... 570 | $ NODE_ENV=development nodemon --watch dist --ext js,graphql dist/bin/index.js alert ... 571 | 572 | ``` 573 | 574 | Use `--watch` attribute multiple times to include Palantir project code and your configuration/ test scripts. 575 | 576 | `report` program run in `NODE_ENV=development` use `webpack-hot-middleware` to implement hot reloading. 577 | 578 | ```bash 579 | $ NODE_ENV=development babel-node src/bin/index.js report --service-port 8081 --api-url http://127.0.0.1:8080/ | roarr pretty-print 580 | 581 | ``` 582 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "author": { 3 | "email": "gajus@gajus.com", 4 | "name": "Gajus Kuizinas", 5 | "url": "http://gajus.com" 6 | }, 7 | "ava": { 8 | "require": [ 9 | "@babel/register" 10 | ] 11 | }, 12 | "bin": "./dist/bin/index.js", 13 | "dependencies": { 14 | "apollo-boost": "^0.2.0-verify.4", 15 | "apollo-cache-inmemory": "^1.3.12", 16 | "apollo-client": "^2.5.0-verify.4", 17 | "apollo-link-http": "^1.6.0-alpha.5", 18 | "apollo-server": "^2.3.1", 19 | "body-parser": "^1.18.3", 20 | "cors": "^2.8.5", 21 | "delay": "^4.1.0", 22 | "es6-error": "^4.1.1", 23 | "express": "^5.0.0-alpha.6", 24 | "express-http-proxy": "^1.5.0", 25 | "gitdown": "^2.5.5", 26 | "glob": "^7.1.3", 27 | "graphql": "^14.0.2", 28 | "graphql-iso-date": "^3.6.1", 29 | "graphql-playground-middleware-express": "^1.7.8", 30 | "graphql-server-core": "^1.4.0", 31 | "graphql-tag": "^2.10.0", 32 | "graphql-tools": "^4.0.3", 33 | "graphql-type-json": "^0.2.1", 34 | "human-interval": "^0.1.6", 35 | "json5": "^2.1.0", 36 | "lodash": "^4.17.11", 37 | "node-fetch": "^2.3.0", 38 | "pretty-ms": "^4.0.0", 39 | "react": "^16.7.0", 40 | "react-dom": "^16.7.0", 41 | "react-hot-loader": "^4.6.3", 42 | "regex-parser": "^2.2.9", 43 | "roarr": "^2.12.1", 44 | "serialize-error": "^3.0.0", 45 | "serve-static": "^1.13.2", 46 | "sift": "^7.0.1", 47 | "throat": "^4.1.0", 48 | "uuid": "^3.3.2", 49 | "webpack": "^4.28.3", 50 | "webpack-hot-middleware": "^2.24.3", 51 | "yargs": "^12.0.5" 52 | }, 53 | "description": "Active monitoring and alerting system using user-defined Node.js scripts.", 54 | "devDependencies": { 55 | "@babel/cli": "^7.2.3", 56 | "@babel/core": "^7.2.2", 57 | "@babel/node": "^7.2.2", 58 | "@babel/plugin-proposal-class-properties": "^7.2.3", 59 | "@babel/plugin-transform-flow-strip-types": "^7.2.3", 60 | "@babel/preset-env": "^7.2.3", 61 | "@babel/preset-react": "^7.0.0", 62 | "@babel/register": "^7.0.0", 63 | "ava": "^1.0.1", 64 | "babel-loader": "^8.0.4", 65 | "babel-plugin-istanbul": "^5.1.0", 66 | "coveralls": "^3.0.2", 67 | "css-loader": "^2.1.0", 68 | "css-module-flow": "^1.0.0", 69 | "eslint": "^5.11.1", 70 | "eslint-config-canonical": "^15.0.1", 71 | "flow-bin": "^0.89.0", 72 | "flow-copy-source": "^2.0.2", 73 | "husky": "^1.3.1", 74 | "nock": "^10.0.5", 75 | "node-sass": "^4.11.0", 76 | "nyc": "^13.1.0", 77 | "postcss-loader": "^3.0.0", 78 | "postcss-scss": "^2.0.0", 79 | "resolve-url-loader": "^3.0.0", 80 | "sass-loader": "^7.1.0", 81 | "semantic-release": "^15.13.2", 82 | "sinon": "^7.2.2", 83 | "style-loader": "^0.23.1", 84 | "webpack-cli": "^3.1.2", 85 | "webpack-dev-middleware": "^3.4.0" 86 | }, 87 | "engines": { 88 | "node": ">=10" 89 | }, 90 | "husky": { 91 | "hooks": { 92 | "post-commit": "npm run create-readme && git add README.md && git commit -m 'docs: generate docs' --no-verify", 93 | "pre-commit": "npm run lint && npm run test && npm run build" 94 | } 95 | }, 96 | "keywords": [ 97 | "warning", 98 | "alerting", 99 | "monitoring" 100 | ], 101 | "main": "./dist/index.js", 102 | "name": "palantir", 103 | "nyc": { 104 | "all": true, 105 | "include": [ 106 | "src/**/*.js" 107 | ], 108 | "instrument": false, 109 | "reporter": [ 110 | "html", 111 | "text-summary" 112 | ], 113 | "require": [ 114 | "@babel/register" 115 | ], 116 | "sourceMap": false 117 | }, 118 | "repository": { 119 | "type": "git", 120 | "url": "git@github.com:gajus/palantir.git" 121 | }, 122 | "scripts": { 123 | "build": "rm -fr ./dist && NODE_ENV=production babel ./src --out-dir ./dist --copy-files --source-maps && webpack --config-register @babel/register --config src/report-web-app/webpack.configuration.production.js", 124 | "create-readme": "gitdown ./.README/README.md --output-file ./README.md", 125 | "dev": "NODE_ENV=production babel ./src --out-dir ./dist --copy-files --source-maps --watch", 126 | "lint": "eslint ./src && flow", 127 | "test": "NODE_ENV=test nyc ava --verbose --serial --concurrency 1" 128 | }, 129 | "version": "1.0.0" 130 | } 131 | -------------------------------------------------------------------------------- /src/Logger.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import Roarr from 'roarr'; 4 | 5 | const Logger = Roarr 6 | .child({ 7 | package: 'palantir' 8 | }); 9 | 10 | export default Logger; 11 | -------------------------------------------------------------------------------- /src/assertions/assertUniqueTestIdPayloads.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import createTestId from '../factories/createTestId'; 4 | import Logger from '../Logger'; 5 | import type { 6 | TestIdPayloadInputType 7 | } from '../types'; 8 | 9 | const log = Logger.child({ 10 | namespace: 'assertUniqueTestIdPayloads' 11 | }); 12 | 13 | export default (tests: $ReadOnlyArray): void => { 14 | const testMap = new Map(); 15 | 16 | for (const test of tests) { 17 | const testId = createTestId(test); 18 | 19 | const first = testMap.get(testId); 20 | 21 | if (first) { 22 | log.error({ 23 | first, 24 | second: test 25 | }, 'found two tests with the same name and labels'); 26 | 27 | throw new Error('Test name and labels combination must be unique.'); 28 | } 29 | 30 | testMap.set(testId, test); 31 | } 32 | }; 33 | -------------------------------------------------------------------------------- /src/assertions/index.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | export {default as assertUniqueTestIdPayloads} from './assertUniqueTestIdPayloads'; 4 | -------------------------------------------------------------------------------- /src/bin/commands/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "rules": { 3 | "filenames/match-regex": 0 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /src/bin/commands/alert.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import delay from 'delay'; 4 | import { 5 | differenceBy 6 | } from 'lodash'; 7 | import { 8 | ApolloClient 9 | } from 'apollo-client'; 10 | import gql from 'graphql-tag'; 11 | import { 12 | importModule 13 | } from '../../utilities'; 14 | import { 15 | createGraphqlClient 16 | } from '../../factories'; 17 | import Logger from '../../Logger'; 18 | 19 | type ArgvType = {| 20 | +configuration: string, 21 | +palantirApiUrl: string, 22 | +tests: $ReadOnlyArray 23 | |}; 24 | 25 | const log = Logger.child({ 26 | namespace: 'bin/commands/alert' 27 | }); 28 | 29 | const queryFailingTests = async (graphqlClient: ApolloClient) => { 30 | // eslint-disable-next-line no-restricted-syntax 31 | const query = gql` 32 | { 33 | failingTests { 34 | edges { 35 | node { 36 | id 37 | labels { 38 | name 39 | value 40 | } 41 | name 42 | lastTestedAt 43 | testIsFailing 44 | } 45 | } 46 | } 47 | } 48 | `; 49 | 50 | const result = await graphqlClient.query({ 51 | fetchPolicy: 'no-cache', 52 | query 53 | }); 54 | 55 | return result.data.failingTests.edges.map((edge) => { 56 | return edge.node; 57 | }); 58 | }; 59 | 60 | export const command = 'alert'; 61 | export const description = 'Subscribes to the Palantir HTTP API and alerts other systems using-defined configuration.'; 62 | 63 | // eslint-disable-next-line flowtype/no-weak-types 64 | export const builder = (yargs: Object) => { 65 | return yargs 66 | .env('PALANTIR_ALERT') 67 | .options({ 68 | configuration: { 69 | demand: true, 70 | description: 'Path to the Palantir alert configuration file.', 71 | type: 'string' 72 | }, 73 | 'palantir-api-url': { 74 | demand: true, 75 | type: 'string' 76 | } 77 | }); 78 | }; 79 | 80 | export const handler = async (argv: ArgvType) => { 81 | const configuration = importModule(argv.configuration); 82 | 83 | const graphqlClient = createGraphqlClient(argv.palantirApiUrl); 84 | 85 | let knownFailingTests = []; 86 | 87 | const identifyTest = (test) => { 88 | return test.id; 89 | }; 90 | 91 | const onNewFailingTest = configuration.onNewFailingTest; 92 | const onRecoveredTest = configuration.onRecoveredTest; 93 | 94 | while (true) { 95 | const failingTests = await queryFailingTests(graphqlClient); 96 | 97 | const newFailingTests = differenceBy(failingTests, knownFailingTests, identifyTest); 98 | 99 | knownFailingTests.push(...newFailingTests); 100 | 101 | const recoveredTests = differenceBy(knownFailingTests, failingTests, identifyTest); 102 | 103 | knownFailingTests = differenceBy(knownFailingTests, recoveredTests, identifyTest); 104 | 105 | if (newFailingTests.length || recoveredTests.length) { 106 | log.info({ 107 | failingTests, 108 | newFailingTests, 109 | recoveredTests 110 | }, 'new test state'); 111 | } else { 112 | log.debug('no change of test state'); 113 | } 114 | 115 | if (onNewFailingTest && newFailingTests.length) { 116 | for (const newFailingTest of newFailingTests) { 117 | onNewFailingTest(newFailingTest); 118 | } 119 | } 120 | 121 | if (onRecoveredTest && recoveredTests.length) { 122 | for (const recoveredTest of recoveredTests) { 123 | onRecoveredTest(recoveredTest); 124 | } 125 | } 126 | 127 | log.debug('delaying next check by 5 seconds'); 128 | 129 | // @todo Use GraphQL subscriptions. 130 | await delay(5 * 1000); 131 | } 132 | }; 133 | -------------------------------------------------------------------------------- /src/bin/commands/monitor.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | /* eslint-disable import/no-namespace */ 4 | 5 | import fs from 'fs'; 6 | import path from 'path'; 7 | import express from 'express'; 8 | import cors from 'cors'; 9 | import bodyParser from 'body-parser'; 10 | import GraphQLJSON from 'graphql-type-json'; 11 | import expressPlayground from 'graphql-playground-middleware-express'; 12 | import { 13 | GraphQLDateTime 14 | } from 'graphql-iso-date'; 15 | import { 16 | makeExecutableSchema 17 | } from 'graphql-tools'; 18 | import * as resolvers from '../../resolvers'; 19 | import Logger from '../../Logger'; 20 | import { 21 | createGraphqlMiddleware, 22 | createMonitor 23 | } from '../../factories'; 24 | import { 25 | importModule, 26 | resolveFilePathExpression 27 | } from '../../utilities'; 28 | import type { 29 | TestSuiteType 30 | } from '../../types'; 31 | 32 | type ArgvType = {| 33 | +configuration?: string, 34 | +servicePort: number, 35 | +tests: $ReadOnlyArray 36 | |}; 37 | 38 | export const command = 'monitor '; 39 | export const description = 'Continuously performs user-defined tests and exposes the current state via HTTP API.'; 40 | 41 | // eslint-disable-next-line flowtype/no-weak-types 42 | export const builder = (yargs: Object) => { 43 | return yargs 44 | .env('PALANTIR_MONITOR') 45 | .options({ 46 | configuration: { 47 | description: 'Path to the Palantir monitor configuration file.', 48 | type: 'string' 49 | }, 50 | 'service-port': { 51 | default: 8080, 52 | type: 'number' 53 | } 54 | }); 55 | }; 56 | 57 | export const handler = async (argv: ArgvType) => { 58 | const log = Logger.child({ 59 | namespace: 'bin/commands/monitor' 60 | }); 61 | 62 | const schemaDefinition = fs.readFileSync(path.resolve(__dirname, '../../schema.graphql'), 'utf8'); 63 | 64 | const schema = makeExecutableSchema({ 65 | resolvers: { 66 | ...resolvers, 67 | DateTime: GraphQLDateTime, 68 | JSON: GraphQLJSON, 69 | Node: { 70 | // eslint-disable-next-line id-match 71 | __resolveType () { 72 | return null; 73 | } 74 | } 75 | }, 76 | resolverValidationOptions: { 77 | requireResolversForResolveType: false 78 | }, 79 | typeDefs: schemaDefinition 80 | }); 81 | 82 | let configuration = {}; 83 | 84 | if (argv.configuration) { 85 | configuration = importModule(argv.configuration); 86 | } 87 | 88 | const testFilePaths = resolveFilePathExpression(argv.tests); 89 | 90 | log.debug({ 91 | tests: testFilePaths 92 | }, 'received %d test file path(s)', testFilePaths.length); 93 | 94 | const monitor = await createMonitor(configuration); 95 | 96 | const app = express(); 97 | 98 | app.use(cors()); 99 | 100 | app.set('trust proxy', true); 101 | app.set('x-powered-by', false); 102 | 103 | const graphqlMiddleware = createGraphqlMiddleware(async () => { 104 | return { 105 | context: { 106 | configuration, 107 | monitor 108 | }, 109 | schema 110 | }; 111 | }); 112 | 113 | app.use('/playground', expressPlayground({ 114 | endpoint: '/', 115 | settings: { 116 | 'editor.cursorShape': 'line', 117 | 'request.credentials': 'same-origin' 118 | } 119 | })); 120 | 121 | app.use('/', bodyParser.json(), graphqlMiddleware); 122 | 123 | app.listen(argv.servicePort, '0.0.0.0'); 124 | 125 | const registerTestSuite = async (createTestSuite) => { 126 | const testSuite: TestSuiteType = await createTestSuite(() => { 127 | for (const test of testSuite.tests) { 128 | monitor.unregisterTest(test); 129 | } 130 | 131 | registerTestSuite(createTestSuite); 132 | }); 133 | 134 | for (const test of testSuite.tests) { 135 | monitor.registerTest(test); 136 | } 137 | }; 138 | 139 | for (const testFilePath of testFilePaths) { 140 | const createTestSuite = importModule(testFilePath); 141 | 142 | registerTestSuite(createTestSuite); 143 | } 144 | }; 145 | -------------------------------------------------------------------------------- /src/bin/commands/report.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import { 4 | parse as parseUrl 5 | } from 'url'; 6 | import path from 'path'; 7 | import express from 'express'; 8 | import serveStatic from 'serve-static'; 9 | import proxy from 'express-http-proxy'; 10 | import { 11 | createResponseBody, 12 | createWebpackConfiguration 13 | } from '../../factories'; 14 | 15 | type ArgvType = {| 16 | +apiUrl: string, 17 | +basePath: string, 18 | +servicePort: number 19 | |}; 20 | 21 | export const command = 'report'; 22 | export const description = 'Creates a web UI for the Palantir HTTP API.'; 23 | 24 | // eslint-disable-next-line flowtype/no-weak-types 25 | export const builder = (yargs: Object) => { 26 | return yargs 27 | .env('PALANTIR_REPORT') 28 | .options({ 29 | 'api-url': { 30 | demand: true, 31 | type: 'string' 32 | }, 33 | 'base-path': { 34 | default: '/', 35 | type: 'string' 36 | }, 37 | 'service-port': { 38 | default: 8080, 39 | type: 'number' 40 | } 41 | }); 42 | }; 43 | 44 | export const handler = async (argv: ArgvType) => { 45 | const app = express(); 46 | 47 | const router = express.Router(); 48 | 49 | app.set('etag', 'strong'); 50 | app.set('trust proxy', true); 51 | app.set('x-powered-by', false); 52 | 53 | const webpackConfiguration = createWebpackConfiguration(argv.basePath); 54 | 55 | // eslint-disable-next-line no-process-env 56 | if (process.env.NODE_ENV === 'development') { 57 | /* eslint-disable global-require */ 58 | const webpack = require('webpack'); 59 | const webpackDevMiddleware = require('webpack-dev-middleware'); 60 | const webpackHotMiddleware = require('webpack-hot-middleware'); 61 | /* eslint-enable global-require */ 62 | 63 | const compiler = webpack(webpackConfiguration); 64 | 65 | const devServerOptions = { 66 | publicPath: argv.basePath + 'static/', 67 | stats: 'minimal' 68 | }; 69 | 70 | router.use(webpackDevMiddleware(compiler, devServerOptions)); 71 | router.use(webpackHotMiddleware(compiler)); 72 | } else { 73 | router.use('/static', serveStatic(path.resolve(__dirname, '../../../.static'), { 74 | fallthrough: true, 75 | index: false 76 | })); 77 | } 78 | 79 | router.use('/api', proxy(argv.apiUrl, { 80 | proxyReqPathResolver: () => { 81 | return parseUrl(argv.apiUrl).path; 82 | } 83 | })); 84 | 85 | router.use('/', (req, res) => { 86 | const scriptUrls = []; 87 | const styleUrls = []; 88 | 89 | scriptUrls.push(argv.basePath + 'static/main.bundle.js'); 90 | 91 | styleUrls.push('https://fonts.googleapis.com/css?family=Source+Code+Pro|Roboto'); 92 | 93 | const response = createResponseBody( 94 | { 95 | API_URL: argv.apiUrl, 96 | BASE_PATH: argv.basePath 97 | }, 98 | scriptUrls, 99 | styleUrls, 100 | '' 101 | ); 102 | 103 | res 104 | .send(response); 105 | }); 106 | 107 | app.use(argv.basePath, router); 108 | 109 | app.listen(argv.servicePort); 110 | }; 111 | -------------------------------------------------------------------------------- /src/bin/commands/test.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import parseRegex from 'regex-parser'; 4 | import Logger from '../../Logger'; 5 | import { 6 | createMonitor 7 | } from '../../factories'; 8 | import { 9 | importModule, 10 | resolveFilePathExpression 11 | } from '../../utilities'; 12 | import type { 13 | TestSuiteType 14 | } from '../../types'; 15 | 16 | type ArgvType = {| 17 | +configuration?: string, 18 | +matchLabel?: string, 19 | +matchName?: string, 20 | +tests: $ReadOnlyArray 21 | |}; 22 | 23 | export const command = 'test '; 24 | export const description = 'Runs tests once. Used for test development.'; 25 | 26 | // eslint-disable-next-line flowtype/no-weak-types 27 | export const builder = (yargs: Object) => { 28 | return yargs 29 | .env('PALANTIR_TEST') 30 | .options({ 31 | configuration: { 32 | description: 'Path to the Palantir monitor configuration file.', 33 | type: 'string' 34 | }, 35 | 'match-label': { 36 | description: 'Regex rule used to filter tests by their labels. Labels are normalised to "key=value" values.', 37 | type: 'string' 38 | }, 39 | 'match-name': { 40 | description: 'Regex rule used to filter tests by name.', 41 | type: 'string' 42 | } 43 | }); 44 | }; 45 | 46 | // eslint-disable-next-line complexity 47 | export const handler = async (argv: ArgvType) => { 48 | let isTestNameMatching; 49 | 50 | if (argv.matchName) { 51 | isTestNameMatching = (testName: string): boolean => { 52 | return parseRegex(argv.matchName).test(testName); 53 | }; 54 | } 55 | 56 | let isTestLabelMatching; 57 | 58 | if (argv.matchLabel) { 59 | isTestLabelMatching = (testLabel: string): boolean => { 60 | return parseRegex(argv.matchLabel).test(testLabel); 61 | }; 62 | } 63 | 64 | const log = Logger.child({ 65 | namespace: 'bin/commands/test' 66 | }); 67 | 68 | let configuration = {}; 69 | 70 | if (argv.configuration) { 71 | configuration = importModule(argv.configuration); 72 | } 73 | 74 | const testFilePaths = resolveFilePathExpression(argv.tests); 75 | 76 | log.debug({ 77 | tests: testFilePaths 78 | }, 'received %d test file path(s)', testFilePaths.length); 79 | 80 | const monitor = await createMonitor(configuration); 81 | 82 | for (const testFilePath of testFilePaths) { 83 | const createTestSuite = importModule(testFilePath); 84 | 85 | const testSuite: TestSuiteType = await createTestSuite(); 86 | 87 | for (const test of testSuite.tests) { 88 | if (isTestNameMatching && isTestLabelMatching) { 89 | const matchingLabels = Object 90 | .keys(test.labels) 91 | .map((label) => { 92 | return label + '=' + test.labels[label]; 93 | }) 94 | .filter(isTestLabelMatching); 95 | 96 | if (!isTestNameMatching(test.name) || !matchingLabels.length) { 97 | log.debug('skipping test; test name or none of the test labels match the respective filters'); 98 | 99 | // eslint-disable-next-line no-continue 100 | continue; 101 | } 102 | } else if (isTestNameMatching) { 103 | if (!isTestNameMatching(test.name)) { 104 | log.debug('skipping test; test name does not match the filter'); 105 | 106 | // eslint-disable-next-line no-continue 107 | continue; 108 | } 109 | } else if (isTestLabelMatching) { 110 | const matchingLabels = Object 111 | .keys(test.labels) 112 | .map((label) => { 113 | return label + '=' + test.labels[label]; 114 | }) 115 | .filter(isTestLabelMatching); 116 | 117 | if (!matchingLabels.length) { 118 | log.debug('skipping test; none of the test labels match the filter'); 119 | 120 | // eslint-disable-next-line no-continue 121 | continue; 122 | } 123 | } 124 | 125 | log.debug('running test %s', test.name); 126 | 127 | monitor.runTest(test); 128 | } 129 | } 130 | }; 131 | -------------------------------------------------------------------------------- /src/bin/index.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import yargs from 'yargs'; 4 | 5 | yargs 6 | .commandDir('commands') 7 | .help() 8 | .wrap(80) 9 | .parse(); 10 | -------------------------------------------------------------------------------- /src/errors.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import ExtendableError from 'es6-error'; 4 | 5 | export class UserError extends ExtendableError { 6 | code: string; 7 | 8 | constructor (message: string, code: string = 'USER_ERROR') { 9 | super(message); 10 | 11 | this.code = code; 12 | } 13 | } 14 | 15 | export class NotFoundError extends UserError { 16 | constructor () { 17 | super('Resource not found.', 'RESOURCE_NOT_FOUND'); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/factories/createAlertController.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import type { 4 | RegisteredTestType 5 | } from '../types'; 6 | 7 | /** 8 | * @property delayFailure Returns test-specific number of milliseconds to wait before considering the test to be failing. 9 | * @property delayRecovery Returns test-specific number of milliseconds to wait before considering the test to be recovered. 10 | * @property onFailure Called when test is considered to be failing. 11 | * @property onRecovery Called when test is considered to be recovered. 12 | */ 13 | type ConfigurationType = {| 14 | +delayFailure: (test: RegisteredTestType) => number, 15 | +delayRecovery: (test: RegisteredTestType) => number, 16 | +onFailure: (test: RegisteredTestType) => void, 17 | +onRecovery: (test: RegisteredTestType) => void 18 | |}; 19 | 20 | type AlertControllerType = {| 21 | +getDelayedFailingTests: () => $ReadOnlyArray, 22 | +getDelayedRecoveringTests: () => $ReadOnlyArray, 23 | +registerTestFailure: (test: RegisteredTestType) => void, 24 | +registerTestRecovery: (test: RegisteredTestType) => void 25 | |}; 26 | 27 | type TimerIndexType = { 28 | [key: string]: TimeoutID 29 | }; 30 | 31 | export default (configuration: ConfigurationType): AlertControllerType => { 32 | let delayedFailingTests: Array = []; 33 | let delayedRecoveringTests: Array = []; 34 | 35 | const failingTestTimerIndex: TimerIndexType = {}; 36 | const recoveringTestTimerIndex: TimerIndexType = {}; 37 | 38 | return { 39 | getDelayedFailingTests: () => { 40 | return delayedFailingTests; 41 | }, 42 | getDelayedRecoveringTests: () => { 43 | return delayedRecoveringTests; 44 | }, 45 | registerTestFailure: (test: RegisteredTestType) => { 46 | const maybeDelayedRecoveringTestIndex = delayedRecoveringTests.findIndex((maybeTargetTest) => { 47 | return maybeTargetTest.id === test.id; 48 | }); 49 | 50 | if (maybeDelayedRecoveringTestIndex !== -1) { 51 | clearTimeout(recoveringTestTimerIndex[test.id]); 52 | 53 | delayedRecoveringTests.splice(maybeDelayedRecoveringTestIndex, 1); 54 | } 55 | 56 | const maybeDelayedTestIndex = delayedFailingTests.findIndex((maybeTargetTest) => { 57 | return maybeTargetTest.id === test.id; 58 | }); 59 | 60 | if (maybeDelayedTestIndex !== -1) { 61 | return; 62 | } 63 | 64 | delayedFailingTests.push(test); 65 | 66 | failingTestTimerIndex[test.id] = setTimeout(() => { 67 | delayedFailingTests = delayedFailingTests.filter((maybeTargetTest) => { 68 | return maybeTargetTest.id !== test.id; 69 | }); 70 | 71 | configuration.onFailure(test); 72 | }, configuration.delayFailure(test)); 73 | }, 74 | registerTestRecovery: (test: RegisteredTestType) => { 75 | const maybeDelayedRecoveringTestIndex = delayedRecoveringTests.findIndex((maybeTargetTest) => { 76 | return maybeTargetTest.id === test.id; 77 | }); 78 | 79 | if (maybeDelayedRecoveringTestIndex !== -1) { 80 | return; 81 | } 82 | 83 | delayedRecoveringTests.push(test); 84 | 85 | const maybeDelayedFailingTestIndex = delayedFailingTests.findIndex((maybeTargetTest) => { 86 | return maybeTargetTest.id === test.id; 87 | }); 88 | 89 | if (maybeDelayedFailingTestIndex !== -1) { 90 | clearTimeout(failingTestTimerIndex[test.id]); 91 | 92 | delayedFailingTests.splice(maybeDelayedFailingTestIndex, 1); 93 | } 94 | 95 | recoveringTestTimerIndex[test.id] = setTimeout(() => { 96 | delayedRecoveringTests = delayedRecoveringTests.filter((maybeTargetTest) => { 97 | return maybeTargetTest.id !== test.id; 98 | }); 99 | 100 | configuration.onRecovery(test); 101 | }, configuration.delayRecovery(test)); 102 | } 103 | }; 104 | }; 105 | -------------------------------------------------------------------------------- /src/factories/createGraphqlClient.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import fetch from 'node-fetch'; 4 | import { 5 | ApolloClient 6 | } from 'apollo-client'; 7 | import { 8 | HttpLink 9 | } from 'apollo-link-http'; 10 | import { 11 | InMemoryCache 12 | } from 'apollo-cache-inmemory'; 13 | 14 | export default (apiUrl: string) => { 15 | const graphqlClient = new ApolloClient({ 16 | cache: new InMemoryCache(), 17 | link: new HttpLink({ 18 | credentials: 'include', 19 | fetch, 20 | uri: apiUrl 21 | }) 22 | }); 23 | 24 | return graphqlClient; 25 | }; 26 | -------------------------------------------------------------------------------- /src/factories/createGraphqlMiddleware.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import { 4 | Request, 5 | Response 6 | } from 'express'; 7 | import { 8 | GraphQLOptions, 9 | runHttpQuery 10 | } from 'graphql-server-core'; 11 | 12 | /** 13 | * @todo What is the reason we have written this wrapper? 14 | */ 15 | export default (options: GraphQLOptions) => { 16 | return async (req: Request, res: Response): Promise => { 17 | try { 18 | const gqlResponse = await runHttpQuery([req, res], { 19 | method: req.method, 20 | options, 21 | query: req.method === 'POST' ? req.body : req.query 22 | }); 23 | 24 | // eslint-disable-next-line no-process-env 25 | res.setHeader('Content-Type', 'application/json'); 26 | res.write(gqlResponse); 27 | res.end(); 28 | } catch (error) { 29 | if (error.name === 'HttpQueryError') { 30 | if (error.headers) { 31 | Object.keys(error.headers).forEach((header) => { 32 | res.setHeader(header, error.headers[header]); 33 | }); 34 | } 35 | 36 | res.statusCode = error.statusCode; 37 | res.write(error.message); 38 | res.end(); 39 | } else { 40 | throw error; 41 | } 42 | } 43 | }; 44 | }; 45 | -------------------------------------------------------------------------------- /src/factories/createIntervalRoutine.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import delay from 'delay'; 4 | 5 | type IntervalRoutineType = () => Promise; 6 | 7 | export default (routine: IntervalRoutineType) => { 8 | let run = true; 9 | 10 | (async () => { 11 | // eslint-disable-next-line no-unmodified-loop-condition 12 | while (run) { 13 | await delay(await routine()); 14 | } 15 | })(); 16 | 17 | return () => { 18 | run = false; 19 | }; 20 | }; 21 | -------------------------------------------------------------------------------- /src/factories/createLabelCollection.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import localeCompareProperty from '../utilities/localeCompareProperty'; 4 | import type { 5 | LabelCollectionType, 6 | LabelsType 7 | } from '../types'; 8 | 9 | export default (labels: LabelsType): LabelCollectionType => { 10 | return Object 11 | .keys(labels) 12 | .sort((a, b) => { 13 | return localeCompareProperty(a, b); 14 | }) 15 | .map((label) => { 16 | return { 17 | name: label, 18 | value: labels[label] 19 | }; 20 | }); 21 | }; 22 | -------------------------------------------------------------------------------- /src/factories/createMonitor.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import prettyMs from 'pretty-ms'; 4 | import createThroat from 'throat'; 5 | import { 6 | evaluateRegisteredTest 7 | } from '../routines'; 8 | import { 9 | assertUniqueTestIdPayloads 10 | } from '../assertions'; 11 | import Logger from '../Logger'; 12 | import type { 13 | MonitorConfigurationType, 14 | RegisteredTestType, 15 | TestType 16 | } from '../types'; 17 | import createIntervalRoutine from './createIntervalRoutine'; 18 | import createTestId from './createTestId'; 19 | import createTestPointer from './createTestPointer'; 20 | 21 | const log = Logger.child({ 22 | namespace: 'factories/createMonitor' 23 | }); 24 | 25 | const MAX_PRIORITY = 100; 26 | 27 | export default async (configuration: MonitorConfigurationType) => { 28 | const throat = createThroat(1); 29 | 30 | if (configuration.before) { 31 | await configuration.before(); 32 | } 33 | 34 | const registeredTests: Array = []; 35 | 36 | // @todo Validate the shape of the test at a runtime, e.g. the priority range. 37 | const createRegisteredTest = (test): RegisteredTestType => { 38 | assertUniqueTestIdPayloads(registeredTests.concat([ 39 | { 40 | labels: test.labels, 41 | name: test.name 42 | } 43 | ])); 44 | 45 | const id = createTestId({ 46 | labels: test.labels, 47 | name: test.name 48 | }); 49 | 50 | return { 51 | consecutiveFailureCount: null, 52 | id, 53 | lastError: null, 54 | lastTestedAt: null, 55 | testIsFailing: null, 56 | ...test, 57 | // eslint-disable-next-line sort-keys 58 | priority: typeof test.priority === 'number' ? test.priority : MAX_PRIORITY 59 | }; 60 | }; 61 | 62 | const scheduleTest = (registeredTest: RegisteredTestType) => { 63 | const cancelIntervalRoutine = createIntervalRoutine(async () => { 64 | await throat(async () => { 65 | await evaluateRegisteredTest(configuration, registeredTest); 66 | }); 67 | 68 | const delay = registeredTest.interval(registeredTest.consecutiveFailureCount || 0); 69 | 70 | log.debug('assertion complete; delaying the next iteration for %s', prettyMs(delay, { 71 | verbose: true 72 | })); 73 | 74 | return delay; 75 | }); 76 | 77 | return cancelIntervalRoutine; 78 | }; 79 | 80 | // @todo Implement configuration.after. 81 | 82 | const testScheduleWeakMap = new WeakMap(); 83 | 84 | return { 85 | getRegisteredTests: () => { 86 | return registeredTests; 87 | }, 88 | registerTest: (test: TestType) => { 89 | const registeredTest = createRegisteredTest(test); 90 | 91 | registeredTests.push(registeredTest); 92 | 93 | const cancelTestSchedule = scheduleTest(registeredTest); 94 | 95 | testScheduleWeakMap.set(registeredTest, cancelTestSchedule); 96 | 97 | log.info({ 98 | test: createTestPointer(registeredTest) 99 | }, 'registered test'); 100 | }, 101 | runTest: async (test: TestType) => { 102 | const registeredTest = createRegisteredTest(test); 103 | 104 | await evaluateRegisteredTest(configuration, registeredTest); 105 | }, 106 | unregisterTest: (test: TestType) => { 107 | const targetTestId = createTestId(test); 108 | 109 | const maybeRegisteredTest = registeredTests.find((maybeTargetRegisteredTest) => { 110 | return maybeTargetRegisteredTest.id === targetTestId; 111 | }); 112 | 113 | if (!maybeRegisteredTest) { 114 | throw new Error('Test not found.'); 115 | } 116 | 117 | const cancelTestSchedule = testScheduleWeakMap.get(maybeRegisteredTest); 118 | 119 | if (!cancelTestSchedule) { 120 | throw new Error('Cancel test schedule callback not found.'); 121 | } 122 | 123 | cancelTestSchedule(); 124 | 125 | registeredTests.splice(registeredTests.indexOf(maybeRegisteredTest), 1); 126 | 127 | log.info({ 128 | test: createTestPointer(maybeRegisteredTest) 129 | }, 'unregistered test'); 130 | } 131 | }; 132 | }; 133 | -------------------------------------------------------------------------------- /src/factories/createResponseBody.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | export default ( 4 | // eslint-disable-next-line flowtype/no-weak-types 5 | config: Object, 6 | deferredScriptUrls: $ReadOnlyArray, 7 | styleUrls: $ReadOnlyArray, 8 | head: string = '' 9 | ) => { 10 | const scripts = deferredScriptUrls 11 | .map((deferredScriptUrl) => { 12 | return ``; 13 | }) 14 | .join('\n'); 15 | 16 | const styles = styleUrls 17 | .map((styleUrl) => { 18 | return ``; 19 | }) 20 | .join('\n'); 21 | 22 | const globalState = { 23 | config 24 | }; 25 | 26 | return ` 27 | 28 | 29 | 30 | 31 | 32 | ${styles} 33 | 34 | 37 | 38 | 39 | 40 | ${scripts} 41 | 42 | 43 | 44 | 45 | 46 | ${head} 47 | 48 | 49 |
50 | 51 | 52 | `; 53 | }; 54 | -------------------------------------------------------------------------------- /src/factories/createTestId.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import uuidv5 from 'uuid/v5'; 4 | import type { 5 | TestIdPayloadInputType 6 | } from '../types'; 7 | import createTestIdPayload from './createTestIdPayload'; 8 | 9 | const PALANTIR_TEST = '6b53c21d-8d21-4352-b268-3542d8d9adf0'; 10 | 11 | export default (test: TestIdPayloadInputType): string => { 12 | return uuidv5(JSON.stringify(createTestIdPayload(test)), PALANTIR_TEST); 13 | }; 14 | -------------------------------------------------------------------------------- /src/factories/createTestIdPayload.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import type { 4 | LabelCollectionType, 5 | TestIdPayloadInputType 6 | } from '../types'; 7 | import createLabelCollection from './createLabelCollection'; 8 | 9 | type TestIdPayloadType = {| 10 | +labels: LabelCollectionType, 11 | +name: string 12 | |}; 13 | 14 | export default (test: TestIdPayloadInputType): TestIdPayloadType => { 15 | return { 16 | labels: createLabelCollection(test.labels), 17 | name: test.name 18 | }; 19 | }; 20 | -------------------------------------------------------------------------------- /src/factories/createTestPointer.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import type { 4 | TestType, 5 | RegisteredTestType 6 | } from '../types'; 7 | import createLabelCollection from './createLabelCollection'; 8 | 9 | type TestPointerType = {| 10 | +id?: string, 11 | +labels: $ReadOnlyArray, 12 | +name: string 13 | |}; 14 | 15 | export default (test: TestType | RegisteredTestType): TestPointerType => { 16 | // eslint-disable-next-line flowtype/no-weak-types 17 | const pointer: Object = { 18 | labels: Array.isArray(test.labels) ? test.labels : createLabelCollection(test.labels), 19 | name: test.name 20 | }; 21 | 22 | if (test.id) { 23 | pointer.id = test.id; 24 | } 25 | 26 | return pointer; 27 | }; 28 | -------------------------------------------------------------------------------- /src/factories/createWebpackConfiguration.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import path from 'path'; 4 | import webpack from 'webpack'; 5 | 6 | export default (basePath: string) => { 7 | let configuration = { 8 | devtool: 'source-map', 9 | entry: { 10 | main: [ 11 | path.resolve(__dirname, '../report-web-app') 12 | ] 13 | }, 14 | mode: 'production', 15 | module: { 16 | rules: [ 17 | { 18 | include: path.resolve(__dirname, '../report-web-app'), 19 | loader: 'babel-loader', 20 | test: /\.js$/ 21 | }, 22 | { 23 | loaders: [ 24 | 'style-loader?sourceMap', 25 | { 26 | loader: 'css-loader', 27 | options: { 28 | camelCase: true, 29 | importLoaders: 1, 30 | localIdentName: '[path]___[local]___[hash:base64:5]', 31 | modules: true 32 | } 33 | }, 34 | 'resolve-url-loader', 35 | { 36 | loader: 'postcss-loader' 37 | }, 38 | 'sass-loader' 39 | ], 40 | test: /\.scss/ 41 | } 42 | ] 43 | }, 44 | output: { 45 | filename: '[name].bundle.js', 46 | path: path.resolve(__dirname, '../../.static'), 47 | publicPath: basePath + 'static/' 48 | } 49 | }; 50 | 51 | // eslint-disable-next-line no-process-env 52 | if (process.env.NODE_ENV === 'development') { 53 | configuration = { 54 | ...configuration, 55 | devtool: 'inline-source-map', 56 | entry: { 57 | ...configuration.entry, 58 | main: [ 59 | 'webpack-hot-middleware/client', 60 | ...configuration.entry.main 61 | ] 62 | }, 63 | mode: 'development', 64 | plugins: [ 65 | new webpack.HotModuleReplacementPlugin() 66 | ] 67 | }; 68 | } 69 | 70 | return configuration; 71 | }; 72 | -------------------------------------------------------------------------------- /src/factories/index.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | export {default as createGraphqlClient} from './createGraphqlClient'; 4 | export {default as createGraphqlMiddleware} from './createGraphqlMiddleware'; 5 | export {default as createLabelCollection} from './createLabelCollection'; 6 | export {default as createMonitor} from './createMonitor'; 7 | export {default as createResponseBody} from './createResponseBody'; 8 | export {default as createTestPointer} from './createTestPointer'; 9 | export {default as createWebpackConfiguration} from './createWebpackConfiguration'; 10 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | export {default as createAlertController} from './factories/createAlertController'; 4 | export type { 5 | AlertConfigurationType, 6 | MonitorConfigurationType, 7 | RegisteredTestType, 8 | TestSuiteFactoryType 9 | } from './types'; 10 | -------------------------------------------------------------------------------- /src/report-web-app/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "plugins": [ 3 | "@babel/transform-flow-strip-types", 4 | "@babel/plugin-proposal-class-properties" 5 | ], 6 | "presets": [ 7 | [ 8 | "@babel/env", 9 | { 10 | "exclude": [ 11 | "transform-regenerator" 12 | ], 13 | "loose": true, 14 | "targets": { 15 | "browsers": [ 16 | ">1%" 17 | ] 18 | }, 19 | "useBuiltIns": false 20 | } 21 | ], 22 | "@babel/preset-react" 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /src/report-web-app/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "canonical/react" 4 | ], 5 | "rules": { 6 | "filenames/match-regex": 0, 7 | "react/jsx-no-bind": 0, 8 | "react/jsx-one-expression-per-line": 0, 9 | "react/no-set-state": 0, 10 | "react/prefer-stateless-function": 0 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/report-web-app/components/FailingTestComponent.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import React from 'react'; 4 | import styles from '../main.scss'; 5 | import type { 6 | FailingTestType 7 | } from '../types'; 8 | import TextButtonComponent from './TextButtonComponent'; 9 | 10 | type FailingTestComponentPropsType = {| 11 | +onExplainRegisteredTest: (registeredTestId: string) => void, 12 | +onFilterExpressionChange: (filterExpression: string) => void, 13 | +registeredTest: FailingTestType 14 | |}; 15 | 16 | class FailingTestComponent extends React.Component { 17 | render () { 18 | const { 19 | onExplainRegisteredTest, 20 | onFilterExpressionChange, 21 | registeredTest 22 | } = this.props; 23 | 24 | const labelElements = Object.keys(registeredTest.labels).map((labelName) => { 25 | return
  • { 28 | onFilterExpressionChange(JSON.stringify({ 29 | labels: { 30 | [labelName]: { 31 | // eslint-disable-next-line id-match 32 | $eq: registeredTest.labels[labelName] 33 | } 34 | } 35 | })); 36 | }} 37 | > 38 |
    39 |
    40 | {labelName} 41 |
    42 |
    43 | {registeredTest.labels[labelName]} 44 |
    45 |
    46 |
    • ; 47 | }); 48 | 49 | const labelsElement = ; 58 | 59 | let navigationElement; 60 | 61 | if (registeredTest.explainIsAvailable) { 62 | navigationElement =
    63 | { 65 | onExplainRegisteredTest(registeredTest.id); 66 | }} 67 | > 68 | Explain 69 | 70 |
    ; 71 | } 72 | 73 | let errorElement; 74 | 75 | if (registeredTest.lastError) { 76 | errorElement = ; 87 | } 88 | 89 | return
    90 |
    91 |
    92 | Failing test 93 |
    94 |
    95 | {registeredTest.name} 96 |
    97 |
    98 |
    99 |
    100 | Priority 101 |
    102 |
    103 | {registeredTest.priority} 104 |
    105 |
    106 | {labelsElement} 107 | {errorElement} 108 | {navigationElement} 109 |
    ; 110 | } 111 | } 112 | 113 | export default FailingTestComponent; 114 | -------------------------------------------------------------------------------- /src/report-web-app/components/TextButtonComponent.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import React from 'react'; 4 | import styles from '../main.scss'; 5 | 6 | type TextButtonComponentPropsType = {| 7 | +children: string, 8 | +onClick: () => void 9 | |}; 10 | 11 | class TextButtonComponent extends React.Component { 12 | handleClick = (event: *) => { 13 | event.preventDefault(); 14 | 15 | this.props.onClick(); 16 | }; 17 | 18 | render () { 19 | return ; 26 | } 27 | } 28 | 29 | export default TextButtonComponent; 30 | -------------------------------------------------------------------------------- /src/report-web-app/config.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | export const API_URL = window.PALANTIR.config.API_URL; 4 | -------------------------------------------------------------------------------- /src/report-web-app/containers/Root.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import delay from 'delay'; 4 | import React from 'react'; 5 | // eslint-disable-next-line import/no-named-as-default 6 | import ApolloClient from 'apollo-boost'; 7 | import sift from 'sift'; 8 | import JSON5 from 'json5'; 9 | import { 10 | hot 11 | } from 'react-hot-loader'; 12 | import styles from '../main.scss'; 13 | import { 14 | getFailingRegisteredTests, 15 | getRegisteredTestById 16 | } from '../services'; 17 | import FailingTestComponent from '../components/FailingTestComponent'; 18 | import type { 19 | FailingTestType, 20 | SubjectTestType 21 | } from '../types'; 22 | import { 23 | API_URL 24 | } from '../config'; 25 | import TestFilter from './TestFilter'; 26 | 27 | /** 28 | * @property userFailingRegisteredTests Sorted and filtered `failingRegisteredTests`. 29 | */ 30 | type RootStateType = {| 31 | +failingRegisteredTests: $ReadOnlyArray, 32 | +failingRegisteredTestsError: Error | null, 33 | +failingRegisteredTestsIsLoaded: boolean, 34 | +subjectTest: SubjectTestType | null, 35 | +subjectTestError: Error | null, 36 | +subjectTestIsLoading: boolean, 37 | +testFilterExpression: string, 38 | +testFilterExpressionError: Error | null, 39 | +userFailingRegisteredTests: $ReadOnlyArray 40 | |}; 41 | 42 | const graphqlClient = new ApolloClient({ 43 | uri: API_URL 44 | }); 45 | 46 | const pool = async (update) => { 47 | while (true) { 48 | try { 49 | const failingRegisteredTests = await getFailingRegisteredTests(graphqlClient); 50 | 51 | update(null, failingRegisteredTests); 52 | } catch (error) { 53 | update(error, []); 54 | } 55 | 56 | // @todo Use GraphQL subscriptions. 57 | await delay(5000); 58 | } 59 | }; 60 | 61 | const createUserFailingRegisteredTests = (failingRegisteredTests: $ReadOnlyArray, filterExpression: string) => { 62 | let userTests = failingRegisteredTests 63 | .slice(0) 64 | .sort((a, b) => { 65 | return a.priority - b.priority; 66 | }); 67 | 68 | if (filterExpression) { 69 | let parsedExpression; 70 | 71 | try { 72 | parsedExpression = JSON5.parse(filterExpression); 73 | } catch (error) { 74 | throw new Error('Invalid JSON5 expression.'); 75 | } 76 | 77 | try { 78 | userTests = sift(parsedExpression, userTests); 79 | } catch (error) { 80 | throw new Error('Invalid MongoDB query.'); 81 | } 82 | } 83 | 84 | return userTests; 85 | }; 86 | 87 | class Root extends React.Component { 88 | constructor () { 89 | super(); 90 | 91 | this.state = { 92 | failingRegisteredTests: [], 93 | failingRegisteredTestsError: null, 94 | failingRegisteredTestsIsLoaded: false, 95 | subjectTest: null, 96 | subjectTestError: null, 97 | subjectTestIsLoading: false, 98 | testFilterExpression: '', 99 | testFilterExpressionError: null, 100 | userFailingRegisteredTests: [] 101 | }; 102 | 103 | pool((error, failingRegisteredTests) => { 104 | this.setState((currentState) => { 105 | return { 106 | ...currentState, 107 | failingRegisteredTests: failingRegisteredTests || [], 108 | failingRegisteredTestsError: error || null, 109 | failingRegisteredTestsIsLoaded: true 110 | }; 111 | }); 112 | }); 113 | } 114 | 115 | componentDidUpdate (prevProps, prevState) { 116 | if (prevState.testFilterExpression !== this.state.testFilterExpression || JSON.stringify(prevState.failingRegisteredTests) !== JSON.stringify(this.state.failingRegisteredTests)) { 117 | // eslint-disable-next-line react/no-did-update-set-state 118 | this.setState((currentState) => { 119 | try { 120 | return { 121 | testFilterExpressionError: null, 122 | userFailingRegisteredTests: createUserFailingRegisteredTests(currentState.failingRegisteredTests, currentState.testFilterExpression) 123 | }; 124 | } catch (error) { 125 | return { 126 | testFilterExpressionError: error 127 | }; 128 | } 129 | }); 130 | } 131 | } 132 | 133 | handleExplainRegisteredTest = (registeredTestId: string) => { 134 | this.setState({ 135 | subjectTestIsLoading: true 136 | }); 137 | 138 | (async () => { 139 | try { 140 | const subjectTest = await getRegisteredTestById(graphqlClient, registeredTestId); 141 | 142 | // eslint-disable-next-line no-console 143 | console.info('subjectTest', subjectTest); 144 | 145 | this.setState({ 146 | subjectTest, 147 | subjectTestError: null, 148 | subjectTestIsLoading: false 149 | }); 150 | } catch (error) { 151 | this.setState({ 152 | subjectTest: null, 153 | subjectTestError: error, 154 | subjectTestIsLoading: false 155 | }); 156 | } 157 | })(); 158 | }; 159 | 160 | handleTestFilterExpressionChange = (testFilterExpression: string) => { 161 | this.setState({ 162 | testFilterExpression 163 | }); 164 | }; 165 | 166 | render () { 167 | const { 168 | userFailingRegisteredTests, 169 | failingRegisteredTestsError, 170 | failingRegisteredTestsIsLoaded, 171 | subjectTest, 172 | subjectTestError, 173 | subjectTestIsLoading 174 | } = this.state; 175 | 176 | let bodyElement; 177 | 178 | if (!failingRegisteredTestsIsLoaded) { 179 | bodyElement =
    180 | Loading the failing test cases. 181 |
    ; 182 | } else if (failingRegisteredTestsError) { 183 | bodyElement =
    184 | Unable to load the failing test cases. 185 |
    ; 186 | } else if (userFailingRegisteredTests.length) { 187 | const failingTestElements = userFailingRegisteredTests 188 | .map((registeredTest) => { 189 | return
  • 190 | 195 |
    • ; 196 | }); 197 | 198 | bodyElement =
      199 | {failingTestElements} 200 |
    ; 201 | } else { 202 | bodyElement =
    203 |

    204 | All tests are passing. 205 |

    206 |
    ; 207 | } 208 | 209 | let testPanelElement; 210 | 211 | if (subjectTestError) { 212 | testPanelElement =
    213 |
    214 | {subjectTestError.message} 215 |
    216 |
    ; 217 | } else if (subjectTestIsLoading) { 218 | testPanelElement =
    219 |
    220 | Loading... 221 |
    222 |
    ; 223 | } else if (subjectTest) { 224 | const explanationElements = subjectTest.explain.map((explanation) => { 225 | return
  • 226 |
    227 |
    228 | {explanation.name} 229 |
    230 |
    231 | {JSON.stringify(explanation.explanation, null, 2)} 232 |
    233 |
    234 |
    • ; 235 | }); 236 | 237 | testPanelElement =
    238 |
      239 | {explanationElements} 240 |
    241 |
    ; 242 | } else { 243 | testPanelElement =
    244 |
    245 | Select a test. 246 |
    247 |
    ; 248 | } 249 | 250 | return
    251 | 256 | {bodyElement} 257 | {testPanelElement} 258 |
    ; 259 | } 260 | } 261 | 262 | export default hot(module)(Root); 263 | -------------------------------------------------------------------------------- /src/report-web-app/containers/TestFilter.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import React from 'react'; 4 | import styles from '../main.scss'; 5 | 6 | type TestFilterPropsType = {| 7 | +filterExpression: string, 8 | +filterExpressionError: Error | null, 9 | +onFilterExpressionChange: (filterExpression: string) => void 10 | |}; 11 | 12 | class TestFilter extends React.Component { 13 | render () { 14 | let filterErrorElement; 15 | 16 | if (this.props.filterExpressionError) { 17 | filterErrorElement =
    18 | {this.props.filterExpressionError.message} 19 |
    ; 20 | } 21 | 22 | return
    23 | { 25 | this.props.onFilterExpressionChange(event.target.value); 26 | }} 27 | placeholder='Failing test filter' 28 | type='text' 29 | value={this.props.filterExpression} 30 | /> 31 |
    32 |

    33 | Failing tests can be filtered using MongoDB query expressions. 34 |

    35 |
    36 | {filterErrorElement} 37 |
    ; 38 | } 39 | } 40 | 41 | export default TestFilter; 42 | -------------------------------------------------------------------------------- /src/report-web-app/factories/createLabelsObject.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import type { 4 | LabelCollectionType, 5 | LabelsType 6 | } from '../../types'; 7 | 8 | export default (labelCollection: LabelCollectionType): LabelsType => { 9 | // eslint-disable-next-line flowtype/no-weak-types 10 | const labels: Object = {}; 11 | 12 | for (const label of labelCollection) { 13 | labels[label.name] = label.value; 14 | } 15 | 16 | return labels; 17 | }; 18 | -------------------------------------------------------------------------------- /src/report-web-app/factories/index.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | export {default as createLabelsObject} from './createLabelsObject'; 4 | -------------------------------------------------------------------------------- /src/report-web-app/index.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import React from 'react'; 4 | import ReactDOM from 'react-dom'; 5 | import Root from './containers/Root'; 6 | 7 | const main = async () => { 8 | const app = document.getElementById('app'); 9 | 10 | if (!app) { 11 | throw new Error('app element is not present'); 12 | } 13 | 14 | ReactDOM.render(, app); 15 | }; 16 | 17 | main(); 18 | -------------------------------------------------------------------------------- /src/report-web-app/main.scss: -------------------------------------------------------------------------------- 1 | $color-text-secondary-text: #99a1a3; 2 | $color-border-primary: #858c8f; 3 | $color-border-secondary: #282a2f; 4 | $color-error-background: #cc0000; 5 | 6 | :global { 7 | html, 8 | body { 9 | height: 100%; 10 | } 11 | 12 | body { 13 | font-size: 16px; 14 | font-family: 'Roboto', sans-serif; 15 | line-height: 1.5; 16 | display: flex; 17 | flex-flow: column; 18 | flex-shrink: 0; 19 | flex-grow: 1; 20 | overflow-x: hidden; 21 | overflow-y: scroll; 22 | 23 | font-smoothing: grayscale; 24 | text-rendering: optimizeLegibility; 25 | 26 | background: #151619; 27 | 28 | -webkit-text-size-adjust: 100%; 29 | -webkit-font-smoothing: antialiased; 30 | -moz-osx-font-smoothing: grayscale; 31 | } 32 | 33 | * { 34 | box-sizing: border-box; 35 | margin: 0; 36 | padding: 0; 37 | outline: none; 38 | } 39 | 40 | a { 41 | text-decoration: none; 42 | color: inherit; 43 | } 44 | 45 | button { 46 | border: none; 47 | } 48 | 49 | h1, 50 | h2, 51 | h3, 52 | h4, 53 | h5, 54 | h5 { 55 | font-weight: inherit; 56 | font: inherit; 57 | } 58 | 59 | li { 60 | list-style: none; 61 | } 62 | 63 | table { 64 | border-spacing: 0; 65 | } 66 | 67 | p + p { 68 | margin-top: 5px; 69 | } 70 | 71 | // @see https://stackoverflow.com/a/23211766/368691 72 | input { 73 | -webkit-appearance: none; 74 | -moz-appearance: none; 75 | appearance: none; 76 | } 77 | 78 | // @see https://stackoverflow.com/a/36043286/368691 79 | input:not([type="radio"]):not([type="checkbox"]) { 80 | border-radius: 0; 81 | } 82 | 83 | #app { 84 | display: flex; 85 | flex-flow: column; 86 | flex-shrink: 0; 87 | flex-grow: 1; 88 | } 89 | 90 | #dashboard { 91 | margin: 20px; 92 | margin-right: calc(30vw + 20px); 93 | overflow: hidden; 94 | } 95 | } 96 | 97 | .activity-message { 98 | border: 1px solid $color-border-secondary; 99 | color: $color-text-secondary-text; 100 | padding: 20px; 101 | text-align: center; 102 | } 103 | 104 | .error-message { 105 | background: $color-error-background; 106 | color: #fff; 107 | padding: 21px; 108 | text-align: center; 109 | } 110 | 111 | .tests { 112 | display: grid; 113 | grid-template-columns: repeat(auto-fill, minmax(400px, 1fr)); 114 | grid-gap: 20px; 115 | 116 | h2 { 117 | font-size: 16px; 118 | font-weight: bold; 119 | } 120 | 121 | & > li { 122 | background: #191b1f; 123 | border: 1px solid $color-border-secondary; 124 | color: #fff; 125 | overflow: hidden; 126 | box-shadow: 0px 3px 1px -2px rgba(0,0,0,0.2), 0px 2px 2px 0px rgba(0,0,0,0.14), 0px 1px 5px 0px rgba(0,0,0,0.12); 127 | 128 | &::before { 129 | content: ''; 130 | display: block; 131 | height: 5px; 132 | background: $color-error-background; 133 | } 134 | } 135 | } 136 | 137 | .failing-test { 138 | padding: 10px; 139 | 140 | & > * { 141 | margin: 10px; 142 | } 143 | 144 | .error { 145 | background: $color-error-background; 146 | overflow: hidden; 147 | 148 | h1 { 149 | font-weight: bold; 150 | } 151 | 152 | .stack { 153 | background: #ffe7e8; 154 | border-left: 5px solid #e66465; 155 | font-family: consolas,"Liberation Mono",courier,monospace; 156 | padding: 10px; 157 | white-space: pre; 158 | color: #111; 159 | overflow: scroll; 160 | } 161 | 162 | & > * { 163 | margin: 10px; 164 | } 165 | } 166 | 167 | .name { 168 | dt { 169 | font-size: 16px; 170 | color: $color-text-secondary-text; 171 | margin: 0 0 5px 0; 172 | } 173 | 174 | dd { 175 | color: #fff; 176 | font-weight: bold; 177 | } 178 | } 179 | 180 | .labels { 181 | overflow: hidden; 182 | 183 | & > h1 { 184 | color: $color-text-secondary-text; 185 | margin: 0 0 5px 0; 186 | } 187 | 188 | & > ul { 189 | li { 190 | display: block; 191 | border: 1px solid $color-border-secondary; 192 | padding: 5px 10px; 193 | margin: 0 5px 5px 0; 194 | cursor: pointer; 195 | 196 | dl { 197 | display: block; 198 | 199 | dt { 200 | display: inline-block; 201 | font-weight: bold; 202 | } 203 | } 204 | 205 | &:hover { 206 | background: rgba(0,0,0,0.2); 207 | } 208 | } 209 | } 210 | } 211 | } 212 | 213 | .all-tests-passing-message { 214 | background: #39aa55; 215 | color: #fff; 216 | padding: 20px; 217 | text-align: center; 218 | } 219 | 220 | .text-button { 221 | display: inline-flex; 222 | user-select: none; 223 | cursor: pointer; 224 | font: inherit; 225 | padding: 10px 20px; 226 | border: 1px solid $color-border-primary; 227 | background: rgba(0,0,0,.2); 228 | color: $color-text-secondary-text; 229 | 230 | &:hover { 231 | background: rgba(0,0,0,.5); 232 | border-color: $color-border-secondary; 233 | color: #fff; 234 | } 235 | } 236 | 237 | .test-panel { 238 | border-left: 1px solid $color-border-secondary; 239 | background: #191b1f; 240 | position: fixed; 241 | width: 30vw; 242 | top: 0; 243 | bottom: 0; 244 | right: 0; 245 | overflow-y: scroll; 246 | 247 | .message { 248 | border: 1px solid $color-border-secondary; 249 | text-align: center; 250 | padding: 20px; 251 | margin: 20px; 252 | color: $color-text-secondary-text; 253 | } 254 | 255 | .explanations { 256 | margin: 20px; 257 | 258 | li { 259 | margin-top: 20px; 260 | 261 | &:first-child { 262 | margin: 0; 263 | } 264 | } 265 | 266 | dt { 267 | display: block; 268 | color: $color-text-secondary-text; 269 | margin: 0 0 10px 0; 270 | } 271 | 272 | dd { 273 | background: #ffe7e8; 274 | border-left: 5px solid #e66465; 275 | font-family: consolas,"Liberation Mono",courier,monospace; 276 | padding: 10px; 277 | white-space: pre; 278 | display: block; 279 | } 280 | } 281 | } 282 | 283 | .test-filter { 284 | margin-bottom: 20px; 285 | 286 | input { 287 | background: #0a0b0c; 288 | border: 1px solid $color-border-secondary; 289 | box-shadow: 0px 3px 1px -2px rgba(0,0,0,0.2), 0px 2px 2px 0px rgba(0,0,0,0.14), 0px 1px 5px 0px rgba(0,0,0,0.12); 290 | display: block; 291 | position: relative; 292 | width: 100%; 293 | height: 40px; 294 | font: inherit; 295 | color: $color-text-secondary-text; 296 | padding: 10px; 297 | 298 | &:focus { 299 | background: #252e2e; 300 | border-color: #73c990; 301 | color: #fff; 302 | } 303 | } 304 | 305 | .instructions { 306 | padding: 10px; 307 | margin-top: 10px; 308 | color: $color-text-secondary-text; 309 | 310 | a { 311 | text-decoration: underline; 312 | } 313 | } 314 | 315 | .error-message { 316 | background: $color-error-background; 317 | color: #fff; 318 | padding: 10px; 319 | margin-top: 10px; 320 | } 321 | } 322 | -------------------------------------------------------------------------------- /src/report-web-app/postcss.config.js: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line import/no-commonjs 2 | module.exports = () => { 3 | return {}; 4 | }; 5 | -------------------------------------------------------------------------------- /src/report-web-app/services.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import gql from 'graphql-tag'; 4 | // eslint-disable-next-line import/no-named-as-default 5 | import ApolloClient from 'apollo-boost'; 6 | import { 7 | createLabelsObject 8 | } from './factories'; 9 | import type { 10 | FailingTestType, 11 | SubjectTestType 12 | } from './types'; 13 | 14 | export const getFailingRegisteredTests = async (graphqlClient: ApolloClient): Promise<$ReadOnlyArray> => { 15 | // eslint-disable-next-line no-restricted-syntax 16 | const query = gql` 17 | { 18 | failingRegisteredTests { 19 | edges { 20 | node { 21 | id 22 | name 23 | explainIsAvailable 24 | labels { 25 | name 26 | value 27 | } 28 | lastError { 29 | message 30 | name 31 | stack 32 | } 33 | lastTestedAt 34 | priority 35 | testIsFailing 36 | } 37 | } 38 | } 39 | } 40 | `; 41 | 42 | const result = await graphqlClient.query({ 43 | fetchPolicy: 'no-cache', 44 | query 45 | }); 46 | 47 | return result.data.failingRegisteredTests.edges.map((edge) => { 48 | return { 49 | ...edge.node, 50 | labels: createLabelsObject(edge.node.labels) 51 | }; 52 | }); 53 | }; 54 | 55 | export const getRegisteredTestById = async (graphqlClient: ApolloClient, registeredTestId: string): Promise => { 56 | // eslint-disable-next-line no-restricted-syntax 57 | const query = gql` 58 | query getRegisteredTestById ( 59 | $registeredTestId: ID! 60 | ) { 61 | getRegisteredTestById ( 62 | registeredTestId: $registeredTestId 63 | ) { 64 | id 65 | name 66 | explain { 67 | explanation 68 | name 69 | } 70 | } 71 | } 72 | `; 73 | 74 | const result = await graphqlClient.query({ 75 | fetchPolicy: 'no-cache', 76 | query, 77 | variables: { 78 | registeredTestId 79 | } 80 | }); 81 | 82 | return result.data.getRegisteredTestById; 83 | }; 84 | -------------------------------------------------------------------------------- /src/report-web-app/types.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | // eslint-disable-next-line flowtype/no-weak-types 4 | export type FailingTestType = Object; 5 | 6 | // eslint-disable-next-line flowtype/no-weak-types 7 | export type SubjectTestType = Object; 8 | -------------------------------------------------------------------------------- /src/report-web-app/webpack.configuration.production.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import createWebpackConfiguration from '../factories/createWebpackConfiguration'; 4 | 5 | export default createWebpackConfiguration('/'); 6 | -------------------------------------------------------------------------------- /src/resolvers/Query.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import { 4 | NotFoundError 5 | } from '../errors'; 6 | import type { 7 | ResolverType 8 | } from '../types'; 9 | 10 | const Query: ResolverType = { 11 | failingRegisteredTests: (root, parameters, context) => { 12 | const registeredTests = context.monitor.getRegisteredTests(); 13 | 14 | const pageInfo = { 15 | hasNextPage: false 16 | }; 17 | 18 | const edges = registeredTests 19 | .filter((registeredTest) => { 20 | return registeredTest.testIsFailing; 21 | }) 22 | .map((registeredTest) => { 23 | return { 24 | cursor: Buffer.from(registeredTest.id, 'binary').toString('base64'), 25 | node: registeredTest 26 | }; 27 | }); 28 | 29 | return { 30 | edges, 31 | pageInfo, 32 | totalCount: registeredTests.length 33 | }; 34 | }, 35 | getRegisteredTestById: (root, parameters, context) => { 36 | const registeredTests = context.monitor.getRegisteredTests(); 37 | 38 | const subjectRegisteredTest = registeredTests.find((maybeSubjectRegisteredTest) => { 39 | return maybeSubjectRegisteredTest.id === parameters.registeredTestId; 40 | }); 41 | 42 | if (!subjectRegisteredTest) { 43 | throw new NotFoundError(); 44 | } 45 | 46 | return subjectRegisteredTest; 47 | }, 48 | registeredTests: (root, parameters, context) => { 49 | const registeredTests = context.monitor.getRegisteredTests(); 50 | 51 | const pageInfo = { 52 | hasNextPage: false 53 | }; 54 | 55 | const edges = registeredTests.map((registeredTest) => { 56 | return { 57 | cursor: Buffer.from(registeredTest.id, 'binary').toString('base64'), 58 | node: registeredTest 59 | }; 60 | }); 61 | 62 | return { 63 | edges, 64 | pageInfo, 65 | totalCount: registeredTests.length 66 | }; 67 | } 68 | }; 69 | 70 | export default Query; 71 | -------------------------------------------------------------------------------- /src/resolvers/RegisteredTest.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import prettyMs from 'pretty-ms'; 4 | import { 5 | createLabelCollection 6 | } from '../factories'; 7 | import { 8 | explainTest 9 | } from '../routines'; 10 | import type { 11 | RegisteredTestType, 12 | ResolverType 13 | } from '../types'; 14 | 15 | const RegisteredTest: ResolverType = { 16 | explain: (registeredTest, parameters, context) => { 17 | if (!registeredTest.explain) { 18 | return []; 19 | } 20 | 21 | // @todo Add runtime validation of the explain output. 22 | return explainTest(context.configuration, registeredTest); 23 | }, 24 | explainIsAvailable: (registeredTest) => { 25 | return Boolean(registeredTest.explain); 26 | }, 27 | interval: (registeredTest) => { 28 | return { 29 | human: prettyMs(registeredTest.interval(0), { 30 | verbose: true 31 | }), 32 | milliseconds: registeredTest.interval(0) 33 | }; 34 | }, 35 | labels: (registeredTest) => { 36 | return createLabelCollection(registeredTest.labels); 37 | }, 38 | lastTestedAt: (registeredTest) => { 39 | return registeredTest.lastTestedAt ? new Date(registeredTest.lastTestedAt) : null; 40 | } 41 | }; 42 | 43 | export default RegisteredTest; 44 | -------------------------------------------------------------------------------- /src/resolvers/index.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | export {default as Query} from './Query'; 4 | export {default as RegisteredTest} from './RegisteredTest'; 5 | -------------------------------------------------------------------------------- /src/routines/evaluateRegisteredTest.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import serializeError from 'serialize-error'; 4 | import Logger from '../Logger'; 5 | import { 6 | createTestPointer 7 | } from '../factories'; 8 | import type { 9 | MonitorConfigurationType, 10 | RegisteredTestType 11 | } from '../types'; 12 | 13 | const log = Logger.child({ 14 | namespace: 'evaluateRegisteredTest' 15 | }); 16 | 17 | const updateTest = (registeredTest: RegisteredTestType, assertionResult?: boolean, assertionError?: Error) => { 18 | const testIsFailing = Boolean(assertionResult === false || assertionError); 19 | 20 | registeredTest.consecutiveFailureCount = testIsFailing ? (registeredTest.consecutiveFailureCount || 0) + 1 : 0; 21 | registeredTest.lastError = assertionError ? serializeError(assertionError) : null; 22 | registeredTest.lastTestedAt = Date.now(); 23 | registeredTest.testIsFailing = testIsFailing; 24 | }; 25 | 26 | export default async (configuration: MonitorConfigurationType, registeredTest: RegisteredTestType) => { 27 | const context = configuration.beforeTest ? await configuration.beforeTest(registeredTest) : {}; 28 | 29 | let assertionResult; 30 | let assertionError; 31 | 32 | try { 33 | assertionResult = await registeredTest.assert(context); 34 | } catch (error) { 35 | assertionError = error; 36 | 37 | log.warn({ 38 | error: serializeError(assertionError), 39 | test: createTestPointer(registeredTest) 40 | }, '%s test assertion resulted in an error', registeredTest.name); 41 | } 42 | 43 | updateTest(registeredTest, assertionResult, assertionError); 44 | 45 | if (configuration.afterTest) { 46 | await configuration.afterTest(registeredTest, context); 47 | } 48 | }; 49 | -------------------------------------------------------------------------------- /src/routines/explainTest.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import serializeError from 'serialize-error'; 4 | import Logger from '../Logger'; 5 | import { 6 | createTestPointer 7 | } from '../factories'; 8 | import type { 9 | MonitorConfigurationType, 10 | RegisteredTestType 11 | } from '../types'; 12 | 13 | const log = Logger.child({ 14 | namespace: 'explainTest' 15 | }); 16 | 17 | export default async (configuration: MonitorConfigurationType, registeredTest: RegisteredTestType) => { 18 | if (!registeredTest.explain) { 19 | throw new Error('Test does not have explain method.'); 20 | } 21 | 22 | const context = configuration.beforeTest ? await configuration.beforeTest(registeredTest) : {}; 23 | 24 | let explanationResult; 25 | let explanationError; 26 | 27 | try { 28 | explanationResult = await registeredTest.explain(context); 29 | } catch (error) { 30 | explanationError = error; 31 | 32 | log.warn({ 33 | error: serializeError(explanationError), 34 | test: createTestPointer(registeredTest) 35 | }, '%s test explanation resulted in an error', registeredTest.name); 36 | } 37 | 38 | if (configuration.afterTest) { 39 | await configuration.afterTest(registeredTest, context); 40 | } 41 | 42 | if (explanationError) { 43 | throw explanationError; 44 | } 45 | 46 | return explanationResult; 47 | }; 48 | -------------------------------------------------------------------------------- /src/routines/index.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | export {default as evaluateRegisteredTest} from './evaluateRegisteredTest'; 4 | export {default as explainTest} from './explainTest'; 5 | -------------------------------------------------------------------------------- /src/schema.graphql: -------------------------------------------------------------------------------- 1 | scalar JSON 2 | scalar DateTime 3 | 4 | interface Node { 5 | id: ID! 6 | } 7 | 8 | interface Edge { 9 | cursor: String! 10 | } 11 | 12 | interface Connection { 13 | totalCount: Int! 14 | pageInfo: PageInfo! 15 | } 16 | 17 | type PageInfo { 18 | hasNextPage: Boolean! 19 | # hasPreviousPage: Boolean! 20 | # startCursor: String 21 | # endCursor: String 22 | } 23 | 24 | type RegisteredTestsConnection implements Connection { 25 | totalCount: Int! 26 | edges: [RegisteredTestsEdge!]! 27 | pageInfo: PageInfo! 28 | } 29 | 30 | type RegisteredTestsEdge implements Edge { 31 | cursor: String! 32 | node: RegisteredTest 33 | } 34 | 35 | type Interval { 36 | milliseconds: Int! 37 | human: String! 38 | } 39 | 40 | type Error { 41 | name: String! 42 | message: String! 43 | stack: String! 44 | } 45 | 46 | type Label { 47 | name: String! 48 | value: String! 49 | } 50 | 51 | type Explanation { 52 | name: String! 53 | explanation: JSON 54 | } 55 | 56 | type RegisteredTest implements Node { 57 | configuration: JSON 58 | consecutiveFailureCount: Int 59 | explain: [Explanation!]! 60 | explainIsAvailable: Boolean! 61 | id: ID! 62 | interval: Interval! 63 | labels: [Label!]! 64 | lastError: Error 65 | lastTestedAt: DateTime 66 | name: String! 67 | priority: Int 68 | testIsFailing: Boolean 69 | } 70 | 71 | type Query { 72 | getRegisteredTestById ( 73 | registeredTestId: ID! 74 | ): RegisteredTest! 75 | registeredTests: RegisteredTestsConnection! 76 | failingRegisteredTests: RegisteredTestsConnection! 77 | } 78 | -------------------------------------------------------------------------------- /src/types.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | /* eslint-disable import/exports-last */ 4 | 5 | // eslint-disable-next-line no-use-before-define 6 | type SerializableObjectValueType = string | number | boolean | SerializableObjectType | $ReadOnlyArray; 7 | 8 | export type SerializableObjectType = { 9 | +[key: string]: SerializableObjectValueType 10 | }; 11 | 12 | // eslint-disable-next-line flowtype/no-weak-types 13 | type TestContextType = Object; 14 | 15 | export type TestSubjectType = SerializableObjectValueType; 16 | 17 | export type LabelsType = { 18 | +[key: string]: string 19 | }; 20 | 21 | type LabelPairType = {| 22 | +name: string, 23 | +value: string 24 | |}; 25 | 26 | export type LabelCollectionType = $ReadOnlyArray; 27 | 28 | type ExplanationType = {| 29 | +explanation: $ReadOnlyArray | SerializableObjectType, 30 | +name: string 31 | |}; 32 | 33 | /** 34 | * @property assert Evaluates user defined script. The result (boolean) indicates if test is passing. 35 | * @property configuration User defined configuration accessible by the `beforeTest`. 36 | * @property explain Provides debugging information about the test. 37 | * @property interval A function that describes the time when the test needs to be re-run. 38 | * @property labels Arbitrary key=value labels used to categorise the tests. 39 | * @property name Unique name of the test. A combination of test + labels must be unique across all test suites. 40 | * @property priority A numeric value (0-100) indicating the importance of the test. Low value indicates high priority. 41 | */ 42 | export type TestType = {| 43 | +assert: (context: TestContextType) => Promise, 44 | +configuration?: SerializableObjectType, 45 | +explain?: (context: TestContextType) => Promise<$ReadOnlyArray>, 46 | +interval: (consecutiveFailureCount: number) => number, 47 | +labels: LabelsType, 48 | +name: string, 49 | +priority: number 50 | |}; 51 | 52 | export type TestIdPayloadInputType = { 53 | +labels: LabelsType, 54 | +name: string 55 | }; 56 | 57 | export type TestSuiteType = {| 58 | +tests: $ReadOnlyArray 59 | |}; 60 | 61 | export type TestSuiteFactoryType = (refreshTestSuite: () => void) => Promise | TestSuiteType; 62 | 63 | type NormalizedErrorType = {| 64 | +message: string, 65 | +name: string, 66 | +stack: string 67 | |}; 68 | 69 | export type TestExecutionType = {| 70 | +error: NormalizedErrorType | null, 71 | +executedAt: number, 72 | +testIsFailing: boolean 73 | |}; 74 | 75 | export type RegisteredTestType = {| 76 | +id: string, 77 | ...TestType, 78 | consecutiveFailureCount: number | null, 79 | lastError: NormalizedErrorType | null, 80 | lastTestedAt: number | null, 81 | testIsFailing: boolean | null 82 | |}; 83 | 84 | export type AlertConfigurationType = {| 85 | +onNewFailingTest?: (registeredTest: RegisteredTestType) => void, 86 | +onRecoveredTest?: (registeredTest: RegisteredTestType) => void 87 | |}; 88 | 89 | /** 90 | * @property beforeTest Creates test execution context. 91 | */ 92 | export type MonitorConfigurationType = {| 93 | +after?: () => Promise | void, 94 | +afterTest?: (test?: RegisteredTestType, context?: TestContextType) => Promise | void, 95 | +before?: () => Promise | void, 96 | +beforeTest?: (test?: RegisteredTestType) => Promise | TestContextType 97 | |}; 98 | 99 | type MonitorType = {| 100 | +getRegisteredTests: () => $ReadOnlyArray 101 | |}; 102 | 103 | export type ResolverContextType = {| 104 | +configuration: MonitorConfigurationType, 105 | +monitor: MonitorType 106 | |}; 107 | 108 | export type ResolverType = { 109 | +[key: string]: (parent: T, parameters: P, context: ResolverContextType) => * 110 | }; 111 | -------------------------------------------------------------------------------- /src/utilities/importModule.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | /* eslint-disable global-require, import/no-dynamic-require */ 4 | 5 | import path from 'path'; 6 | 7 | export default (modulePath: string): * => { 8 | // $FlowFixMe 9 | const module = require(path.resolve(process.cwd(), modulePath)); 10 | 11 | return module.default || module; 12 | }; 13 | -------------------------------------------------------------------------------- /src/utilities/index.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | export {default as importModule} from './importModule'; 4 | export {default as resolveFilePathExpression} from './resolveFilePathExpression'; 5 | -------------------------------------------------------------------------------- /src/utilities/localeCompareProperty.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | export default (a: ?string, b: ?string) => { 4 | if (a || b) { 5 | if (!a) { 6 | return -1; 7 | } 8 | 9 | return b ? a.localeCompare(b) : 1; 10 | } 11 | 12 | return 0; 13 | }; 14 | -------------------------------------------------------------------------------- /src/utilities/resolveFilePathExpression.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import { 4 | sync as glob 5 | } from 'glob'; 6 | 7 | export default (input: string | $ReadOnlyArray) => { 8 | const filePathExpressions = Array.isArray(input) ? input : [input]; 9 | 10 | const filePaths = []; 11 | 12 | for (const filePathExpression of filePathExpressions) { 13 | const resolvedFilePaths = glob(filePathExpression); 14 | 15 | filePaths.push(...resolvedFilePaths); 16 | } 17 | 18 | return filePaths; 19 | }; 20 | -------------------------------------------------------------------------------- /test/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "canonical/ava", 3 | "rules": { 4 | "filenames/match-regex": 0, 5 | "id-length": 0 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /test/palantir/assertions/assertUniqueTestIdPayloads.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import test from 'ava'; 4 | import assertUniqueTestIdPayloads from '../../../src/assertions/assertUniqueTestIdPayloads'; 5 | 6 | test('does not throw error if all descriptions are unique', (t) => { 7 | const tests = [ 8 | { 9 | labels: {}, 10 | name: 'foo' 11 | }, 12 | { 13 | labels: {}, 14 | name: 'bar' 15 | } 16 | ]; 17 | 18 | t.notThrows((): void => { 19 | assertUniqueTestIdPayloads(tests); 20 | }); 21 | }); 22 | 23 | test('throws error if not all descriptions are unique', (t) => { 24 | const tests = [ 25 | { 26 | labels: {}, 27 | name: 'foo' 28 | }, 29 | { 30 | labels: {}, 31 | name: 'foo' 32 | } 33 | ]; 34 | 35 | t.throws((): void => { 36 | assertUniqueTestIdPayloads(tests); 37 | }); 38 | }); 39 | -------------------------------------------------------------------------------- /test/palantir/factories/createAlertController.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import test from 'ava'; 4 | import sinon from 'sinon'; 5 | import delay from 'delay'; 6 | import type { 7 | RegisteredTestType 8 | } from '../../../src/types'; 9 | import createAlertController from '../../../src/factories/createAlertController'; 10 | 11 | const createRegisteredTest = (id: number): RegisteredTestType => { 12 | // eslint-disable-next-line flowtype/no-weak-types 13 | const registeredTest: any = { 14 | id 15 | }; 16 | 17 | return registeredTest; 18 | }; 19 | 20 | const assertControllerState = ( 21 | t, 22 | controller, 23 | failureSpy, 24 | recoverySpy, 25 | onFailureIsCalled: boolean, 26 | onRecoveryIsCalled: boolean, 27 | delayedFailingTestCount: number, 28 | delayedRecoveringTestCount: number 29 | ) => { 30 | t.true(failureSpy.called === onFailureIsCalled); 31 | t.true(recoverySpy.called === onRecoveryIsCalled); 32 | t.true(controller.getDelayedFailingTests().length === delayedFailingTestCount); 33 | t.true(controller.getDelayedRecoveringTests().length === delayedRecoveringTestCount); 34 | }; 35 | 36 | const constant = (value) => { 37 | return () => { 38 | return value; 39 | }; 40 | }; 41 | 42 | test('registered failing test is delayed `delayFailure` milliseconds before `onFailure` is triggered', async (t) => { 43 | const failureSpy = sinon.spy(); 44 | const recoverySpy = sinon.spy(); 45 | 46 | const controller = createAlertController({ 47 | delayFailure: constant(100), 48 | delayRecovery: constant(100), 49 | onFailure: failureSpy, 50 | onRecovery: recoverySpy 51 | }); 52 | 53 | assertControllerState(t, controller, failureSpy, recoverySpy, false, false, 0, 0); 54 | 55 | const failingTest = createRegisteredTest(1); 56 | 57 | controller.registerTestFailure(failingTest); 58 | 59 | assertControllerState(t, controller, failureSpy, recoverySpy, false, false, 1, 0); 60 | 61 | await delay(100); 62 | 63 | assertControllerState(t, controller, failureSpy, recoverySpy, true, false, 0, 0); 64 | 65 | t.true(failureSpy.calledOnce); 66 | t.true(failureSpy.calledWith(failingTest)); 67 | }); 68 | 69 | test('registered failing test does not trigger `onFailure` if test recovers within `delayFailure` milliseconds', async (t) => { 70 | const failureSpy = sinon.spy(); 71 | const recoverySpy = sinon.spy(); 72 | 73 | const controller = createAlertController({ 74 | delayFailure: constant(100), 75 | delayRecovery: constant(100), 76 | onFailure: failureSpy, 77 | onRecovery: recoverySpy 78 | }); 79 | 80 | assertControllerState(t, controller, failureSpy, recoverySpy, false, false, 0, 0); 81 | 82 | const failingTest = createRegisteredTest(1); 83 | 84 | controller.registerTestFailure(failingTest); 85 | 86 | assertControllerState(t, controller, failureSpy, recoverySpy, false, false, 1, 0); 87 | 88 | await delay(50); 89 | 90 | controller.registerTestRecovery(failingTest); 91 | 92 | await delay(100); 93 | 94 | assertControllerState(t, controller, failureSpy, recoverySpy, false, true, 0, 0); 95 | }); 96 | 97 | test('registering the same test multiple times does not create duplicate `delayedTest` entries', (t) => { 98 | const controller = createAlertController({ 99 | delayFailure: constant(100), 100 | delayRecovery: constant(100), 101 | onFailure: () => {}, 102 | onRecovery: () => {} 103 | }); 104 | 105 | t.true(controller.getDelayedFailingTests().length === 0); 106 | 107 | controller.registerTestFailure(createRegisteredTest(1)); 108 | 109 | t.true(controller.getDelayedFailingTests().length === 1); 110 | 111 | controller.registerTestFailure(createRegisteredTest(1)); 112 | 113 | t.true(controller.getDelayedFailingTests().length === 1); 114 | 115 | controller.registerTestFailure(createRegisteredTest(2)); 116 | 117 | t.true(controller.getDelayedFailingTests().length === 2); 118 | }); 119 | 120 | test('registered recovering test is delayed `delayRecovery` milliseconds before `onRecovery` is triggered', async (t) => { 121 | const failureSpy = sinon.spy(); 122 | const recoverySpy = sinon.spy(); 123 | 124 | const controller = createAlertController({ 125 | delayFailure: constant(100), 126 | delayRecovery: constant(100), 127 | onFailure: failureSpy, 128 | onRecovery: recoverySpy 129 | }); 130 | 131 | assertControllerState(t, controller, failureSpy, recoverySpy, false, false, 0, 0); 132 | 133 | const failingTest = createRegisteredTest(1); 134 | 135 | controller.registerTestFailure(failingTest); 136 | 137 | assertControllerState(t, controller, failureSpy, recoverySpy, false, false, 1, 0); 138 | 139 | await delay(50); 140 | 141 | controller.registerTestRecovery(failingTest); 142 | 143 | await delay(50); 144 | 145 | assertControllerState(t, controller, failureSpy, recoverySpy, false, false, 0, 1); 146 | 147 | await delay(100); 148 | 149 | assertControllerState(t, controller, failureSpy, recoverySpy, false, true, 0, 0); 150 | 151 | t.true(recoverySpy.calledOnce); 152 | t.true(recoverySpy.calledWith(failingTest)); 153 | }); 154 | 155 | test('registered recovering test fails if another failure is detected within a `delayRecovery` timeframe without further recovery within `delayFailure` timeframe', async (t) => { 156 | const failureSpy = sinon.spy(); 157 | const recoverySpy = sinon.spy(); 158 | 159 | const controller = createAlertController({ 160 | delayFailure: constant(100), 161 | delayRecovery: constant(100), 162 | onFailure: failureSpy, 163 | onRecovery: recoverySpy 164 | }); 165 | 166 | assertControllerState(t, controller, failureSpy, recoverySpy, false, false, 0, 0); 167 | 168 | const failingTest = createRegisteredTest(1); 169 | 170 | controller.registerTestFailure(failingTest); 171 | 172 | await delay(50); 173 | 174 | assertControllerState(t, controller, failureSpy, recoverySpy, false, false, 1, 0); 175 | 176 | controller.registerTestRecovery(failingTest); 177 | 178 | await delay(50); 179 | 180 | assertControllerState(t, controller, failureSpy, recoverySpy, false, false, 0, 1); 181 | 182 | controller.registerTestFailure(failingTest); 183 | 184 | await delay(50); 185 | 186 | assertControllerState(t, controller, failureSpy, recoverySpy, false, false, 1, 0); 187 | 188 | await delay(200); 189 | 190 | assertControllerState(t, controller, failureSpy, recoverySpy, true, false, 0, 0); 191 | }); 192 | -------------------------------------------------------------------------------- /test/palantir/factories/createIntervalRoutine.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import test from 'ava'; 4 | import sinon from 'sinon'; 5 | import delay from 'delay'; 6 | import createIntervalRoutine from '../../../src/factories/createIntervalRoutine'; 7 | 8 | test('repeats routine every X milliseconds until cancelled', async (t) => { 9 | const spy = sinon.stub().returns(100); 10 | 11 | const cancel = createIntervalRoutine(spy); 12 | 13 | await delay(550); 14 | 15 | cancel(); 16 | 17 | t.true(spy.callCount === 6); 18 | }); 19 | -------------------------------------------------------------------------------- /test/palantir/factories/createLabelCollection.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import test from 'ava'; 4 | import createLabelCollection from '../../../src/factories/createLabelCollection'; 5 | 6 | test('converts object to a {key: value} collection', (t) => { 7 | const collection = createLabelCollection({ 8 | foo: 'bar' 9 | }); 10 | 11 | t.deepEqual(collection, [ 12 | { 13 | name: 'foo', 14 | value: 'bar' 15 | } 16 | ]); 17 | }); 18 | 19 | test('sorts collection by label name', (t) => { 20 | const collection = createLabelCollection({ 21 | foo: 'bar', 22 | 23 | // eslint-disable-next-line sort-keys 24 | baz: 'qux' 25 | }); 26 | 27 | t.deepEqual(collection, [ 28 | { 29 | name: 'baz', 30 | value: 'qux' 31 | }, 32 | { 33 | name: 'foo', 34 | value: 'bar' 35 | } 36 | ]); 37 | }); 38 | -------------------------------------------------------------------------------- /test/palantir/routines/evaluateRegisteredTest.js: -------------------------------------------------------------------------------- 1 | // @flow 2 | 3 | import test from 'ava'; 4 | import sinon from 'sinon'; 5 | import type { 6 | MonitorConfigurationType, 7 | RegisteredTestType 8 | } from '../../../src/types'; 9 | import evaluateRegisteredTest from '../../../src/routines/evaluateRegisteredTest'; 10 | 11 | const createTest = (registeredTest: $Shape<{...RegisteredTestType}> = {}): RegisteredTestType => { 12 | return { 13 | // @see https://github.com/facebook/flow/issues/6974 14 | ...{ 15 | // eslint-disable-next-line no-unused-vars 16 | assert: async (context) => { 17 | return true; 18 | }, 19 | consecutiveFailureCount: null, 20 | id: '1', 21 | // eslint-disable-next-line no-unused-vars 22 | interval: (consecutiveFailureCount) => { 23 | return 100; 24 | }, 25 | labels: {}, 26 | lastError: null, 27 | lastTestedAt: null, 28 | name: '', 29 | testIsFailing: null 30 | }, 31 | ...registeredTest 32 | }; 33 | }; 34 | 35 | const createTestConfiguration = (input: $Shape<{...MonitorConfigurationType}>): MonitorConfigurationType => { 36 | return { 37 | after: () => {}, 38 | afterTest: () => {}, 39 | before: () => {}, 40 | beforeTest: () => { 41 | return {}; 42 | }, 43 | ...input 44 | }; 45 | }; 46 | 47 | test('calls assert', async (t) => { 48 | const registeredTest = createTest(); 49 | 50 | const spy = sinon.spy(registeredTest, 'assert'); 51 | 52 | await evaluateRegisteredTest({}, registeredTest); 53 | 54 | t.true(spy.calledOnce); 55 | t.true(registeredTest.consecutiveFailureCount === 0); 56 | t.true(registeredTest.lastTestedAt !== null); 57 | t.true(registeredTest.testIsFailing === false); 58 | }); 59 | 60 | test('uses beforeTest to create assertion context', async (t) => { 61 | const expectedContext = {}; 62 | 63 | const configuration = createTestConfiguration({ 64 | beforeTest: () => { 65 | return expectedContext; 66 | } 67 | }); 68 | 69 | const registeredTest = createTest(); 70 | 71 | const spy = sinon.spy(registeredTest, 'assert'); 72 | 73 | await evaluateRegisteredTest(configuration, registeredTest); 74 | 75 | t.true(spy.calledOnce); 76 | t.true(spy.calledWith(expectedContext)); 77 | }); 78 | 79 | test('uses afterTest to teardown code', async (t) => { 80 | const expectedContext = {}; 81 | 82 | const configuration = createTestConfiguration({ 83 | beforeTest: () => { 84 | return expectedContext; 85 | } 86 | }); 87 | 88 | const registeredTest = createTest(); 89 | 90 | const spy = sinon.spy(configuration, 'afterTest'); 91 | 92 | await evaluateRegisteredTest(configuration, registeredTest); 93 | 94 | t.true(spy.calledOnce); 95 | t.true(spy.calledWith(registeredTest, expectedContext)); 96 | }); 97 | 98 | test('marks test as failing if assertion throws an error', async (t) => { 99 | const registeredTest = createTest(); 100 | 101 | const spy = sinon 102 | .stub(registeredTest, 'assert') 103 | .callsFake(() => { 104 | throw new Error('foo'); 105 | }); 106 | 107 | await evaluateRegisteredTest({}, registeredTest); 108 | 109 | t.true(spy.calledOnce); 110 | t.true(registeredTest.consecutiveFailureCount === 1); 111 | t.true(registeredTest.lastError && registeredTest.lastError.message === 'foo'); 112 | t.true(registeredTest.lastTestedAt !== null); 113 | t.true(registeredTest.testIsFailing === true); 114 | }); 115 | 116 | test('marks test as failing if assert returns false', async (t) => { 117 | const registeredTest = createTest(); 118 | 119 | const spy = sinon 120 | .stub(registeredTest, 'assert') 121 | .callsFake(() => { 122 | return false; 123 | }); 124 | 125 | await evaluateRegisteredTest({}, registeredTest); 126 | 127 | t.true(spy.calledOnce); 128 | t.true(registeredTest.consecutiveFailureCount === 1); 129 | t.true(registeredTest.lastError === null); 130 | t.true(registeredTest.lastTestedAt !== null); 131 | t.true(registeredTest.testIsFailing === true); 132 | }); 133 | --------------------------------------------------------------------------------