├── .editorconfig ├── .eslintrc.js ├── .gitignore ├── .gitlab └── issue_templates │ └── story.md ├── .nvmrc ├── .vscode ├── launch.json └── settings.json ├── CHANGELOG.md ├── Dockerfile ├── LICENSE.md ├── README.md ├── bin ├── cleanup ├── evaluate ├── serve-metrics ├── server └── sync-cache ├── config ├── default.js └── sample.js ├── frontend ├── .eslintrc.js ├── package-lock.json ├── package.json ├── src │ ├── Autocomplete.js │ ├── LighthouseReport.js │ ├── PresetSelect.js │ ├── Selection.js │ ├── UrlState.js │ ├── View.js │ ├── index.html │ ├── index.js │ ├── material-icons.css │ └── styles.scss └── webpack.config.js ├── package-lock.json ├── package.json ├── public ├── index.html ├── main.js └── screenshot.png ├── src ├── bootstrap.js ├── evaluate.js ├── lighthouse.js ├── lighthouse.spec.js ├── prom-client.learn-spec.js ├── receivers │ ├── directory.js │ ├── directory.spec.js │ ├── file-prometheus.js │ ├── file-prometheus.spec.js │ ├── json-metrics.js │ ├── new-relic.integration-spec.js │ ├── new-relic.js │ ├── new-relic.spec.js │ ├── prometheus.js │ ├── prometheus.spec.js │ ├── push-prometheus.js │ ├── timestamped-directory.js │ └── timestamped-directory.spec.js ├── report-cleanup.js ├── report-cleanup.spec.js ├── report.js ├── report.spec.js ├── reports.js ├── reports.spec.js ├── test-fixtures │ ├── report.json │ └── reports │ │ ├── other │ │ └── 2018-08-03T09_20_00.059Z │ │ │ ├── KITS_desktop-fast_config.json.gz │ │ │ ├── KITS_desktop-fast_report.html.gz │ │ │ ├── KITS_desktop-fast_report.json.gz │ │ │ ├── KITS_desktop-slow_config.json.gz │ │ │ ├── KITS_desktop-slow_report.html.gz │ │ │ ├── KITS_desktop-slow_report.json.gz │ │ │ ├── KITS_mobile-fast_config.json.gz │ │ │ ├── KITS_mobile-fast_report.html.gz │ │ │ ├── KITS_mobile-fast_report.json.gz │ │ │ ├── KITS_mobile-slow_config.json.gz │ │ │ ├── KITS_mobile-slow_report.html.gz │ │ │ └── KITS_mobile-slow_report.json.gz │ │ └── store │ │ ├── 2018-08-02T00_00_00.034Z │ │ └── .gitkeep │ │ ├── 2018-08-02T14_10_57.975Z │ │ ├── KITS_desktop-slow_config.json.gz │ │ ├── KITS_desktop-slow_report.html.gz │ │ ├── KITS_desktop-slow_report.json.gz │ │ ├── KITS_mobile-fast_config.json.gz │ │ ├── KITS_mobile-fast_report.html.gz │ │ ├── KITS_mobile-fast_report.json.gz │ │ ├── KITS_mobile-slow_config.json.gz │ │ ├── KITS_mobile-slow_report.html.gz │ │ └── KITS_mobile-slow_report.json.gz │ │ └── 2018-08-03T09_10_00.013Z │ │ ├── KITS_desktop-fast_config.json.gz │ │ ├── KITS_desktop-fast_report.html.gz │ │ ├── KITS_desktop-fast_report.json.gz │ │ ├── KITS_desktop-slow_config.json.gz │ │ ├── KITS_desktop-slow_report.html.gz │ │ ├── KITS_desktop-slow_report.json.gz │ │ ├── KITS_mobile-fast_config.json.gz │ │ ├── KITS_mobile-fast_report.html.gz │ │ ├── KITS_mobile-fast_report.json.gz │ │ ├── KITS_mobile-slow_config.json.gz │ │ ├── KITS_mobile-slow_report.html.gz │ │ └── KITS_mobile-slow_report.json.gz └── webserver │ ├── healthz.js │ ├── html-report.js │ ├── index.js │ ├── metrics.js │ ├── static-files.js │ └── webserver.js └── tools └── prometheus ├── Dockerfile ├── entrypoint.sh ├── prometheus.yml └── start-docker.sh /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 4 6 | end_of_line = lf 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | insert_final_newline = false -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "env": { 3 | "browser": true, 4 | "es6": true, 5 | "mocha": true, 6 | "node": true, 7 | "jquery": true 8 | }, 9 | "globals": { 10 | "$": true, 11 | "moment": true, 12 | "Mustache": true 13 | }, 14 | "extends": "eslint:recommended", 15 | "parserOptions": { 16 | "ecmaVersion": 2018 17 | }, 18 | "rules": { 19 | "indent": [ 20 | "warn", 21 | 4 22 | ], 23 | "linebreak-style": [ 24 | "warn", 25 | "unix" 26 | ], 27 | "quotes": [ 28 | "warn", 29 | "single" 30 | ], 31 | // "semi": "info", 32 | "mocha/no-exclusive-tests": "warn", 33 | "no-console": "off" 34 | }, 35 | "plugins": [ 36 | "mocha", 37 | "node" 38 | ] 39 | }; -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea/ 2 | /config/local*.* 3 | node_modules/ 4 | /public/metrics/ 5 | /public/metrics.json 6 | /reports/ 7 | /tmp/ 8 | *.local.js 9 | .*.sw[op] 10 | -------------------------------------------------------------------------------- /.gitlab/issue_templates/story.md: -------------------------------------------------------------------------------- 1 | **As a** user 2 | **I want** to do stuff 3 | **so that** I have benefit 4 | 5 | 6 | ## Acceptance Criteria 7 | 8 | * [ ] A 9 | * [ ] B 10 | * [ ] C 11 | 12 | 13 | ## Notes and Assumptions 14 | 15 | * D 16 | * E 17 | * F 18 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 12 -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Verwendet IntelliSense zum Ermitteln möglicher Attribute. 3 | // Zeigen Sie auf vorhandene Attribute, um die zugehörigen Beschreibungen anzuzeigen. 4 | // Weitere Informationen finden Sie unter https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "node", 9 | "request": "launch", 10 | "name": "Debug npm start", 11 | "runtimeExecutable": "npm", 12 | "runtimeArgs": [ 13 | "run", 14 | "start:debug" 15 | ], 16 | "port": 9229 17 | }, 18 | { 19 | "type": "node", 20 | "request": "launch", 21 | "name": "Debug npm run mocha", 22 | "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", 23 | "args": [ 24 | "-u", 25 | "tdd", 26 | "--timeout", 27 | "999999", 28 | "--colors", 29 | "src/**/*[.-]spec.js" 30 | ], 31 | "internalConsoleOptions": "openOnSessionStart" 32 | } 33 | ] 34 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "mocha.showInExplorer": true, 3 | "mocha.files.glob": "src/**/*.spec.js", 4 | "mocha.coverage": { 5 | "enable": false 6 | } 7 | } -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # 2.4.0 2 | 3 | * New option `reportsPollingEverySec` to poll the filesystem every X seconds, specifically for network shares with limited available inode watcher settings 4 | * New endpoint `/healthz`, which looks when the last report has been registered and returns a 500, if a configurable amount of time has passed since `expectedLastReportInSec` 5 | 6 | 7 | # 2.3.0 8 | 9 | * Higher performance by better utilizing the sqlite database report cache 10 | * New option to define the report cache via `cacheDir` - now defaults to `os.tmpdir()` 11 | * cache dir file renamed to `lightmon-cache.sqlite3` 12 | 13 | 14 | # 2.2.0 15 | 16 | * Lightmon now uses a file system cache based on sqlite to decrease startup times dramatically 17 | * FIX: Cleanup now correctly removes empty directories 18 | 19 | 20 | # 2.1.0 21 | 22 | * Receivers can provide an `afterEvaluation()`-method, which will be called before shutting down 23 | * A new receiver `jsonMetrics` for a reduced set of json results is added. 24 | 25 | 26 | # 2.0.0 27 | 28 | * Switch to puppeteer as launcher 29 | * Ability to add prehook-scripts to enable automation steps before launching an evaluation (e.g. logging into a homepage or filling a form) 30 | 31 | ## Breaking changes 32 | 33 | Due to the usage of puppeteer instead of chromeLauncher, the syntax of the configuration has changed. `chromeFlags` have been deprecated and replaced with a new `browserOptions` object, which takes in [puppeteer launch options](https://github.com/puppeteer/puppeteer/blob/v2.0.0/docs/api.md#puppeteerlaunchoptions). 34 | 35 | To migrate, you can use the `args` key inside: 36 | 37 | ``` 38 | // before 2.0.0 39 | { 40 | chromeFlags: [ 41 | "--some-option-for-chromium", 42 | "--no-sandbox" 43 | ] 44 | } 45 | 46 | // from 2.0.0 47 | { 48 | browserOptions: { 49 | headless: false, 50 | args: [ 51 | "--some-option-for-chromium", 52 | "--no-sandbox" 53 | ] 54 | } 55 | } 56 | ``` 57 | 58 | 59 | # 1.0.0 60 | 61 | Initial release -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:12-slim 2 | 3 | WORKDIR /app 4 | COPY . . 5 | 6 | RUN apt-get update \ 7 | && apt-get install -y fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 libxtst6 \ 8 | && apt-get install -y adwaita-icon-theme dconf-gsettings-backend dconf-service glib-networking glib-networking-common \ 9 | && apt-get install -y glib-networking-services gsettings-desktop-schemas gtk-update-icon-cache hicolor-icon-theme libasound2 \ 10 | && apt-get install -y libasound2-data libatk-bridge2.0-0 libatk1.0-0 libatk1.0-data libatspi2.0-0 libavahi-client3 libavahi-common-data \ 11 | && apt-get install -y libavahi-common3 libcairo-gobject2 libcairo2 libcolord2 libcroco3 libcups2 libcurl3-gnutls libdatrie1 libdbus-1-3 \ 12 | && apt-get install -y libdconf1 libdrm2 libegl1-mesa libepoxy0 libgbm1 libgdk-pixbuf2.0-0 libgdk-pixbuf2.0-common libglib2.0-0 \ 13 | && apt-get install -y libgraphite2-3 libgssapi-krb5-2 libgtk-3-0 libgtk-3-common libharfbuzz0b libicu57 libjbig0 libjpeg62-turbo \ 14 | && apt-get install -y libjson-glib-1.0-0 libjson-glib-1.0-common libk5crypto3 libkeyutils1 libkrb5-3 libkrb5support0 liblcms2-2 \ 15 | && apt-get install -y libnghttp2-14 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libpangoft2-1.0-0 libpixman-1-0 libproxy1v5 \ 16 | && apt-get install -y librest-0.7-0 librsvg2-2 librsvg2-common librtmp1 libsoup-gnome2.4-1 libsoup2.4-1 libssh2-1 libthai-data libthai0 \ 17 | && apt-get install -y libtiff5 libwayland-client0 libwayland-cursor0 libwayland-egl1-mesa libwayland-server0 libx11-xcb1 libxcb-dri2-0 \ 18 | && apt-get install -y libxcb-dri3-0 libxcb-present0 libxcb-render0 libxcb-shm0 libxcb-sync1 libxcb-xfixes0 libxcomposite1 libxcursor1 \ 19 | && apt-get install -y libxdamage1 libxfixes3 libxinerama1 libxkbcommon0 libxml2 libxrandr2 libxrender1 libxshmfence1 shared-mime-info \ 20 | && apt-get install -y xdg-utils xkb-data \ 21 | && apt-get clean \ 22 | && npm install -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Verivox 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Lighthouse Monitor 2 | ![License](https://img.shields.io/github/license/verivox/lighthouse-monitor.svg?style=flat-square) 3 | ![Snyk Vulnerabilities for GitHub Repo](https://img.shields.io/snyk/vulnerabilities/github/verivox/lighthouse-monitor.svg?style=flat-square) 4 | 5 | A lighthouse server, reporter and comparator to monitor multiple sites 6 | 7 | ![Screenshot](public/screenshot.png) 8 | 9 | 10 | 11 | ## Why 12 | 13 | Lighthouse is a very extensive performance monitor and Google provides it's own public instance through their webpage speedtest. But those results do not give any hint on the hardware used behind the scenes and are not exportable. And what do you do with the results of your self-run lighthouse instance? How do you ensure having enough historic data to see an optimization journey? How do you get the reports into other monitoring systems like Prometheus or Grafana Dashboards? 14 | 15 | Lightmon provides exactly that: 16 | 17 | * running lighthouse regularly against 18 | * a set of URLs, with 19 | * a set of profiles (e.g. desktop-on-cable, mobile-on-3G, mobile-on-LTE) 20 | * a webserver to easily choose and compare reports for 21 | * different URLs, 22 | * different profiles, and 23 | * different times 24 | * automatically cleanup older reports with intelligent retention 25 | * squash intra-daily reports to one per URL, profile and day after x days 26 | * squash intra-weekly reports to one per URL, profile and week after y weeks 27 | * multiple report receivers / connectors to other data analyzation and retention systems like 28 | * Directory 29 | * New Relic 30 | * Prometheus 31 | * easily extendable through programmatic use of lighthouse 32 | * easily add pages or different profiles for each page 33 | * send extra-headers to specific URLs (e.g. for authentication) 34 | * add a report receiver to alert on errors 35 | * ... 36 | * runs on every major OS or headless in docker 37 | 38 | 39 | 40 | ## Requirements 41 | 42 | * Node 10+ 43 | * chromium or chrome v54+ 44 | * disk-space 45 | 46 | 47 | 48 | ## Usage 49 | 50 | 1. install dependencies: `npm ci` 51 | 2. run an evaluation to evaluate a standard url set: `npm run evaluation` 52 | 3. run the webserver to view the results: `npm run server` open [http://localhost:3000](http://localhost:3000) 53 | 4. run cleanup: `npm run cleanup` 54 | 55 | 56 | 57 | ## Architecture 58 | 59 | The system consists of two components: a *webserver* to view reports and *evaluators* to evaluate websites. The former can run continuously, the latter needs to be scheduled (e.g. via a cronjob). 60 | 61 | You can run all of them on the same machine, but to keep the results consistent, you should consider splitting the webserver and evaluator. 62 | 63 | If you plan to scale up, see the section "Operation Considerations" in this document. 64 | 65 | 66 | 67 | ## Configuration 68 | 69 | Because we prefer *code over configuration*, we use javascript files that export the configuration. Therefore, the primary way of configuring Lightmon is through javascript configuration files - but it's easy to inject secret variables via environment variables. 70 | 71 | Copy over `config/sample.js` to `config/local.js` and edit it according to your needs. 72 | 73 | Have a look at `config/default.js` for the defaults we set and how everything ties together. If you would like to add environment support or a more complex setup, this is also the main file to look at. 74 | 75 | 76 | 77 | ## Contributing 78 | 79 | We are open for pull-requests and contributions. 80 | 81 | During evaluation, we'll check if 82 | 83 | * your PR has a proper test coverage, 84 | * test suite is green (`npm run test:all`), and 85 | * no linting errors are present (`npm run lint`) 86 | 87 | You can check everything by running `npm run preflight`. 88 | 89 | 90 | 91 | ## Environment Variables 92 | 93 | * `CHROME_PATH` to select the chrome installation manually 94 | * `NO_HEADLESS` to show the browser in the foreground (will not work in docker) 95 | * `PUPPETEER_SKIP_CHROMIUM_DOWNLOAD` to any value in order to avoid unnecessary downloads of chrome 96 | * `REPORT_DIR` changes directory for output of reports 97 | * `DEBUG` you can set the output level by specifying e.g. `export DEBUG='LIGHTMON:*'` 98 | 99 | 100 | 101 | ## Report Receivers 102 | 103 | There are several receivers you can configure to your liking in the aforementioned [config/local.js](./config/local.js). 104 | 105 | They are initialized during the configuration phase and will receive each report, after it has been returned by lighthouse. 106 | 107 | You can easily create your own receiver - have a look at these default receivers for examples. 108 | 109 | 110 | 111 | ### Timestamped-Directory 112 | 113 | This receiver takes a base directory (e.g. "reports/") as `reportDir` in the config file or as `REPORT_DIR` as environment variable. Under it, a timestamped directory for one run over all defined urls will be created with the following structure: 114 | 115 | ```text 116 | +- reports/ 117 | +- 2018-07-15T17:22:01.423Z/ 118 | +- vx-energy_mobile-fast_artifacts.json.gz 119 | +- vx-energy_mobile-fast_config.json.gz 120 | +- vx-energy_mobile-fast_report.json.gz 121 | +- [...] 122 | ``` 123 | 124 | The general form is `__[artifacts|config|report].json.gz`. 125 | 126 | You can view the reports of the *directory-receiver* in the [Lighthouse Viewer](https://googlechrome.github.io/lighthouse/viewer/). 127 | 128 | 129 | 130 | ### Directory 131 | 132 | Same as above, but it will not create a timestamped directory - the filestructure will be in the given target directory. 133 | 134 | 135 | 136 | ### New Relic 137 | 138 | This will push the results into [NewRelic](https://www.newrelic.com). Required configuration: 139 | 140 | * `NEW_RELIC_API_KEY` - your api key in New Relic 141 | * `NEW_RELIC_ACCOUNT_ID` - the account id of where to save the reports 142 | 143 | Alternatively, you can give the local configuration by overwriting receivers in a [config/local.js](./config/local.js). 144 | 145 | 146 | 147 | ### Prometheus 148 | 149 | [Prometheus](https://prometheus.io) is a monitoring system, which pulls (*scrapes*) data from different sources. The data will be available if you activate the Prometheus Receiver under the `/metrics` path of the webserver. It does not require any configuration. 150 | 151 | If you want to develop against prometheus, you can run a dockerized scraper, which will try to scrape your hosts port 9099 by running `tools/prometheus/start-docker.sh`. Please use the receiver *WebPrometheus* instead of *Prometheus*, which will start it's own express endpoint on that port. 152 | 153 | 154 | 155 | ### Implementing your own 156 | 157 | Implementing your own receiver is dead simple. Create a new class that implements one method: 158 | 159 | ```javascript 160 | // my-receiver.js 161 | class MyReceiver { 162 | constructor(options) { 163 | // whatever options you need to give to the receiver 164 | } 165 | 166 | async receive(report, config) { 167 | // handle report and config 168 | } 169 | } 170 | 171 | module.exports = { MyReceiver } 172 | ``` 173 | 174 | Then register your new receiver in the configuration file: 175 | 176 | ```javascript 177 | // config/local.js 178 | const { MyReceiver } = require('my-receiver') 179 | 180 | const receivers = [ 181 | new MyReceiver() 182 | ] 183 | 184 | module.exports = { 185 | receivers 186 | } 187 | ``` 188 | 189 | That's it - if you want to keep the default receivers, just add them to the array. 190 | 191 | 192 | 193 | ## Operation Considerations 194 | 195 | Running Lightmon is quite simple and runs nice on one node - however, in our tests we found quite some variance in the results when running evaluations in parallel. This is why every URL and every profile is not tested in parallel, but sequentially. 196 | 197 | To perform the tests in parallel, we suggest to run the webserver and evaluation nodes separately (not to mention other services like prometheus). If you want to run the webserver, every node needs access to a common data store, which supports the linux [Inotify](https://en.wikipedia.org/wiki/Inotify) interface - both samba-shares as well as windows-shares work fine. 198 | 199 | 200 | ### Storage considerations 201 | 202 | The necessary storage can grow quite big very fast - the necessary storage depends on 203 | 204 | * the number of URLs to check, 205 | * the number of profiles, 206 | * the size of the URL resources, 207 | * the number of tests, and 208 | * the retention policy 209 | 210 | All resources are gzipped by default to keep the size down. Here at Verivox, we test ~60 URLs with 3 profiles on an hourly basis, with a good mix of static pages and rich-client web-applications and on average, we get 3.5mb of gzipped data per URL. If you want to keep an intra-daily resolution for 14 days, daily resolution for half a year and weekly resolution for 5 years, the required space is 211 | 212 | * intra-daily: 3.5mb * 60 URLs * 24 hours * 14 days = ~70gb 213 | * daily: 3.5mb * 60 URLs * (182-14) days = ~35gb 214 | * weekly: 3.5mb * 60 URLs * 52 weeks * 5 years = ~54gb 215 | 216 | We provide a cleanup script in `/bin/cleanup` to automate the process - we run this via a nightly cronjob. Remember to restart the webserver to re-read the reports. 217 | 218 | 219 | 220 | ### Hardware and OS requirements 221 | 222 | The nodes for evaluation do not require much - one CPU and a gigabyte of RAM is enough. 223 | 224 | The webserver itself needs more memory, since it keeps a cache of the report meta data in memory - and if you need to double that if you want to run the cleanup from the same VM. For the aforementioned specifications, we use 4gb. 225 | 226 | Also note, that the webserver uses an efficient [file watcher](https://www.npmjs.com/package/chokidar) to handle the addition of results from evaluations, which uses the INotify interface on linux. This means, that you need to have a high amount of allowed file watchers for amount of files present - doubled, if you want to run the cleanup from the same VM. This can be increased on most linux systems via the `/etc/sysctl.conf`-file: 227 | 228 | ``` 229 | // /etc/sysctl.conf 230 | fs.inotify.max_user_watches=524288 231 | ``` 232 | 233 | The necessary number of file watches is dependent on the same variables as the storage. Taking the same example as before with the same retention strategy, we need 234 | 235 | * intra-daily: 3 profiles * 60 urls * 24 reports * 14 days = ~60.000 236 | * daily: 3 profiles * 60 urls * (182-14) days = ~30.000 237 | * weekly: 3 profiles * 60 urls * 52 weeks * 5 years = ~47.000 238 | 239 | So, around 140.000 watchers - doubled for cleanup and add some buffer on top for good measure of other processes. 240 | 241 | 242 | ### Speed considerations and variance 243 | 244 | Running the evaluation on your own hardware gives you full control and much better introspective into the results. But this also gives you the operational responsibility. 245 | 246 | You must ensure a good connectivity to the tested URLs and have a lookout for network congestion. 247 | 248 | If you run the evaluation on VMs, you must ensure that the VMs are not provisioned on different machines - if the underlying hardware CPU changes, your results will be change as well! This can usually be achieved by VM pinning. 249 | 250 | Also, server CPUs are usually a lot better than desktop CPUs and therefore perform a lot better than your target machines. You need to accommodate for that by setting an appropriate CPU slowdown multiplier in the configuration. 251 | 252 | The best results are achieved by getting some example laptops and smartphones, that resemble a good mixture of the hardware your customers use and run the evaluation on those. This will keep the variance between the results as low as possible. 253 | 254 | 255 | 256 | ## Authors 257 | 258 | Written by 259 | 260 | * [Horst Schneider](https://kumbier.it) 261 | * [Kim Almasan](https://kumbier.it) 262 | * [Lars Kumbier](https://kumbier.it) 263 | 264 | 265 | ## Licence 266 | 267 | [Licensed](./LICENSE.md) under MIT by the Verivox GmbH 268 | -------------------------------------------------------------------------------- /bin/cleanup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | const config = require('../config/default') 3 | const debug = require('debug') 4 | const fs = require('fs') 5 | const program = require('commander') 6 | const { Reports } = require('../src/reports') 7 | const { ReportCleanup } = require('../src/report-cleanup') 8 | const packageJson = require('../package.json') 9 | 10 | 11 | // bootstraps programs and runs submodules 12 | async function run(options) { 13 | debug('LIGHTMON:CLEANUP:DEBUG')('Setting up reports handler...') 14 | options.reports = new Reports(options.reportDir, false, null, false) 15 | const cleanup = new ReportCleanup(options) 16 | 17 | debug('LIGHTMON:CLEANUP:INFO')('Cleaning up reports') 18 | await cleanup.clean() 19 | 20 | debug('LIGHTMON:CLEANUP:INFO')('Deleting empty directories') 21 | await cleanup.purgeEmptyDirs() 22 | } 23 | 24 | 25 | program 26 | .version(packageJson.version, '-V, --version') 27 | .option('-d, --dry-run', 'do not actually do something') 28 | .option('-r, --report-dir ', 'report directory to clean up') 29 | .option('--retain-weekly ', 'reports older than are crunched to weekly resolution', parseInt) 30 | .option('--retain-daily ', 'reports older than are crunched to daily resolution', parseInt) 31 | .option('-v, --verbose', 'show debugging information') 32 | .parse(process.argv) 33 | 34 | 35 | if (program.retainWeekly === undefined && program.retainDaily === undefined) { 36 | console.error('ERROR: At least one option --retain-weekly or --retain-daily is required. Exiting.\n') 37 | program.outputHelp() 38 | process.exit(1) 39 | } 40 | 41 | if (program.retainWeekly !== undefined && isNaN(program.retainWeekly) || program.retainDaily !== undefined && isNaN(program.retainDaily)) { 42 | console.error('ERROR: --retain-weekly and --retain-daily require a number. Exiting.\n') 43 | program.outputHelp() 44 | process.exit(2) 45 | } 46 | 47 | if (program.retainWeekly && program.retainDaily && program.retainDaily >= program.retainWeekly) { 48 | console.error('ERROR: given both options, --retainDaily cannot be greater or equal than --retainWeekly. Exiting.\n') 49 | process.exit(3) 50 | } 51 | 52 | if (program.verbose) { 53 | debug.enable('LIGHTMON:*') 54 | } 55 | 56 | const reportDir = program.reportDir || config.reportDir 57 | 58 | if (!fs.existsSync(reportDir) || !fs.statSync(reportDir).isDirectory()) { 59 | console.error(`ERROR: reportDir does not exist or is not a directory: ${reportDir}. Exiting.\n`) 60 | process.exit(4) 61 | } 62 | 63 | 64 | run( 65 | { 66 | dryRun: program.dryRun !== undefined ? true : false, 67 | reportDir, 68 | retainDaily: program.retainDaily, 69 | retainWeekly: program.retainWeekly 70 | } 71 | ).catch(console.error) 72 | -------------------------------------------------------------------------------- /bin/evaluate: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | let config = require('../config/default') 3 | const debug = require('debug') 4 | const fs = require('fs-extra') 5 | const program = require('commander') 6 | const { evaluate } = require('../src/evaluate') 7 | const lockfile = require('proper-lockfile') 8 | const join = require('path').join 9 | 10 | const packageJson = require('../package.json') 11 | 12 | 13 | function validatePreset(preset, _config) { 14 | return preset === undefined ? 15 | true : 16 | new RegExp('^(' + Object.values(_config.presets).map(p => p.preset).join('|') + ')$').test(preset) 17 | } 18 | 19 | 20 | async function run(_config) { 21 | if (program.force === undefined && _config.lockFile) { 22 | try { 23 | await fs.ensureFile(_config.lockFile) 24 | await lockfile.lockSync(_config.lockFile, _config.lockFileOptions || {}) 25 | } catch (e) { 26 | console.error(`Could not acquire lock - there seems to be another process running. Error was:`) 27 | console.error(e) 28 | process.exit(2) 29 | } 30 | } 31 | 32 | await evaluate(_config) 33 | } 34 | 35 | 36 | program 37 | .version(packageJson.version, '-V, --version') 38 | .option('-f, --force', 'ignore lockfile') 39 | .option('-p, --preset ', `run only preset (${Object.values(config.presets).map(p => p.preset).join('|')})`) 40 | .option('-c, --config ', 'use this config file instead of config/default.js') 41 | .option('-v, --verbose', 'show verbose information') 42 | .parse(process.argv) 43 | 44 | 45 | if (program.verbose) { 46 | debug.enable('LIGHTMON:*') 47 | } 48 | 49 | if (program.config) { 50 | config = require(join(__dirname, '..', program.config)) 51 | } 52 | 53 | if (!validatePreset(program.preset, config)) { 54 | console.error(`Preset '${program.preset}' is unknown, exiting.`) 55 | program.outputHelp() 56 | process.exit(1) 57 | } 58 | 59 | if (program.preset) { 60 | config.presets = Object.values(config.presets).filter(p => p.preset === program.preset) 61 | debug('LIGHTMON:INFO')(`Restricting run to profile ${program.preset}`) 62 | } 63 | 64 | run(config).catch(console.error) 65 | -------------------------------------------------------------------------------- /bin/serve-metrics: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | const { Webserver, Metrics } = require('../src/webserver') 3 | const { WebserverOptions, prometheusMetricsFile, jsonMetricsFile } = require('../config/default') 4 | 5 | 6 | // bootstraps programs and runs submodules 7 | async function run() { 8 | new Metrics({ Webserver, prometheusMetricsFile, jsonMetricsFile }) 9 | new Webserver(WebserverOptions).start() 10 | } 11 | 12 | 13 | run().catch(console.error) -------------------------------------------------------------------------------- /bin/server: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | const config = require('../config/default') 3 | const { Reports } = require('../src/reports') 4 | const { Webserver, StaticFiles, HtmlReport, Healthz } = require('../src/webserver') 5 | const { WebserverOptions } = require('../config/default') 6 | 7 | // bootstraps programs and runs submodules 8 | async function run() { 9 | new StaticFiles({ ...WebserverOptions, Webserver }) 10 | new HtmlReport({ Webserver }) 11 | new Healthz({ Webserver, reports: new Reports(config.reportDir, false, null, false), expectedLastReportInSec: config.expectedLastReportInSec}) 12 | new Webserver({ ...WebserverOptions }).start() 13 | } 14 | 15 | run().catch(console.error) -------------------------------------------------------------------------------- /bin/sync-cache: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | const chokidar = require('chokidar') 3 | const config = require('../config/default') 4 | const debug = require('debug') 5 | const fs = require('fs-extra') 6 | const path = require('path') 7 | const program = require('commander') 8 | const packageJson = require('../package.json') 9 | const {Report} = require('../src/report'); 10 | const {ReportsCache} = require('../src/reports'); 11 | 12 | 13 | program 14 | .option('-r, --report-dir ', 'report directory to clean up') 15 | .option('-c, --cache-dir ', 'directory where the cache lives in') 16 | .version(packageJson.version, '-V, --version') 17 | .option('-v, --verbose', 'show debugging information') 18 | .parse(process.argv) 19 | 20 | const reportDir = program.reportDir || config.reportDir 21 | const cacheDir = program.cacheDir || config.cacheDir 22 | 23 | if (!fs.existsSync(reportDir) || !fs.statSync(reportDir).isDirectory()) { 24 | console.error(`ERROR: reportDir does not exist or is not a directory: ${reportDir}. Exiting.\n`) 25 | process.exit(2) 26 | } 27 | 28 | if (program.verbose) { 29 | debug.enable('LIGHTMON:*') 30 | } 31 | 32 | 33 | const reportsCache = new ReportsCache(path.join(cacheDir, 'lightmon-cache.sqlite3'), false) 34 | 35 | 36 | const chokidarOptions = { 37 | ignored: (candidate) => { 38 | if (fs.existsSync(candidate) && fs.lstatSync(candidate).isDirectory()) { 39 | return false 40 | } 41 | return !candidate.endsWith('_config.json.gz') 42 | }, 43 | ignoreInitial: false, 44 | persistent: true 45 | } 46 | 47 | if (config.reportsPollingEverySec === false) { 48 | chokidarOptions['usePolling'] = false 49 | } else { 50 | chokidarOptions['usePolling'] = true 51 | chokidarOptions['interval'] = config.reportsPollingEverySec * 1000 52 | chokidarOptions['binaryInterval'] = config.reportsPollingEverySec * 1000 53 | } 54 | 55 | debug('LIGHTMON:SYNC:INFO')('Sync started, setting up watcher') 56 | const watcher = chokidar.watch(reportDir, chokidarOptions).on('add', (changed) => { 57 | debug('LIGHTMON:SYNC:DEBUG')(`Adding report file ${changed}`) 58 | const report = Report.fromFile(path.dirname(changed), path.basename(changed)) 59 | reportsCache.upsert(report) 60 | }).on('ready', () => { 61 | debug('LIGHTMON:SYNC:INFO')('Finished adding watchers, checking for outdated entries') 62 | for (const r of reportsCache.outdated()) { 63 | debug('LIGHTMON:SYNC:DEBUG')(`Checking possibly deleted report ${r.id}`) 64 | 65 | if (r.config === null) { 66 | debug('LIGHTMON:SYNC:DEBUG')('+ Report does NOT exist, deleting from cache') 67 | reportsCache.delete(r) 68 | } else { 69 | debug('LIGHTMON:SYNC:DEBUG')('+ Report exists, touching cache') 70 | reportsCache.upsert(r) 71 | } 72 | } 73 | debug('LIGHTMON:SYNC:INFO')('Done checking for outdated entries') 74 | }).on('error', err => { 75 | debug('LIGHTMON:SYNC:INFO')('ERROR while syncing cache, continuing') 76 | debug('LIGHTMON:SYNC:INFO')(err) 77 | }).on('unlink', (filepath) => { 78 | if (!filepath.endsWith('_config.json.gz')) { 79 | return 80 | } 81 | 82 | if (filepath.substr(0,reportDir.length) !== reportDir) { 83 | debug('LIGHTMON:SYNC:WARNING')(`Report path does not start with reportDir, skipping: ${filepath}`) 84 | return 85 | } 86 | 87 | const cutPos = reportDir.substr(-1) === '/' ? reportDir.length : reportDir.length + 1 88 | let [created, filename] = filepath.substr(cutPos).split('/') 89 | if (reportDir.substr(reportDir.length-1) === '/') { 90 | created = created.substr(1) 91 | } 92 | created = created.split('_').join(':') // equivalent to replaceAll('_', ':'), which is not yet supported in node 93 | if (isNaN(Date.parse(created))) { 94 | debug('LIGHTMON:SYNC:WARNING')(`Could not create a valid created date from string, skipping: ${created}`) 95 | return 96 | } 97 | 98 | // presets may contain underscores, cause I was very stupid. This splits the filename correctly into the metadata. 99 | const [name, ] = filename.split('_') 100 | const preset = filename.substr(name.length + 1, filename.length - name.length - 1 - '_config.json.gz'.length) 101 | 102 | reportsCache.deleteByMeta(name, preset, created) 103 | }) 104 | 105 | 106 | process.on('SIGINT', () => { 107 | debug('LIGHTMON:SYNC:INFO')('Caught SIGINT, finishing up') 108 | watcher.close() 109 | process.exit(0) 110 | }) 111 | 112 | 113 | // run indefinitely 114 | ;(function runIndefinitely () { 115 | setTimeout(runIndefinitely, 1E9); 116 | })(); 117 | -------------------------------------------------------------------------------- /config/default.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This is the main configuration file. 3 | * 4 | * Note that this will be checked into the repository, so it MUST NOT contain 5 | * any secrets whatsoever. 6 | * 7 | * 8 | * ======================================================== 9 | * DO NOT MAKE CHANGES IN THIS FILE FOR DEPLOYMENT REASONS! 10 | * ======================================================== 11 | * 12 | * If you want to change settings, you can overwrite any by copying over 13 | * ./sample.js to ./local.js and change your settings there. 14 | * 15 | * 16 | * Some configuration options can be given via environment variables as well. 17 | */ 18 | const os = require('os') 19 | const path = require('path') 20 | 21 | const { TimestampedDirectory } = require('../src/receivers/timestamped-directory') 22 | const { FilePrometheus } = require('../src/receivers/file-prometheus') 23 | const { NewRelic } = require('../src/receivers/new-relic') 24 | 25 | const debug = require('debug') 26 | 27 | let localConfig = {} 28 | try { 29 | localConfig = require('./local') 30 | } catch (e) { 31 | if (e.code !== 'MODULE_NOT_FOUND') { 32 | debug('LIGHTMON:ERROR')('config/local.js found, but threw exception while importing: ', e.toString()) 33 | process.exit(1) 34 | } 35 | } 36 | 37 | 38 | 39 | /** 40 | * These are the default settings, allowing lighthouse to run in docker 41 | */ 42 | const presetDefault = { 43 | browserOptions: { 44 | headless: true, 45 | args: [ 46 | '--no-sandbox' 47 | ] 48 | }, 49 | onlyCategories: [ 50 | 'performance' 51 | ], 52 | output: [ 53 | 'html' 54 | ] 55 | } 56 | 57 | if (process.env.NO_HEADLESS) { 58 | presetDefault.browserOptions.headless = false 59 | } 60 | 61 | 62 | /** 63 | * By default, lighthouse will run through each of these presets for every 64 | * target url. 65 | */ 66 | const presets = { 67 | 'mobile_mid-slow': Object.assign( 68 | {}, 69 | presetDefault, 70 | { 71 | preset: 'mobile_mid-slow', 72 | formFactor: 'mobile', 73 | screenEmulation: { 74 | mobile: true, 75 | width: 360, 76 | height: 640, 77 | deviceScaleFactor: 2.625, 78 | disabled: false, 79 | }, 80 | emulatedUserAgent: 'Mozilla/5.0 (Linux; Android 7.0; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4590.2 Mobile Safari/537.36 Chrome-Lighthouse', 81 | throttlingMethod: 'simulate', 82 | throttling: { 83 | rttMs: 150, 84 | throughputKbps: 1638.4, 85 | cpuSlowdownMultiplier: 4 86 | } 87 | } 88 | ), 89 | 'mobile_mid-fast': Object.assign( 90 | {}, 91 | presetDefault, 92 | { 93 | preset: 'mobile_mid-fast', 94 | formFactor: 'mobile', 95 | screenEmulation: { 96 | mobile: true, 97 | width: 360, 98 | height: 640, 99 | deviceScaleFactor: 2.625, 100 | disabled: false, 101 | }, 102 | emulatedUserAgent: 'Mozilla/5.0 (Linux; Android 7.0; Moto G (4)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4590.2 Mobile Safari/537.36 Chrome-Lighthouse', 103 | throttlingMethod: 'simulate', 104 | throttling: { 105 | rttMs: 28, 106 | throughputKbps: 16000, 107 | cpuSlowdownMultiplier: 4 108 | } 109 | } 110 | ), 111 | 'desktop-fast': Object.assign( 112 | {}, 113 | presetDefault, 114 | { 115 | preset: 'desktop-fast', 116 | formFactor: 'desktop', 117 | screenEmulation: { 118 | mobile: false, 119 | width: 1350, 120 | height: 940, 121 | deviceScaleFactor: 1, 122 | disabled: false 123 | }, 124 | emulatedUserAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4590.2 Safari/537.36 Chrome-Lighthouse', 125 | throttlingMethod: 'simulate', 126 | throttling: { 127 | rttMs: 28, 128 | throughputKbps: 16000, 129 | cpuSlowdownMultiplier: 1 130 | } 131 | } 132 | ) 133 | } 134 | 135 | 136 | 137 | /** 138 | * These are the targets to run through. The name needs to be unique and will 139 | * be used in the receivers. 140 | */ 141 | const targets = [ 142 | { 143 | name: 'verivox-startpage', 144 | url: 'https://www.verivox.de/', 145 | }, 146 | { 147 | name: 'kumbierit', 148 | url: 'https://kumbier.it', 149 | }, 150 | ] 151 | 152 | 153 | const WebserverOptions = { 154 | port: process.env.NODE_PORT || 3000, 155 | publicFolder: 'public', 156 | } 157 | 158 | 159 | /** 160 | * Build the configuration object with some dynamic receiver insertion 161 | * depending on environment variables 162 | */ 163 | const receivers = [] 164 | 165 | /** 166 | * The directory where reports are stored. 167 | * @type {string} 168 | */ 169 | const reportDir = process.env.REPORT_DIR || localConfig.reportDir || path.join(os.tmpdir(), 'lightmon') 170 | 171 | /** 172 | * The directory, in which the cache for the reports are stored 173 | * @type {string|*} 174 | */ 175 | const cacheDir = process.env.CACHE_DIR || localConfig.cacheDir || os.tmpdir() 176 | 177 | const prometheusMetricsFile = path.join(WebserverOptions.publicFolder, 'metrics', 'index.html') 178 | const jsonMetricsFile = path.join(WebserverOptions.publicFolder, 'metrics.json') 179 | 180 | 181 | /** 182 | * How often do we expect reports? /healthz will return a 500, if no report has been seen in that time 183 | * defaults to once every 25h 184 | * @type {number} 185 | */ 186 | const expectedLastReportInSec = localConfig.expectedLastReportInSec || 25*60*60 187 | 188 | 189 | /** 190 | * If the reports are saved on a network folder, the filesystem watcher will most likely not work. 191 | * This instructs chokidar to use polling instead. Set to a non-number to disable polling. 192 | * @type {number|false} 193 | */ 194 | const reportsPollingEverySec = false 195 | 196 | 197 | receivers.push( 198 | new TimestampedDirectory(reportDir), 199 | new FilePrometheus(prometheusMetricsFile) 200 | ) 201 | 202 | const newRelicOptions = { 203 | accountId: process.env.NEW_RELIC_ACCOUNT_ID, 204 | apiKey: process.env.NEW_RELIC_API_KEY 205 | } 206 | if (newRelicOptions.accountId || newRelicOptions.apiKey) { 207 | receivers.push( 208 | new NewRelic(newRelicOptions) 209 | ) 210 | } 211 | 212 | 213 | /** 214 | * Do you want to use a mutex to make sure, that no evaluation will run in parallel? 215 | * 216 | * Uncomment the following line for the default linux behavior 217 | */ 218 | //const lockFile = process.geteuid() === 0 ? '/run/lightmon-evaluate.lock' : `/run/user/${process.geteuid()}/lightmon-evaluate.lock` 219 | const lockFile = null 220 | 221 | /** 222 | * We user proper-lockfile for process locking - you can add your options here if you want to. 223 | * @see https://github.com/moxystudio/node-proper-lockfile 224 | */ 225 | const lockFileOptions = { 226 | stale: 60000, 227 | update: 5000 228 | } 229 | 230 | 231 | /** 232 | * Overwrite this default configuration with the local.js, if present 233 | */ 234 | const config = Object.assign( 235 | { 236 | presets, 237 | receivers, 238 | targets, 239 | WebserverOptions, 240 | prometheusMetricsFile, 241 | jsonMetricsFile, 242 | lockFile, 243 | lockFileOptions, 244 | cacheDir, 245 | reportDir, 246 | expectedLastReportInSec, 247 | reportsPollingEverySec 248 | }, 249 | localConfig 250 | ) 251 | 252 | // TODO: this is getting ridiculous - create a real config class 253 | config.reportDir = reportDir 254 | 255 | // el cheapo validity check 256 | if (!presets || Object.keys(presets).length === 0 || 257 | !receivers || receivers.length === 0 || 258 | !targets || targets.length === 0) { 259 | debug('LIGHTMON:ERROR')('Invalid config. I require at least one preset, one receiver and one target') 260 | process.exit(1) 261 | } 262 | 263 | // all set 264 | module.exports = config 265 | -------------------------------------------------------------------------------- /config/sample.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This is a sample file, which MUST be copied to ./local.js to be loaded 3 | * 4 | * You can overwrite the module.exports from ./default.js here. 5 | * 6 | * An example: This would overwrite the default configuration of lighthouse. 7 | * During the configuration phase, the objects are merged in this order: 8 | * preset > target 9 | * This means, that we can overwrite preset data, if we provide a similar 10 | * key in the target definition. 11 | * In the example, we do not only change the default presets, but also add 12 | * a cookie to be sent for one specific target and activate all categories. 13 | */ 14 | 15 | const presets = { 16 | 'mobile-snail': { 17 | preset: 'mobile-snail', 18 | // browserOptions - is an options object that is accepted by launch command of puppeteer 19 | // https://github.com/puppeteer/puppeteer/blob/v2.0.0/docs/api.md#puppeteerlaunchoptions 20 | browserOptions: { 21 | headless: false, 22 | args: [ 23 | '--remote-debugging-port=0', 24 | '--no-sandbox', 25 | ] 26 | }, 27 | onlyCategories: [ 28 | 'performance', 29 | ], 30 | throttlingMethod: 'devtools' 31 | } 32 | } 33 | 34 | 35 | class LogMeIn { 36 | constructor(mailAddress, password, url) { 37 | this.url = url 38 | this.mailAddress = mailAddress 39 | this.password = password 40 | } 41 | 42 | async setup(browser) { 43 | const page = await browser.newPage() 44 | await page.goto(this.url) 45 | 46 | const emailInput = await page.waitForSelector('input#mail') 47 | await emailInput.type(this.mailAddress) 48 | const passwordInput = await page.waitForSelector('input#password') 49 | await passwordInput.type(this.password) 50 | 51 | const signInButtonSelector = 'submit' 52 | await page.waitForSelector(signInButtonSelector) 53 | await page.click(signInButtonSelector) 54 | 55 | await page.waitForSelector('#dashboard') 56 | await page.close() 57 | } 58 | } 59 | 60 | 61 | const loginPrehook = new LogMeIn('email@example.com', 62 | 'password', 'https://verivox.de/login-url/'); 63 | 64 | 65 | const targets = [ 66 | // this will take the default presets 67 | { 68 | name: 'landingpage', 69 | url: 'https://kumbier.it' 70 | }, 71 | 72 | // this will overwrite 'flags' and 'onlyCategories' in every preset 73 | // for this specific target only. 74 | { 75 | name: 'dashboard', 76 | url: 'https://kumbier.it', 77 | flags: { 78 | extraHeaders: {Cookie: 'sessionId=.....'} 79 | }, 80 | onlyCategories: [] 81 | }, 82 | /* 83 | This is a target with prehook. 84 | "prehook.setup" will be run before the evaluation of the target url. 85 | "loginPrehook" here is an instance of a class wich has a function called "setup" 86 | "setup" function accepts a browser "setup(browser)" instance of puppeteer for futher actions 87 | This case is a login prehook - opens a page with login form, fills in the email and password, 88 | submits the form, cookies are set and the evaluation of the account dashboard starts. 89 | */ 90 | { 91 | name: 'Account dashboard', 92 | url: 'https://verivox.de/mein-konto/', 93 | prehook: loginPrehook 94 | }, 95 | ] 96 | 97 | module.exports = { 98 | presets, 99 | targets 100 | } -------------------------------------------------------------------------------- /frontend/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "env": { 3 | "browser": true, 4 | "es6": true 5 | }, 6 | "extends": "eslint:recommended", 7 | "globals": { 8 | "Atomics": "readonly", 9 | "SharedArrayBuffer": "readonly" 10 | }, 11 | "parserOptions": { 12 | "ecmaVersion": 2018, 13 | "sourceType": "module" 14 | }, 15 | "rules": { 16 | } 17 | }; -------------------------------------------------------------------------------- /frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lightmon-frontend", 3 | "version": "1.0.0", 4 | "description": "", 5 | "private": true, 6 | "scripts": { 7 | "start": "webpack --watch", 8 | "build": "webpack --mode=production", 9 | "lint": "eslint src/", 10 | "preflight": "npm run lint && npm-check -p --skip-unused && npm audit", 11 | "preflight:fix": "npm run lint --fix && npm-check -p --skip-unused -y && npm audit --fix" 12 | }, 13 | "author": "", 14 | "license": "MIT", 15 | "devDependencies": { 16 | "@material/button": "^3.0.0", 17 | "@material/dialog": "^3.0.0", 18 | "@material/fab": "^3.0.0", 19 | "@material/icon-button": "^3.0.0", 20 | "@material/layout-grid": "^3.0.0", 21 | "@material/linear-progress": "^3.0.0", 22 | "@material/list": "^3.0.0", 23 | "@material/menu": "^3.0.0", 24 | "@material/menu-surface": "^3.0.0", 25 | "@material/ripple": "^3.0.0", 26 | "@material/select": "^3.0.0", 27 | "@material/textfield": "^3.0.0", 28 | "@material/theme": "^3.0.0", 29 | "@material/typography": "^3.0.0", 30 | "css-loader": "^4.2.2", 31 | "dayjs": "^1.8.34", 32 | "eslint": "^7.7.0", 33 | "extract-loader": "^5.1.0", 34 | "html-webpack-plugin": "^4.3.0", 35 | "mustache": "^4.0.1", 36 | "node-sass": "^4.14.1", 37 | "npm-check": "^5.9.2", 38 | "sass-loader": "^10.0.1", 39 | "style-loader": "^1.2.1", 40 | "webpack": "^4.44.1", 41 | "webpack-bundle-analyzer": "^3.8.0", 42 | "webpack-cli": "^3.3.12" 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /frontend/src/Autocomplete.js: -------------------------------------------------------------------------------- 1 | import {MDCTextField} from '@material/textfield'; 2 | import {MDCMenu} from '@material/menu' 3 | 4 | class DropdownList extends EventTarget { 5 | constructor(name, view) { 6 | super() 7 | this.menu = document.getElementById(`${name}-menu-${view}`) 8 | this.mdcMenu = new MDCMenu(this.menu) 9 | this.currentSelectedElement = -1 10 | 11 | this.mdcMenu.listen('MDCMenu:selected', ({detail: {item}}) => { 12 | this.dispatchEvent(new CustomEvent('selected', {detail: {value: item.innerText}})) 13 | }) 14 | } 15 | 16 | generateItem(text) { 17 | const li = document.createElement('li', {'role': 'menuitem'}) 18 | const span = document.createElement('span') 19 | span.classList.add('mdc-list-item__text') 20 | span.classList.add('mdc-typography--body1') 21 | span.innerText = text 22 | li.classList.add('mdc-list-item') 23 | li.appendChild(span) 24 | return li 25 | } 26 | 27 | generateItems(texts) { 28 | return texts.map(this.generateItem) 29 | } 30 | 31 | appendChild(elem) { 32 | this.menu.childNodes[1].appendChild(elem) 33 | } 34 | 35 | appendChildren(elems) { 36 | for (const elem of elems) { 37 | this.appendChild(elem) 38 | } 39 | } 40 | 41 | setItems(items) { 42 | const elems = this.generateItems(items) 43 | this.appendChildren(elems) 44 | } 45 | 46 | clear() { 47 | while (this.menu.childNodes[1].hasChildNodes()) { 48 | this.menu.childNodes[1].removeChild(this.menu.childNodes[1].firstChild) 49 | } 50 | } 51 | 52 | replaceItems(items) { 53 | this.clear() 54 | this.setItems(items) 55 | } 56 | 57 | replaceItemsAndShow(items) { 58 | if (items.length) { 59 | if (!this.isOpen()) { 60 | this.open() 61 | } 62 | this.replaceItems(items) 63 | } else if (items.length === 0) { 64 | this.currentSelectedElement = -1 65 | this.close() 66 | } 67 | } 68 | 69 | open() { 70 | this.mdcMenu.open = true 71 | this.currentSelectedElement = -1 72 | } 73 | 74 | close() { 75 | this.mdcMenu.open = false 76 | } 77 | 78 | isOpen() { 79 | return this.mdcMenu.open 80 | } 81 | 82 | scroll(up) { 83 | const height = this.menu.querySelector('li').getBoundingClientRect().height 84 | const lastElement = this.mdcMenu.list_.listElements.length - 1 85 | const maxScrollTop = height * lastElement 86 | const menuHeight = this.menu.getBoundingClientRect().height 87 | const scrollTop = this.menu.scrollTop 88 | const itemTop = this.mdcMenu.list_.listElements[this.currentSelectedElement].getBoundingClientRect().top 89 | if (this.currentSelectedElement == 0) { 90 | this.menu.scroll({top: 0}) 91 | } else if (this.currentSelectedElement == lastElement) { 92 | this.menu.scroll({top: maxScrollTop}) 93 | } else if (itemTop >= (menuHeight + height * 2) && !up) { 94 | this.menu.scroll({top: scrollTop + height}) 95 | } else if (itemTop <= (height * 2) && up) { 96 | this.menu.scroll({top: scrollTop - height}) 97 | } 98 | 99 | } 100 | 101 | nextElement() { 102 | this.currentSelectedElement = (this.currentSelectedElement + 1) % this.mdcMenu.list_.listElements.length 103 | this.scroll(false) 104 | this.mdcMenu.list_.foundation_.setSelectedIndex(this.currentSelectedElement); 105 | } 106 | 107 | previousElement() { 108 | this.currentSelectedElement-- 109 | if (this.currentSelectedElement <= -1) { 110 | this.currentSelectedElement = this.mdcMenu.list_.listElements.length - 1 111 | } 112 | this.scroll(true) 113 | this.mdcMenu.list_.foundation_.setSelectedIndex(this.currentSelectedElement); 114 | } 115 | 116 | confirmSelection() { 117 | if (this.currentSelectedElement === -1) { 118 | return 119 | } 120 | this.mdcMenu.foundation_.adapter_.notifySelected({index: this.currentSelectedElement}) 121 | } 122 | 123 | selectFirstElement() { 124 | this.mdcMenu.list_.foundation_.setSelectedIndex(0); 125 | this.currentSelectedElement = 0; 126 | } 127 | } 128 | 129 | export class Autocomplete extends EventTarget { 130 | constructor(name, view, items = Promise.resolve([])) { 131 | super() 132 | this.listView = new DropdownList(name, view) 133 | this.textField = document.getElementById(`${name}-text-field-${view}`) 134 | this.mdcTextField = new MDCTextField(this.textField); 135 | this.inputElement = document.getElementById(`input-${name}-${view}`) 136 | this.previousContent = '' 137 | this._registerEvents() 138 | items.then(items => { 139 | this.items = items 140 | this.listView.replaceItems(this.items) 141 | }) 142 | } 143 | 144 | _registerEvents() { 145 | this.textField.addEventListener('keydown', (evt) => { 146 | switch (evt.code) { 147 | case 'ArrowDown': 148 | this.lastKeyEvent = 'ArrowDown' 149 | this.listView.nextElement() 150 | this.inputElement.setAttribute('placeholder', this.items[this.listView.currentSelectedElement]) 151 | break 152 | case 'ArrowUp': 153 | this.lastKeyEvent = 'ArrowUp' 154 | this.listView.previousElement() 155 | this.inputElement.setAttribute('placeholder', this.items[this.listView.currentSelectedElement]) 156 | break 157 | case 'NumpadEnter': 158 | case 'Enter': 159 | this.lastKeyEvent = 'Enter' 160 | this.listView.confirmSelection() 161 | this.listView.close() 162 | this.inputElement.removeAttribute('placeholder') 163 | break 164 | case 'Tab': 165 | this.lastKeyEvent = 'Tab' 166 | if (this.value) { 167 | this.dispatchEvent(new CustomEvent('selected', {detail: {value: this.value}})) 168 | } else { 169 | this.dispatchEvent(new Event('reset')) 170 | } 171 | this.listView.close() 172 | break 173 | default: 174 | break 175 | } 176 | }, true) 177 | 178 | this.textField.oninput = e => { 179 | const text = e.target.value 180 | this._filterAndReplace(text) 181 | this.listView.selectFirstElement() 182 | this.inputElement.setAttribute('placeholder', this.items[this.listView.currentSelectedElement]) 183 | } 184 | 185 | this.textField.addEventListener('focus', (e) => { 186 | this.previousContent = this.inputElement.value 187 | this.inputElement.value = '' 188 | const text = e.target.value 189 | this._filterAndReplace(text) 190 | this.listView.selectFirstElement() 191 | this.inputElement.setAttribute('placeholder', this.items[this.listView.currentSelectedElement]) 192 | }, true) 193 | 194 | this.textField.addEventListener('blur', () => { 195 | setTimeout(() => { 196 | if (!this.inputElement.value && this.previousContent && this.lastKeyEvent !== 'Tab') { 197 | this.inputElement.value = this.previousContent 198 | this.mdcTextField.label_.float(true) 199 | } 200 | this.listView.close() 201 | }, 100) 202 | }, true) 203 | 204 | this.listView.addEventListener('selected', (e) => { 205 | const value = e.detail.value 206 | this.dispatchEvent(new CustomEvent('selected', {detail: {value}})) 207 | this.mdcTextField.label_.float(true) 208 | this.inputElement.value = value 209 | }) 210 | } 211 | 212 | get value() { 213 | return this.inputElement.value 214 | } 215 | 216 | _filter(text) { 217 | text = text.toLowerCase().trim() 218 | try { 219 | return this.items.filter(i => i.toLowerCase().trim().search(text) !== -1) 220 | } catch (e) { 221 | return [] 222 | } 223 | } 224 | 225 | _filterAndReplace(text) { 226 | this.listView.replaceItemsAndShow(this._filter(text)) 227 | } 228 | 229 | enable() { 230 | this.mdcTextField.disabled = false 231 | } 232 | 233 | disable() { 234 | this.mdcTextField.disabled = true 235 | } 236 | 237 | async update(items) { 238 | this.items = await items 239 | this.listView.replaceItems(this.items) 240 | } 241 | 242 | fill(text) { 243 | this.mdcTextField.label_.float(true) 244 | this.inputElement.value = text 245 | } 246 | 247 | fillWithEvent(text) { 248 | this.mdcTextField.value = text 249 | this.listView.dispatchEvent(new CustomEvent({detail: {value: text}})) 250 | } 251 | 252 | fillSelected() { 253 | this.fill(this.value) 254 | } 255 | 256 | fillIfEmpty(text) { 257 | if (!this.value) { 258 | this.fill(text) 259 | } 260 | } 261 | 262 | clear() { 263 | this.items = [] 264 | this.inputElement.value = '' 265 | this.listView.clear() 266 | this.inputElement.removeAttribute('placeholder') 267 | } 268 | 269 | clearInput() { 270 | this.inputElement.value = '' 271 | } 272 | 273 | reset(emitEvent = true) { 274 | this.clear() 275 | this.disable() 276 | if (emitEvent) { 277 | this.dispatchEvent(new Event('reset')) 278 | } 279 | } 280 | } 281 | 282 | -------------------------------------------------------------------------------- /frontend/src/LighthouseReport.js: -------------------------------------------------------------------------------- 1 | import * as dayjs from 'dayjs' 2 | import customParseFormat from 'dayjs/plugin/customParseFormat' 3 | 4 | dayjs.extend(customParseFormat) 5 | 6 | // eslint-disable-next-line no-unused-vars 7 | export class LighthouseReports { 8 | constructor(baseUrl = '') { 9 | this._baseUrl = baseUrl 10 | this._dateFormat = 'DD.MM.YYYY HH:mm:ss.SSS' 11 | } 12 | 13 | async getReport(url, preset, date) { 14 | date = dayjs(date, this._dateFormat).toISOString() 15 | const response = await fetch(`${this._baseUrl}/report/url/${encodeURIComponent(url)}/${encodeURIComponent(preset)}/${encodeURIComponent(date)}`) 16 | const content = await response.text() 17 | if (content) { 18 | return JSON.parse(content) 19 | } 20 | } 21 | 22 | async getReportById(id) { 23 | const response = await fetch(`${this._baseUrl}/report/${id}`) 24 | const report = await response.json(); 25 | report.date = dayjs(report.date).format(this._dateFormat) 26 | return report 27 | } 28 | 29 | async getUrls() { 30 | const response = await fetch(`${this._baseUrl}/report/url/`) 31 | return response.json() 32 | } 33 | 34 | async getPresetsFromUrl(url) { 35 | const response = await fetch(`${this._baseUrl}/report/url/${encodeURIComponent(url)}/`) 36 | return response.json() 37 | } 38 | 39 | async getDatesFromUrl(url, preset) { 40 | const data = await fetch(`${this._baseUrl}/report/url/${encodeURIComponent(url)}/${encodeURIComponent(preset)}/`) 41 | .then(response => response.json()) 42 | return Object.keys(data) 43 | .map(date => dayjs(date)) 44 | .sort((a, b) => a.isAfter(b) ? -1 : 1) 45 | .map(date => date.format(this._dateFormat)) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /frontend/src/PresetSelect.js: -------------------------------------------------------------------------------- 1 | import { MDCSelect } from '@material/select' 2 | 3 | export class PresetSelect extends EventTarget { 4 | constructor(view) { 5 | super() 6 | this.selectContainer = document.getElementById(`input-preset-${view}`) 7 | this.select = this.selectContainer.getElementsByTagName('select')[0] 8 | this.mdcSelect = new MDCSelect(this.selectContainer); 9 | this.mdcSelect.label_.float(true) 10 | this.mdcSelect.listen('MDCSelect:change', () => { 11 | this.dispatchEvent(new CustomEvent('changed', { detail: { value: this.mdcSelect.value } })) 12 | }); 13 | } 14 | 15 | enable() { 16 | this.mdcSelect.disabled = false 17 | } 18 | 19 | disable() { 20 | this.mdcSelect.disabled = true 21 | } 22 | 23 | get value() { 24 | return this.mdcSelect.value 25 | } 26 | 27 | generateItem(text) { 28 | const option = document.createElement('option') 29 | option.innerText = text 30 | option.value = text 31 | return option 32 | } 33 | 34 | generateItems(texts) { 35 | return texts.map(this.generateItem) 36 | } 37 | 38 | appendChild(elem) { 39 | this.select.appendChild(elem) 40 | } 41 | 42 | appendChildren(elems) { 43 | for (const elem of elems) { 44 | this.appendChild(elem) 45 | } 46 | } 47 | 48 | setItems(items) { 49 | const elems = this.generateItems(items) 50 | this.appendChildren(elems) 51 | } 52 | 53 | clear() { 54 | const elems = Array.from(this.select.getElementsByTagName('option')) 55 | while (elems.length) { 56 | const n = elems.pop() 57 | this.select.removeChild(n) 58 | } 59 | } 60 | 61 | replaceItems(items) { 62 | this.clear() 63 | this.setItems(items) 64 | } 65 | 66 | reset(emitEvent = true) { 67 | const el = document.createElement('option') 68 | el.setAttribute('disabled', 'true') 69 | el.setAttribute('selected', 'true') 70 | el.innerText = 'Please choose a URL first' 71 | this.clear() 72 | this.disable() 73 | this.mdcSelect.label_.float(true) 74 | this.appendChild(el) 75 | this.disable() 76 | if (emitEvent) { 77 | this.dispatchEvent(new Event('reset')) 78 | } 79 | } 80 | 81 | selectItem(value) { 82 | this.mdcSelect.value = value 83 | } 84 | 85 | selectItemWithoutTriggerEvent(value) { 86 | this.select.value = value 87 | } 88 | 89 | 90 | } -------------------------------------------------------------------------------- /frontend/src/Selection.js: -------------------------------------------------------------------------------- 1 | import { Autocomplete } from './Autocomplete' 2 | import { PresetSelect } from './PresetSelect' 3 | import { UrlState } from './UrlState' 4 | 5 | 6 | export class Selection extends EventTarget { 7 | constructor(view, lighthouseReports) { 8 | super() 9 | this.lighthouseReports = lighthouseReports 10 | this.view = view 11 | this.url = new Autocomplete('url', view, lighthouseReports.getUrls()) 12 | this.preset = new PresetSelect(view) 13 | this.date = new Autocomplete('date', view) 14 | this.date.disable() 15 | this.registerEvents() 16 | this.setFromURL() 17 | } 18 | 19 | async setFromURL() { 20 | const reportId = UrlState.get(this.view) 21 | if (!reportId) { 22 | return 23 | } 24 | const report = await this.lighthouseReports.getReportById(reportId) 25 | if (!report) { 26 | return 27 | } 28 | this.url.fill(report.url) 29 | this.preset.replaceItems(await this.lighthouseReports.getPresetsFromUrl(report.url)) 30 | this.preset.selectItemWithoutTriggerEvent(report.preset) 31 | this.preset.enable() 32 | this.date.update(await this.lighthouseReports.getDatesFromUrl(report.url, report.preset)) 33 | this.date.fill(report.date) 34 | this.date.enable() 35 | this.emitComplete() 36 | } 37 | 38 | registerEvents() { 39 | this.url.addEventListener('selected', async ({ detail: { value } }) => { 40 | const presets = await this.lighthouseReports.getPresetsFromUrl(value) 41 | if (!presets.length) { 42 | return this.emitResetUrl() 43 | } 44 | 45 | this.preset.replaceItems(presets) 46 | setTimeout(() => { 47 | this.preset.selectItem(presets[0]) 48 | this.preset.enable() 49 | }, 1) 50 | }) 51 | 52 | this.preset.addEventListener('changed', async ({ detail: { value } }) => { 53 | const dates = await this.lighthouseReports.getDatesFromUrl(this.url.value, value) 54 | this.date.update(dates) 55 | this.date.enable() 56 | this.date.fillIfEmpty(dates[0]) 57 | setTimeout(() => { 58 | this.emitComplete() 59 | }, 1) 60 | }) 61 | 62 | this.date.addEventListener('selected', () => { 63 | setTimeout(() => { 64 | this.emitComplete() 65 | }, 1) 66 | }) 67 | 68 | this.url.addEventListener('reset', this.emitResetUrl.bind(this)) 69 | } 70 | 71 | emitResetUrl() { 72 | this.dispatchEvent(new Event('reset:url')) 73 | this.preset.reset() 74 | this.date.reset() 75 | UrlState.clear(this.view) 76 | } 77 | 78 | async check(report) { 79 | if (report) { 80 | return false 81 | } 82 | 83 | const dates = await this.lighthouseReports.getDatesFromUrl(this.url.value, this.preset.value) 84 | if (dates.length === 0) { 85 | this.emitResetUrl() 86 | this.url.dispatchEvent(new CustomEvent('selected', { detail: { value: this.url.value } })) 87 | return true 88 | } 89 | 90 | this.date.fill(dates[0]) 91 | this.emitComplete() 92 | return true 93 | } 94 | 95 | async emitComplete() { 96 | const report = await this.lighthouseReports.getReport(this.url.value, this.preset.value, this.date.value) 97 | if (await this.check(report)) { 98 | return 99 | } 100 | 101 | UrlState.set(this.view, report.id) 102 | this.dispatchEvent( 103 | new CustomEvent('complete', 104 | { 105 | detail: { 106 | report 107 | } 108 | })) 109 | } 110 | 111 | fillDefault() { 112 | this.emitComplete() 113 | } 114 | 115 | emitReset() { 116 | this.dispatchEvent(new Event('reset')) 117 | } 118 | 119 | reset(emitEvent = true) { 120 | this.url.clearInput() 121 | this.preset.reset(false) 122 | this.date.reset(false) 123 | UrlState.clear(this.view) 124 | if (emitEvent) { 125 | this.emitReset() 126 | } 127 | } 128 | } -------------------------------------------------------------------------------- /frontend/src/UrlState.js: -------------------------------------------------------------------------------- 1 | export class UrlState { 2 | static set(view, value) { 3 | const params = new URLSearchParams(window.location.search) 4 | params.set(view, value) 5 | const newurl = window.location.protocol + '//' + window.location.host + window.location.pathname + '?' + params.toString() 6 | window.history.pushState({path:newurl},'',newurl); 7 | } 8 | 9 | static get(view) { 10 | const params = new URLSearchParams(window.location.search) 11 | params.set('some', 'world') 12 | return params.get(view) 13 | } 14 | 15 | static clear(view) { 16 | const params = new URLSearchParams(window.location.search) 17 | params.delete(view) 18 | const newurl = window.location.protocol + '//' + window.location.host + window.location.pathname + '?' + params.toString() 19 | window.history.pushState({path:newurl},'',newurl); 20 | } 21 | } -------------------------------------------------------------------------------- /frontend/src/View.js: -------------------------------------------------------------------------------- 1 | import * as Mustache from 'mustache' 2 | import {MDCMenu} from '@material/menu' 3 | 4 | import {Selection} from './Selection'; 5 | 6 | const attachDownloadMenu = (view) => { 7 | const menu = document.querySelector(`.download_menu.${view}`) 8 | const mdcMenu = new MDCMenu(menu) 9 | const fab = document.querySelector(`.mdc-fab.${view}`) 10 | mdcMenu.setAnchorElement(fab) 11 | fab.onclick = () => { 12 | if (mdcMenu.open) { 13 | mdcMenu.open = false 14 | } else { 15 | mdcMenu.open = true 16 | } 17 | } 18 | 19 | menu.addEventListener('blur', () => { 20 | setTimeout(() => { 21 | mdcMenu.open = false 22 | }, 16.7) 23 | }, true) 24 | 25 | mdcMenu.listen('MDCMenu:selected', (ev) => { 26 | window.location.href = ev.detail.item.dataset.href 27 | }) 28 | } 29 | 30 | class View { 31 | constructor(viewSide, lighthouseReports) { 32 | this.view = viewSide 33 | this.renderTemplates() 34 | this.selection = new Selection(viewSide, lighthouseReports) 35 | 36 | this.selection.addEventListener('complete', async ({detail: {report}}) => { 37 | this.renderReport(await report) 38 | attachDownloadMenu(this.view) 39 | }) 40 | 41 | this.selection.addEventListener('reset:url', this.clearReport.bind(this)) 42 | } 43 | 44 | renderTemplates() { 45 | const template = document.getElementById('tpl_view').innerHTML 46 | const container = document.getElementById(`result-${this.view}`) 47 | const rendered = Mustache.render(template, { view: this.view }) 48 | container.innerHTML = rendered 49 | } 50 | 51 | renderReport(report) { 52 | const template = document.getElementById('tpl_result').innerHTML 53 | const container = document.getElementById(`render-${this.view}`) 54 | const rendered = Mustache.render(template, { 55 | reportId: report.id, 56 | name: report.name, 57 | view: this.view 58 | }) 59 | container.innerHTML = rendered 60 | } 61 | 62 | clearReport() { 63 | const container = document.getElementById(`render-${this.view}`) 64 | container.innerHTML = '' 65 | } 66 | 67 | reset() { 68 | this.selection.reset() 69 | this.clearReport() 70 | } 71 | } 72 | 73 | export class LeftView extends View { 74 | constructor(lighthouseReports) { 75 | super('left', lighthouseReports) 76 | } 77 | } 78 | 79 | export class RightView extends View { 80 | constructor(lighthouseReports) { 81 | super('right', lighthouseReports) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /frontend/src/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | Lighthouse Monitor Comparator 10 | 11 | 12 | 13 |
14 |
15 |
16 |
17 | 18 |
19 |
20 | 21 |
22 |
23 |
24 | Lighthouse Monitor Comparator 25 | 28 | 31 |
32 | 33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 | 46 |
47 | 115 | 116 |
117 | 118 | 146 | 147 | 153 | 154 | 201 | 202 | 203 | 204 | -------------------------------------------------------------------------------- /frontend/src/index.js: -------------------------------------------------------------------------------- 1 | import './styles.scss' 2 | import {MDCRipple} from '@material/ripple' 3 | import {MDCDialog} from '@material/dialog' 4 | import {MDCLinearProgress} from '@material/linear-progress' 5 | 6 | import {LighthouseReports} from './LighthouseReport' 7 | import {LeftView, RightView} from './View' 8 | 9 | const modal = new MDCDialog(document.querySelector('.mdc-dialog')) 10 | const helpBtn = document.getElementById('help-btn') 11 | MDCRipple.attachTo(helpBtn, {isUnbounded: true}) 12 | 13 | helpBtn.onclick = () => { 14 | modal.open() 15 | } 16 | 17 | const resetBtn = document.getElementById('reset-btn') 18 | 19 | document.addEventListener('DOMContentLoaded', async ( ) => { 20 | try { 21 | const progress = document.querySelector('.mdc-linear-progress') 22 | MDCLinearProgress.attachTo(progress) 23 | const reports = new LighthouseReports() 24 | const leftView = new LeftView(reports) 25 | const rightView = new RightView(reports) 26 | 27 | resetBtn.onclick = () => { 28 | leftView.reset() 29 | rightView.reset() 30 | } 31 | progress.setAttribute('hidden', true) 32 | } catch (err) { 33 | console.error(err) 34 | } 35 | }) 36 | 37 | 38 | -------------------------------------------------------------------------------- /frontend/src/material-icons.css: -------------------------------------------------------------------------------- 1 | @font-face { 2 | font-family: 'Material Icons'; 3 | font-style: normal; 4 | font-weight: 400; 5 | src: url(https://fonts.gstatic.com/s/materialicons/v43/flUhRq6tzZclQEJ-Vdg-IuiaDsNZ.ttf) format('truetype'); 6 | } 7 | 8 | .material-icons { 9 | font-family: 'Material Icons'; 10 | font-weight: normal; 11 | font-style: normal; 12 | font-size: 24px; 13 | line-height: 1; 14 | letter-spacing: normal; 15 | text-transform: none; 16 | display: inline-block; 17 | white-space: nowrap; 18 | word-wrap: normal; 19 | direction: ltr; 20 | } 21 | -------------------------------------------------------------------------------- /frontend/src/styles.scss: -------------------------------------------------------------------------------- 1 | $mdc-theme-primary: #FD8A02; 2 | $mdc-theme-accent: $mdc-theme-primary; 3 | 4 | @import "material-icons.css"; 5 | @import "~@material/typography/mdc-typography"; 6 | @import "~@material/theme/mdc-theme"; 7 | @import "~@material/layout-grid/mdc-layout-grid"; 8 | @import "~@material/icon-button/mdc-icon-button"; 9 | @import "~@material/ripple/mdc-ripple"; 10 | @import "~@material/button/mdc-button"; 11 | @import "~@material/dialog/mdc-dialog"; 12 | @import "~@material/list/mdc-list"; 13 | @import "~@material/select/mdc-select"; 14 | @import "~@material/menu-surface/mdc-menu-surface"; 15 | @import "~@material/menu/mdc-menu"; 16 | @import "~@material/textfield/mdc-text-field"; 17 | @import "~@material/menu-surface/mdc-menu-surface"; 18 | @import "~@material/fab/mdc-fab"; 19 | @import "~@material/linear-progress/mdc-linear-progress"; 20 | 21 | .text-center { 22 | text-align: center; 23 | } 24 | 25 | .result { 26 | width: 100%; 27 | 28 | iframe { 29 | width: 100%; 30 | height: 80vh; 31 | } 32 | } 33 | 34 | .mdc-select { 35 | width: 100%; 36 | 37 | select { 38 | -webkit-appearance: none; 39 | } 40 | } 41 | 42 | .mdc-text-field { 43 | width: 100%; 44 | } 45 | 46 | #container { 47 | position: relative; 48 | 49 | &::after { 50 | content: ''; 51 | position: absolute; 52 | top: 0; 53 | left: 50%; 54 | bottom: 0; 55 | border-right: 1px solid lightgray; 56 | height: 90vh; 57 | } 58 | } 59 | 60 | 61 | @media (max-width: 839px) { 62 | #container::after { 63 | content: none; 64 | } 65 | } 66 | 67 | .download_menu { 68 | a { 69 | text-decoration: none; 70 | color: black; 71 | cursor: auto; 72 | } 73 | 74 | } 75 | 76 | .mdc-fab { 77 | position: absolute; 78 | bottom: 1rem; 79 | left: 1rem; 80 | } 81 | 82 | .smaller-margin { 83 | margin-top: 10px; 84 | margin-bottom: 0; 85 | } 86 | 87 | @media (min-width: 700px) { 88 | .non-mobile-size { 89 | max-width: 33%; 90 | } 91 | } 92 | 93 | body { 94 | margin: 0; 95 | } 96 | 97 | iframe { 98 | border: none; 99 | } 100 | -------------------------------------------------------------------------------- /frontend/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const HtmlWebpackPlugin = require('html-webpack-plugin') 3 | const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin 4 | const tmpdir = require('os').tmpdir 5 | const join = require('path').join 6 | 7 | 8 | module.exports = { 9 | mode: 'development', 10 | entry: './src/index.js', 11 | output: { 12 | filename: 'main.js', 13 | path: path.resolve(__dirname, '..', 'public') 14 | }, 15 | performance : { 16 | hints : false 17 | }, 18 | plugins: [ 19 | new HtmlWebpackPlugin({ 20 | filename: path.resolve(__dirname, '..', 'public', 'index.html'), 21 | template: 'src/index.html' 22 | }), 23 | new BundleAnalyzerPlugin({ 24 | analyzerMode: 'static', 25 | reportFilename: join(tmpdir(), 'report.html'), 26 | openAnalyzer: false 27 | }) 28 | ], 29 | module: { 30 | rules: [ 31 | { 32 | test: /\.scss$/, 33 | use: ['style-loader', 'css-loader', { 34 | loader: "sass-loader", 35 | options: { 36 | sassOptions: { 37 | includePaths: ["node_modules/", "src/"] 38 | } 39 | } 40 | }] 41 | } 42 | ] 43 | } 44 | }; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lightmon", 3 | "version": "2.5.0", 4 | "description": "A lighthouse monitor", 5 | "main": "./bin/lightmon", 6 | "scripts": { 7 | "build": "cd frontend && npm ci && npm run build", 8 | "cleanup": "cross-env DEBUG=${DEBUG:-LIGHTMON*} ./bin/cleanup", 9 | "lint": "eslint src/", 10 | "lint:fix": "eslint --fix src/", 11 | "start": "node ./bin/evaluate", 12 | "evaluation": "cross-env DEBUG=${DEBUG:-LIGHTMON*} npm start", 13 | "frontend:preflight": "cd frontend && npm run preflight", 14 | "frontend:preflight:fix": "cd frontend && npm run preflight:fix", 15 | "start:debug": "node --nolazy --inspect-brk=9229 ./bin/evaluate", 16 | "server": "cross-env DEBUG=${DEBUG:-LIGHTMON*} ./bin/server", 17 | "server:watch": "cross-env DEBUG=${DEBUG:-LIGHTMON*} nodemon -w src/ -w config/ -w bin/ ./bin/server", 18 | "update": "npm-check --skip-unused -p --ignore commander --ignore proper-lockfile", 19 | "update:fix": "npm-check --skip-unused -p --ignore commander --ignore proper-lockfile -y", 20 | "test": "mocha 'src/**/*.spec.js'", 21 | "test:integration": "mocha 'src/**/*.integration-spec.js'", 22 | "test:all": "mocha 'src/**/*[-.]spec.js'", 23 | "preflight": "concurrently --names 'audit,lint,update,frontend:preflight,test' \"npm audit --production\" npm:lint npm:update npm:frontend:preflight npm:test:all", 24 | "preflight:fix": "concurrently --names 'audit,lint,update,frontend:preflight:fix' \"npm audit --production --fix\" npm:lint:fix npm:update:fix npm:frontend:preflight:fix && npm run preflight" 25 | }, 26 | "engines": { 27 | "node": ">=11" 28 | }, 29 | "engineStrict": true, 30 | "repository": { 31 | "type": "git", 32 | "url": "git@gitlab.kumbier.it:verivox/lighthouse.git" 33 | }, 34 | "keywords": [ 35 | "lighthouse", 36 | "monitor", 37 | "prometheus", 38 | "metrics", 39 | "performance" 40 | ], 41 | "author": "Lars Kumbier (https://kumbier.it)", 42 | "contributors": [ 43 | "Kim Almasan (https://kumbier.it)", 44 | "Yurii Ivanov " 45 | ], 46 | "license": "MIT", 47 | "dependencies": { 48 | "better-sqlite3": "^7.1.0", 49 | "chokidar": "^3.4.2", 50 | "commander": "^6.1.0", 51 | "cross-env": "^7.0.2", 52 | "debug": "^4.1.1", 53 | "express": "^4.17.1", 54 | "fs-extra": "^9.0.1", 55 | "lighthouse": "^8.5.0", 56 | "mime": "^2.4.6", 57 | "moment": "^2.27.0", 58 | "prom-client": "^12.0.0", 59 | "proper-lockfile": "^4.1.1", 60 | "puppeteer": "^10.4.0", 61 | "request": "^2.88.2", 62 | "request-promise-native": "^1.0.9" 63 | }, 64 | "devDependencies": { 65 | "@types/fs-extra": "^9.0.1", 66 | "chai": "^4.2.0", 67 | "chai-files": "^1.4.0", 68 | "chai-fs": "^2.0.0", 69 | "concurrently": "^5.3.0", 70 | "eslint": "^7.8.1", 71 | "eslint-plugin-mocha": "^8.0.0", 72 | "eslint-plugin-node": "^11.1.0", 73 | "mocha": "^8.1.3", 74 | "nodemon": "^2.0.4", 75 | "npm-check": "^5.9.2", 76 | "rimraf": "^3.0.2", 77 | "sinon": "^9.0.3", 78 | "sinon-chai": "^3.5.0" 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /public/index.html: -------------------------------------------------------------------------------- 1 | Lighthouse Monitor Comparator
Lighthouse Monitor Comparator
-------------------------------------------------------------------------------- /public/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/public/screenshot.png -------------------------------------------------------------------------------- /src/bootstrap.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const cluster = require('cluster') 6 | const { Webserver, SingleRun, PeriodicRun, ReportCruncher } = require('../src/submodules') 7 | 8 | class Program { 9 | /** 10 | * Parses the argv arguments and switches between server and single run mode. 11 | * @param {*} args 12 | */ 13 | constructor(args) { 14 | this._args = args 15 | } 16 | 17 | run() { 18 | return this._args.server 19 | ? new ServerMode(cluster).run() 20 | : new SingleRunMode(cluster).run() 21 | } 22 | } 23 | 24 | class ServerMode { 25 | 26 | /** 27 | * Create instance of ServerMode to continously generate and monitor reports. 28 | * @param _cluster The cluster to fork on 29 | */ 30 | constructor(_cluster = cluster) { 31 | this._cluster = _cluster 32 | } 33 | 34 | /** 35 | * Fork periodic report generation and a monitoring webserver. 36 | */ 37 | run() { 38 | Webserver.fork(this._cluster) 39 | PeriodicRun.fork(this._cluster) 40 | ReportCruncher.fork(this._cluster) 41 | } 42 | } 43 | 44 | class SingleRunMode { 45 | 46 | /** 47 | * Create instance of SingleRunMode to only generate reports once and then exit. 48 | * @param _cluster The cluster to fork on 49 | */ 50 | constructor(_cluster = cluster) { 51 | this._cluster = _cluster 52 | } 53 | 54 | /** 55 | * Fork one instance of the SingleRun submodule 56 | */ 57 | run() { 58 | SingleRun.fork(this._cluster) 59 | } 60 | } 61 | 62 | module.exports = { 63 | Program, 64 | ServerMode, 65 | } 66 | -------------------------------------------------------------------------------- /src/evaluate.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const { Lighthouse } = require('./lighthouse') 6 | const debug = require('debug') 7 | 8 | async function run(options, receivers) { 9 | const lighthouse = new Lighthouse(options) 10 | await lighthouse.reportTo(receivers) 11 | } 12 | 13 | async function evaluate(config) { 14 | const runStartedAt = new Date().toISOString() 15 | 16 | for (const target of config.targets) { 17 | debug('LIGHTMON:INFO')(`Working on ${target.name}: ${target.url}`) 18 | for (const preset of Object.values(config.presets)) { 19 | debug('LIGHTMON:INFO')(`+ Running preset ${preset.preset}`) 20 | const options = Object.assign({}, preset, target) 21 | options.runStartedAt = runStartedAt 22 | await run(options, config.receivers) 23 | } 24 | } 25 | 26 | for (const receiver of config.receivers) { 27 | if (typeof receiver.afterEvaluation === 'function') { 28 | await receiver.afterEvaluation() 29 | } 30 | } 31 | } 32 | 33 | module.exports = { 34 | default: evaluate, 35 | evaluate, 36 | } 37 | 38 | -------------------------------------------------------------------------------- /src/lighthouse.js: -------------------------------------------------------------------------------- 1 | const debug = require('debug') 2 | 3 | /** 4 | * Part of Lightmon: https://github.com/verivox/lightmon 5 | * Licensed under MIT from the Verivox GmbH 6 | */ 7 | const lighthouseLib = require('lighthouse') 8 | const puppeteer = require('puppeteer'); 9 | 10 | const DefaultOptions = { 11 | browserOptions: { 12 | args: ['--remote-debugging-port=0'] 13 | } 14 | } 15 | 16 | class Lighthouse { 17 | constructor(options = DefaultOptions, _lighthouse = lighthouseLib) { 18 | this._options = options 19 | this._lighthouse = _lighthouse 20 | } 21 | 22 | async startChrome(options) { 23 | return await puppeteer.launch(options); 24 | } 25 | 26 | _checkForDeprecatedOptions(options) { 27 | if (typeof options.chromeFlags !== 'undefined') { 28 | debug('LIGHTMON:WARN')('You are using a deprecated config option (chromeFlags) - it is ignored from ' + 29 | 'version 2.0.0. Please check the CHANGELOG.md on the migration process.') 30 | } 31 | } 32 | 33 | // this function evaluates a page and retries for x times 34 | async _evaluate(options, maxRetries=3) { 35 | this._checkForDeprecatedOptions(options) 36 | 37 | let tries = 0 38 | let chrome 39 | 40 | do { 41 | try { 42 | tries++ 43 | chrome = await this.startChrome(options.browserOptions) 44 | 45 | if (!chrome) { 46 | throw new Error('Could not start Chrome') 47 | } 48 | 49 | // eslint-disable-next-line require-atomic-updates 50 | options.port = this.getDebugPort(chrome.wsEndpoint()); 51 | if (options.prehook) { 52 | await options.prehook.setup(chrome); 53 | } 54 | 55 | return await this._lighthouse(options.url, options) 56 | } catch (e) { 57 | debug('LIGHTMON:WARN')(`++ Error evaluating, retry #${tries}/${maxRetries}: ${e}`) 58 | } finally { 59 | await chrome.close() 60 | } 61 | } while (tries < maxRetries) 62 | } 63 | 64 | getDebugPort(url) { 65 | const s1 = url.substr(url.lastIndexOf(':') + 1); 66 | return s1.substr(0, s1.indexOf('/')); 67 | } 68 | 69 | async result() { 70 | if (!this._result) { 71 | this._result = await this._evaluate(this._options) 72 | } 73 | return this._result 74 | } 75 | 76 | async reportTo(receivers) { 77 | const result = await this.result() 78 | for (const receiver of receivers) { 79 | await receiver.receive(result, this._options) 80 | } 81 | } 82 | } 83 | 84 | 85 | class PoppableMap extends Map { 86 | pop(key) { 87 | const value = this.get(key) 88 | this.delete(key) 89 | return value 90 | } 91 | } 92 | 93 | class LighthouseReport { 94 | constructor({ lhr }, config) { 95 | this._metrics = new PoppableMap() 96 | this._meta = { 97 | 'url': config.url, 98 | 'profile': config['preset'], 99 | } 100 | 101 | this.add('performance.first-contentful-paint.score', lhr.audits['first-contentful-paint'].score) 102 | this.add('performance.first-contentful-paint.value', lhr.audits['first-contentful-paint'].numericValue) 103 | 104 | this.add('performance.largest-contentful-paint.score', lhr.audits['largest-contentful-paint'].score) 105 | this.add('performance.largest-contentful-paint.value', lhr.audits['largest-contentful-paint'].numericValue) 106 | 107 | this.add('performance.first-meaningful-paint.score', lhr.audits['first-meaningful-paint'].score) 108 | this.add('performance.first-meaningful-paint.value', lhr.audits['first-meaningful-paint'].numericValue) 109 | 110 | this.add('performance.speed-index.score', lhr.audits['speed-index'].score) 111 | this.add('performance.speed-index.value', lhr.audits['speed-index'].numericValue) 112 | 113 | this.add('performance.total-blocking-time.score', lhr.audits['total-blocking-time'].score) 114 | this.add('performance.total-blocking-time.value', lhr.audits['total-blocking-time'].numericValue) 115 | 116 | this.add('performance.cumulative-layout-shift.score', lhr.audits['cumulative-layout-shift'].score) 117 | this.add('performance.cumulative-layout-shift.value', lhr.audits['cumulative-layout-shift'].numericValue) 118 | 119 | this.add('performance.first-cpu-idle.score', 0) 120 | this.add('performance.first-cpu-idle.value', 0) 121 | 122 | this.add('performance.interactive.score', lhr.audits['interactive'].score) 123 | this.add('performance.interactive.value', lhr.audits['interactive'].numericValue) 124 | 125 | this.add('performance.estimated-input-latency.score', 0) 126 | this.add('performance.estimated-input-latency.value', 0) 127 | 128 | this.add('performance.byte-weight.total.score', lhr.audits['total-byte-weight'].score) 129 | this.add('performance.byte-weight.total.value', lhr.audits['total-byte-weight'].numericValue) 130 | 131 | this.add('performance.longest-request-chain.duration', lhr.audits['critical-request-chains'].details ? lhr.audits['critical-request-chains'].details.longestChain.duration : null) 132 | this.add('performance.longest-request-chain.length', lhr.audits['critical-request-chains'].details ? lhr.audits['critical-request-chains'].details.longestChain.length : null) 133 | this.add('performance.mainthread-work-breakdown', lhr.audits['mainthread-work-breakdown'].numericValue) 134 | this.add('performance.bootup-time', lhr.audits['bootup-time'].numericValue) 135 | 136 | this.add('performance.dom-size.score', lhr.audits['dom-size'].score) 137 | this.add('performance.dom-size.value', lhr.audits['dom-size'].numericValue) 138 | 139 | this.add('performance.total.timing', lhr.timing.total) 140 | 141 | this.add('performance.total.score', lhr.categories['performance'].score) 142 | 143 | let totalBytes = [] 144 | 145 | if (lhr.audits['network-requests'].details) { 146 | lhr.audits['network-requests'].details.items.forEach(resource => { 147 | const type = resource.resourceType // e.g. script, image, document, ... 148 | if (!totalBytes[type]) 149 | totalBytes[type] = 0 150 | 151 | totalBytes[type] = totalBytes[type] + resource.transferSize 152 | }) 153 | } 154 | 155 | for (let key in totalBytes) { 156 | this.add('performance.byte-weight.' + key.toLowerCase(), totalBytes[key]) 157 | } 158 | } 159 | 160 | add(key, value) { 161 | this._metrics.set(key, value) 162 | } 163 | 164 | metrics() { 165 | let obj = {} 166 | for (let [k,v] of this._metrics) { 167 | obj[k] = v 168 | } 169 | return obj 170 | } 171 | 172 | meta() { 173 | return this._meta 174 | } 175 | } 176 | 177 | module.exports = { 178 | Lighthouse, 179 | DefaultOptions, 180 | LighthouseReport 181 | } -------------------------------------------------------------------------------- /src/lighthouse.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const { Lighthouse, DefaultOptions } = require('./lighthouse') 6 | 7 | const chai = require('chai') 8 | const expect = chai.expect 9 | const sinon = require('sinon') 10 | const sinonChai = require('sinon-chai') 11 | chai.use(sinonChai) 12 | 13 | const report = require('./test-fixtures/report.json') 14 | 15 | 16 | describe('Lighthouse', function() { 17 | it('sends a report with config to receivers', async function() { 18 | const spy1 = sinon.spy() 19 | const receiver1 = { receive: spy1 } 20 | const spy2 = sinon.spy() 21 | const receiver2 = { receive: spy2 } 22 | const config = Object.assign({}, DefaultOptions, {url: 'https://kumbier.it'}) 23 | const lighthouse = new Lighthouse(config) 24 | sinon.stub(lighthouse, 'result').returns(new Promise((resolve) => resolve(report))) 25 | 26 | await lighthouse.reportTo([receiver1, receiver2]) 27 | 28 | expect(spy1.called).to.equal(true) 29 | expect(spy2.called).to.equal(true) 30 | expect(spy1).to.have.been.calledWith(report, config) 31 | expect(spy2).to.have.been.calledWith(report, config) 32 | }) 33 | 34 | it('calls a defined prehook before evaluating', async () => { 35 | const setup = sinon.spy(); 36 | const prehook = { setup }; 37 | const evaluation = sinon.spy() 38 | const sut = new Lighthouse({}, evaluation) 39 | 40 | await sut._evaluate({prehook: prehook}) 41 | 42 | expect(setup).to.have.been.calledBefore(evaluation) 43 | }) 44 | }) -------------------------------------------------------------------------------- /src/prom-client.learn-spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const prom = require('prom-client') 6 | const expect = require('chai').expect 7 | 8 | 9 | describe('PromClient', function () { 10 | beforeEach(() => { 11 | prom.register.clear() 12 | prom.register.resetMetrics() 13 | }) 14 | 15 | afterEach(() => { 16 | prom.register.clear() 17 | prom.register.resetMetrics() 18 | prom.collectDefaultMetrics({timeout: 5000}) 19 | }) 20 | 21 | it('deactivates default metrics if asked to', function () { 22 | clearInterval(prom.collectDefaultMetrics()) 23 | prom.register.clear() 24 | expect(prom.register.metrics()).not.to.contain('process_') 25 | }) 26 | 27 | it('has gauges that can increase and decrease', function () { 28 | const gauge = new prom.Gauge({ name: 'some_metric', help: 'A test gauge metric' }) 29 | prom.register.registerMetric(gauge) 30 | gauge.set(10) 31 | expect(prom.register.metrics()).to.include('TYPE some_metric gauge') 32 | expect(prom.register.metrics()).to.include('HELP some_metric A test gauge metric') 33 | expect(prom.register.metrics()).to.include('some_metric 10') 34 | 35 | gauge.set(18) 36 | expect(prom.register.metrics()).to.include('some_metric 18') 37 | 38 | 39 | gauge.set(9) 40 | expect(prom.register.metrics()).to.include('some_metric 9') 41 | }) 42 | 43 | it('metrics require name and help', function () { 44 | expect(() => new prom.Gauge({})).to.throw() 45 | expect(() => new prom.Gauge({ name: 'foo' })).to.throw() 46 | expect(() => new prom.Gauge({ help: 'foo' })).to.throw() 47 | expect(() => new prom.Gauge({ name: 'foo', help: 'foo' })).not.to.throw() 48 | }) 49 | 50 | describe('Labels', function () { 51 | beforeEach(() => { 52 | prom.register.clear() 53 | prom.register.resetMetrics() 54 | 55 | this._gauge = new prom.Gauge({ 56 | name: 'ttfb', 57 | help: 'Time to first byte', 58 | labelNames: [ 59 | 'devicetype', 60 | 'speed', 61 | 'url' 62 | ] 63 | }) 64 | 65 | prom.register.registerMetric(this._gauge) 66 | }) 67 | 68 | it('values can still be set without labels', () => { 69 | this._gauge.set(10) 70 | expect(prom.register.metrics()).to.include('ttfb 10') 71 | }) 72 | 73 | it('cannot set values with wrong labels', () => { 74 | expect(() => this._gauge.set({ 'non-existant-label': 'some-value' }, 100)).to.throw() 75 | }) 76 | 77 | it('can set values with all registered labels given', () => { 78 | this._gauge.set({ 79 | 'devicetype': 'mobile', 80 | 'speed': 'slow', 81 | 'url': 'https://kumbier.it' 82 | }, 15) 83 | expect(prom.register.metrics()).to.include('ttfb') 84 | expect(prom.register.metrics()).to.include('devicetype="mobile"') 85 | expect(prom.register.metrics()).to.include('speed="slow"') 86 | expect(prom.register.metrics()).to.include('url="https://kumbier.it') 87 | }) 88 | 89 | it('can set values with only a subset of the registered metrics', () => { 90 | this._gauge.set({ 91 | 'devicetype': 'mobile', 92 | 'url': 'https://kumbier.it' 93 | }, 15) 94 | expect(prom.register.metrics()).to.include('ttfb') 95 | expect(prom.register.metrics()).to.include('devicetype="mobile"') 96 | expect(prom.register.metrics()).to.include('url="https://kumbier.it') 97 | }) 98 | 99 | it('cannot instantiate a gauge with an existing name', () => { 100 | new prom.Gauge({name: 'g', help: 'g1'}) 101 | expect(() => new prom.Gauge({name: 'g', help: 'g2'})).to.throw() 102 | }) 103 | }) 104 | }) -------------------------------------------------------------------------------- /src/receivers/directory.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const zlib = require('zlib') 6 | const fs = require('fs-extra') 7 | const path = require('path') 8 | const os = require('os') 9 | const debug = require('debug') 10 | 11 | class Directory { 12 | constructor(directory) { 13 | if (!directory) { 14 | directory = os.tmpdir() 15 | debug(`Reports will be saved to ${directory}`) 16 | } 17 | this._directory = directory 18 | } 19 | 20 | async receive(report, config) { 21 | this._saveJson(await this._target(this._directory, 'config.json', config), config) 22 | this._saveJson(await this._target(this._directory, 'report.json', config), report.lhr) 23 | 24 | if (report.artifacts) { 25 | this._saveJson(await this._target(this._directory, 'artifacts.json', config), report.artifacts) 26 | } 27 | 28 | if (config.output instanceof Array && config.output.some(item => item === 'html')) { 29 | const i = config.output.findIndex(item => item === 'html') 30 | const target = await this._target(this._directory, 'report.html', config) 31 | const content = report.report[i] 32 | this._save(target, content) 33 | } 34 | } 35 | 36 | async _target(dir, filename, config={}) { 37 | await fs.ensureDir(dir) 38 | const parts = [] 39 | 40 | if (config.name) { 41 | parts.push(config.name) 42 | } 43 | 44 | if (config.preset) { 45 | parts.push(config.preset) 46 | } 47 | 48 | parts.push(filename) 49 | 50 | const targetfile = parts.join('_') 51 | return path.join(dir, targetfile) 52 | } 53 | 54 | _saveJson(target, data) { 55 | this._save(target, JSON.stringify(data, null, 2)) 56 | } 57 | 58 | _save(target, content) { 59 | const compressed = zlib.gzipSync(content) 60 | fs.writeFileSync(target + '.gz', compressed) 61 | } 62 | } 63 | 64 | module.exports = { Directory } -------------------------------------------------------------------------------- /src/receivers/directory.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const chai = require('chai') 6 | const expect = chai.expect 7 | const chaiFiles = require('chai-files') 8 | chai.use(chaiFiles) 9 | const file = chaiFiles.file 10 | const pathjoin = require('path').join 11 | const rm = require('rimraf') 12 | const fs = require('fs') 13 | const tmpdir = require('os').tmpdir 14 | 15 | const { Directory } = require('./directory') 16 | 17 | const reportFixture = require('../test-fixtures/report.json') 18 | 19 | 20 | describe('Directory', function() { 21 | this.beforeEach(() => { 22 | this._baseDir = fs.mkdtempSync(pathjoin(tmpdir(), 'mocha-')) 23 | this._target = pathjoin(this._baseDir, 'DirectoryReceiverTest') 24 | }) 25 | 26 | it('creates the directory, if it does not exist and saves all run-data', async () => { 27 | const dir = new Directory(this._target) 28 | await dir.receive(reportFixture, {url: 'https://kumbier.it'}) 29 | 30 | const reportFile = pathjoin(this._target, 'report.json.gz') 31 | expect(file(reportFile)).to.exist 32 | expect(file(reportFile)).not.to.be.empty 33 | 34 | const artifactFile = pathjoin(this._target, 'artifacts.json.gz') 35 | expect(file(artifactFile)).to.exist 36 | expect(file(artifactFile)).not.to.be.empty 37 | 38 | const configFile = pathjoin(this._target, 'config.json.gz') 39 | expect(file(configFile)).to.exist 40 | expect(file(configFile)).not.to.be.empty 41 | }) 42 | 43 | it('prefixes the filename optionally', async () => { 44 | const dir = new Directory(this._target) 45 | await dir.receive(reportFixture, {name: 'kits', preset: '3G'}) 46 | const expected = pathjoin(this._target, 'kits_3G_report.json.gz') 47 | expect(file(expected)).to.exist 48 | const notExpected = pathjoin(this._target, 'report.json.gz') 49 | expect(file(notExpected)).not.to.exist 50 | }) 51 | 52 | afterEach(() => { 53 | rm.sync(this._baseDir) 54 | }) 55 | }) -------------------------------------------------------------------------------- /src/receivers/file-prometheus.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const { Prometheus } = require('./prometheus') 6 | 7 | const fs = require('fs-extra') 8 | const prom = require('prom-client') 9 | 10 | 11 | class FilePrometheus extends Prometheus { 12 | constructor(filename) { 13 | super() 14 | 15 | if (!filename) { 16 | throw new Error('Prometheus requires a filename to save metrics to') 17 | } 18 | 19 | this.filename = filename 20 | } 21 | 22 | async receive(report, config) { 23 | super.receive(report, config) 24 | } 25 | 26 | async afterEvaluation() { 27 | await fs.ensureFile(this.filename) 28 | await fs.writeFile(this.filename, prom.register.metrics()) 29 | } 30 | } 31 | 32 | 33 | module.exports = { 34 | FilePrometheus, 35 | } -------------------------------------------------------------------------------- /src/receivers/file-prometheus.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const chai = require('chai') 6 | const expect = chai.expect 7 | const fs = require('fs-extra') 8 | const join = require('path').join 9 | const prom = require('prom-client') 10 | const tmpdir = require('os').tmpdir 11 | 12 | const { FilePrometheus } = require('./file-prometheus') 13 | const reportFixture = require('../test-fixtures/report.json') 14 | 15 | chai.use(require('chai-fs')) 16 | 17 | 18 | describe('FilePrometheus', function() { 19 | beforeEach(() => { 20 | prom.register.clear() 21 | prom.register.resetMetrics() 22 | this._baseDir = fs.mkdtempSync(join(tmpdir(), 'mocha-')) 23 | this._target = join(this._baseDir, 'metrics') 24 | }) 25 | 26 | afterEach(async () => { 27 | prom.register.clear() 28 | prom.register.resetMetrics() 29 | await fs.remove(this._baseDir) 30 | }) 31 | 32 | it('adds new metrics over time', async () => { 33 | const p = new FilePrometheus(this._target) 34 | await p.receive(reportFixture, {url: 'https://kumbier.it'}) 35 | await p.afterEvaluation() 36 | 37 | expect(this._target).to.be.a.file() 38 | expect(this._target).with.contents.that.match(/url="https:\/\/kumbier.it"/) 39 | expect(this._target).not.with.contents.that.match(/url="https:\/\/example.com"/) 40 | 41 | await p.receive(reportFixture, {url: 'https://example.com'}) 42 | await p.afterEvaluation() 43 | expect(this._target).with.contents.that.match(/url="https:\/\/kumbier.it"/) 44 | expect(this._target).with.contents.that.match(/url="https:\/\/example.com"/) 45 | }) 46 | }) 47 | -------------------------------------------------------------------------------- /src/receivers/json-metrics.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const fs = require('fs-extra') 6 | const { LighthouseReport } = require('../lighthouse') 7 | 8 | 9 | class JsonMetrics { 10 | constructor(target=`${__dirname}/../../public/metrics.json`) { 11 | this._target = target 12 | this._reports = [] 13 | } 14 | 15 | async receive(report, config) { 16 | this._reports.push(new LighthouseReport(report, config)) 17 | } 18 | 19 | async afterEvaluation() { 20 | const payload = { 21 | timestamp: new Date().toISOString(), 22 | reports: this._reports.map(report => { 23 | return { 24 | url: report.meta().url, 25 | profile: report.meta().profile, 26 | metrics: report.metrics() 27 | } 28 | }) 29 | } 30 | 31 | await fs.ensureFile(this._target) 32 | await fs.writeFile(this._target, JSON.stringify(payload, null, 2)) 33 | } 34 | } 35 | 36 | module.exports = { JsonMetrics } -------------------------------------------------------------------------------- /src/receivers/new-relic.integration-spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const expect = require('chai').expect 6 | 7 | const { NewRelic } = require('./new-relic') 8 | const reportFixture = require('../test-fixtures/report.json') 9 | 10 | const InvalidCredentialsOptions = { 11 | apiKey: 'asldkfhgaslkdjfhaskldfhaklsdjfh', 12 | accountId: 123456 13 | } 14 | 15 | 16 | describe('NewRelic (Integration)', async function () { 17 | it('fails gracefully when invalid credentials are given', async () => { 18 | const newRelic = new NewRelic(InvalidCredentialsOptions) 19 | expect(async () => await newRelic.receive(reportFixture, {url: 'https://kumbier.it'})).to.not.throw() 20 | }) 21 | }) -------------------------------------------------------------------------------- /src/receivers/new-relic.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const debug = require('debug') 6 | const request = require('request-promise-native') 7 | const { LighthouseReport } = require('../lighthouse') 8 | 9 | class NewRelic { 10 | constructor(options) { 11 | this._config = (options instanceof NewRelicOptions) ? options : new NewRelicOptions(options) 12 | } 13 | 14 | async receive(report, runConfig) { 15 | const requestBody = [this._transform(report, runConfig)] 16 | 17 | const requestOptions = { 18 | method: 'POST', 19 | uri: this._config.apiUrl, 20 | headers: { 21 | 'X-Insert-Key': this._config.apiKey 22 | }, 23 | body: requestBody, 24 | json: true 25 | } 26 | 27 | try { 28 | await request(requestOptions) 29 | } catch(e) { 30 | debug('LIGHTMON:WARN')('Could not send report to NewRelic - error was:', e.toString()) 31 | } 32 | } 33 | 34 | _transform(report, config) { 35 | const flattened = new NewRelicReport(report, config) 36 | flattened.add('eventType', 'PerformanceMonitoring') 37 | const obj = {} 38 | for (const [key, report] of flattened._metrics) { 39 | obj[key] = report 40 | } 41 | return obj 42 | } 43 | } 44 | 45 | 46 | class NewRelicOptions { 47 | constructor(options) { 48 | if (!options.accountId) { 49 | throw Error('Missing required option: accountId') 50 | } 51 | 52 | if (!options.apiKey) { 53 | throw Error('Missing required option: apiKey') 54 | } 55 | 56 | this.apiKey = options.apiKey 57 | this.apiUrl = options.apiUrl || `https://insights-collector.newrelic.com/v1/accounts/${options.accountId}/events` 58 | } 59 | } 60 | 61 | class NewRelicReport extends LighthouseReport { 62 | constructor(report, config) { 63 | super(report, config) 64 | 65 | this._flattenMetaToMetrics() 66 | } 67 | 68 | // for newrelic, meta data is just another metric in the list. 69 | _flattenMetaToMetrics() { 70 | for(const key of Object.keys(this._meta)) { 71 | this.add(key, this._meta[key]) 72 | } 73 | } 74 | 75 | add(key, value) { 76 | this._metrics.set(key, value) 77 | } 78 | } 79 | 80 | module.exports = { 81 | NewRelic, 82 | NewRelicOptions, 83 | NewRelicReport, 84 | } -------------------------------------------------------------------------------- /src/receivers/new-relic.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const expect = require('chai').expect 6 | 7 | const { NewRelic, NewRelicOptions } = require('./new-relic') 8 | const reportFixture = require('../test-fixtures/report.json') 9 | 10 | const NewRelicOptionsValid = { 11 | apiKey: 'some-secret-key', 12 | accountId: 12345, 13 | apiUrl: undefined, 14 | } 15 | 16 | 17 | describe('NewRelic', function() { 18 | it('instantiates a NewRelic Instance from string', () => { 19 | expect(() => new NewRelic(NewRelicOptionsValid)).not.to.throw() 20 | const invalidOptions = Object.assign({}, NewRelicOptionsValid, {accountId: undefined}) 21 | expect(() => new NewRelic(invalidOptions)).to.throw() 22 | }) 23 | 24 | it('extracts audit data from lighthouse report', () => { 25 | const sus = new NewRelic(NewRelicOptionsValid) 26 | const transformed = sus._transform(reportFixture, {url: 'https://kumbier.it'}) 27 | expect(Object.keys(transformed).length).to.be.greaterThan(10) 28 | }) 29 | 30 | it('extracts byte-weights from lighthouse report', () => { 31 | const sus = new NewRelic(NewRelicOptionsValid) 32 | const transformed = sus._transform(reportFixture, {url: 'https://kumbier.it'}) 33 | expect(transformed['performance.byte-weight.document']).to.exist 34 | expect(transformed['performance.byte-weight.image']).to.exist 35 | expect(transformed['performance.byte-weight.font']).to.exist 36 | expect(transformed['performance.byte-weight.other']).to.exist 37 | expect(transformed['performance.byte-weight.script']).to.exist 38 | expect(transformed['performance.byte-weight.stylesheet']).to.exist 39 | }) 40 | 41 | describe('NewRelicOptions', function () { 42 | it('can instantiate a NewRelicOptions instance', () => { 43 | expect(() => new NewRelicOptions(NewRelicOptionsValid)).not.to.throw() 44 | }) 45 | 46 | it('throws when missing data is given', () => { 47 | const missingAccount = Object.assign({}, NewRelicOptionsValid, {accountId: undefined}) 48 | expect(() => new NewRelicOptions(missingAccount)).to.throw() 49 | 50 | const missingApiKey = Object.assign({}, NewRelicOptionsValid, {apiKey: undefined}) 51 | expect(() => new NewRelicOptions(missingApiKey)).to.throw() 52 | }) 53 | }) 54 | }) -------------------------------------------------------------------------------- /src/receivers/prometheus.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const { LighthouseReport } = require('../lighthouse') 6 | 7 | const prom = require('prom-client') 8 | 9 | 10 | class Prometheus { 11 | async receive(report, config) { 12 | new PrometheusReport(report, config).save() 13 | } 14 | } 15 | 16 | 17 | class PrometheusReport extends LighthouseReport { 18 | add(key, value) { 19 | const newKey = key.replace(/\./g, ':').replace(/-/g, '_') 20 | this._metrics.set(newKey, new Metric(this._meta, newKey, value)) 21 | } 22 | 23 | save() { 24 | for (const metric of this._metrics.values()) { 25 | metric.save() 26 | } 27 | } 28 | } 29 | 30 | 31 | class Metric { 32 | constructor(labels, name, value) { 33 | this.name = name 34 | this.labels = labels 35 | this.labelNames = Object.keys(labels) 36 | this.value = value || 0 37 | 38 | this.help = this.name 39 | } 40 | 41 | save() { 42 | let metric = prom.register.getSingleMetric(this.name) 43 | 44 | if (!metric) { 45 | metric = new prom.Gauge(this) 46 | prom.register.registerMetric(metric) 47 | } 48 | 49 | metric.set(this.labels, this.value) 50 | } 51 | } 52 | 53 | 54 | module.exports = { 55 | Prometheus, 56 | PrometheusReport, 57 | } -------------------------------------------------------------------------------- /src/receivers/prometheus.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const chai = require('chai') 6 | const expect = chai.expect 7 | const prom = require('prom-client') 8 | 9 | const { Prometheus, PrometheusReport } = require('./prometheus') 10 | const reportFixture = require('../test-fixtures/report.json') 11 | 12 | chai.use(require('chai-fs')) 13 | 14 | 15 | describe('Prometheus', function() { 16 | beforeEach(() => { 17 | prom.register.clear() 18 | prom.register.resetMetrics() 19 | }) 20 | 21 | afterEach(async () => { 22 | prom.register.clear() 23 | prom.register.resetMetrics() 24 | }) 25 | 26 | it('adds a single metric when receiving a report', async () => { 27 | const p = new Prometheus() 28 | await p.receive(reportFixture, {url: 'https://kumbier.it'}) 29 | 30 | const metric = prom.register.getSingleMetricAsString('performance:first_meaningful_paint:score') 31 | expect(metric).to.contain('performance:first_meaningful_paint:score') 32 | expect(metric).to.contain('url="https://kumbier.it"') 33 | }) 34 | }) 35 | 36 | 37 | describe ('PrometheusReport', function() { 38 | it('only contains valid prometheus metric keys', function() { 39 | const report = new PrometheusReport(reportFixture, {url: 'https://kumbier.it'}) 40 | for (const key in report) { 41 | expect(key).to.match(/^[a-zA-Z_:]([a-zA-Z0-9_:])*$/) 42 | } 43 | }) 44 | }) -------------------------------------------------------------------------------- /src/receivers/push-prometheus.js: -------------------------------------------------------------------------------- 1 | const debug = require('debug') 2 | const { Prometheus } = require('./prometheus') 3 | 4 | const prom = require('prom-client') 5 | 6 | 7 | class PushPrometheus extends Prometheus { 8 | constructor(pushgw) { 9 | super() 10 | 11 | if(!pushgw) { 12 | throw new Error('PushPrometheus requires a pushgateway url') 13 | } 14 | 15 | this.pushgw = new prom.Pushgateway(pushgw) 16 | } 17 | 18 | 19 | async receive(report, config) { 20 | await super.receive(report, config) 21 | 22 | this.pushgw.pushAdd({jobName: 'lightmon'}, (err) => { 23 | if (err) { 24 | debug('LIGHTMON:PUSHGW:WARN')(err) 25 | } 26 | }) 27 | } 28 | } 29 | 30 | 31 | module.exports = { 32 | PushPrometheus, 33 | } -------------------------------------------------------------------------------- /src/receivers/timestamped-directory.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const path = require('path') 6 | 7 | const { Directory } = require('./directory') 8 | 9 | 10 | class TimestampedDirectory extends Directory { 11 | _target(dir, filename, config) { 12 | const startedAt = config.runStartedAt.replace(/[><:|?*]/g, '_') 13 | const now = path.join(dir, startedAt) 14 | return super._target(now, filename, config) 15 | } 16 | } 17 | 18 | 19 | module.exports = { TimestampedDirectory } 20 | -------------------------------------------------------------------------------- /src/receivers/timestamped-directory.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const chai = require('chai') 6 | const expect = chai.expect 7 | const chaiFiles = require('chai-files') 8 | chai.use(chaiFiles) 9 | const file = chaiFiles.file 10 | const pathjoin = require('path').join 11 | const fs = require('fs-extra') 12 | const tmpdir = require('os').tmpdir 13 | 14 | const { TimestampedDirectory } = require('./timestamped-directory') 15 | 16 | const reportFixture = require('../test-fixtures/report.json') 17 | 18 | 19 | describe('TimestampedDirectory', function() { 20 | this.beforeEach(() => { 21 | this._baseDir = fs.mkdtempSync(pathjoin(tmpdir(), 'mocha-')) 22 | }) 23 | 24 | it('creates the directory, if it does not exist and saves report', async () => { 25 | const dir = new TimestampedDirectory(this._baseDir) 26 | const config = {url: 'https://kumbier.it', runStartedAt: new Date().toISOString()} 27 | const reportFile = await dir._target(this._baseDir, 'report.json.gz', config) 28 | 29 | await dir.receive(reportFixture, config) 30 | 31 | expect(file(reportFile)).to.exist 32 | }) 33 | 34 | it('does not contain invalid windows filesystem characters', async () => { 35 | const dir = new TimestampedDirectory(this._baseDir) 36 | const config = {url: 'https://kumbier.it', runStartedAt: new Date().toISOString()} 37 | let reportFile = await dir._target(this._baseDir, 'report.json.gz', config) 38 | 39 | // through away drive letter for windows as it contains colon 40 | if (process.platform === 'win32') { 41 | reportFile = reportFile.substr(reportFile.indexOf('\\')); 42 | } 43 | 44 | expect(reportFile).not.to.match(/[><:|?*]/) 45 | }) 46 | 47 | afterEach(async () => { 48 | await fs.remove(this._baseDir) 49 | }) 50 | }) 51 | -------------------------------------------------------------------------------- /src/report-cleanup.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const Moment = require('moment') 6 | const debug = require('debug')('LIGHTMON:CLEANUP:DEBUG') 7 | const info = require('debug')('LIGHTMON:CLEANUP:INFO') 8 | const warn = require('debug')('LIGHTMON:CLEANUP:WARNING') 9 | const fs = require('fs-extra') 10 | const path = require('path') 11 | 12 | 13 | class ReportCleanup { 14 | constructor({ retainWeekly, retainDaily, reports, dryRun = false }) { 15 | this.retainWeekly = retainWeekly 16 | this.retainDaily = retainDaily 17 | this.reports = reports 18 | this.dryRun = dryRun 19 | } 20 | 21 | async clean() { 22 | if (this.retainWeekly !== undefined) { 23 | const weekly = this.reports.olderThan(new Moment().subtract(this.retainWeekly, 'days').toDate()) 24 | await this.weekly(this.reports, weekly) 25 | } 26 | 27 | if (this.retainDaily !== undefined) { 28 | const daily = this.reports.olderThan(new Moment().subtract(this.retainDaily, 'days').toDate()) 29 | await this.daily(this.reports, daily) 30 | } 31 | } 32 | 33 | async weekly(webDir, reports) { 34 | info(`Crunching reports older than ${this.retainWeekly} days from daily down to weekly resolution`) 35 | const buckets = this.bucketize(reports, this._keyGenWeekly) 36 | await this.reduce(webDir, buckets) 37 | } 38 | 39 | async daily(webDir, reports) { 40 | info(`Crunching reports older than ${this.retainDaily} days from full down to daily resolution`) 41 | const buckets = this.bucketize(reports, this._keyGenDaily) 42 | await this.reduce(webDir, buckets) 43 | } 44 | 45 | 46 | bucketize(reports, keyGen) { 47 | const buckets = new Map() 48 | 49 | for(const report of reports) { 50 | const key = keyGen(report) 51 | if (!buckets.has(key)) { 52 | buckets.set(key, []) 53 | } 54 | const bucket = buckets.get(key) 55 | bucket.push(report) 56 | buckets.set(key, bucket) 57 | } 58 | 59 | return buckets 60 | } 61 | 62 | _keyGenWeekly(report) { 63 | const date = Moment.utc(report.date) 64 | return `${date.format('YYYYww')}_${report.name}_${report.preset}` 65 | } 66 | 67 | 68 | _keyGenDaily(report) { 69 | const date = Moment.utc(report.date) 70 | return `${date.format('YYYYMMDD')}_${report.name}_${report.preset}` 71 | } 72 | 73 | async reduce(reports, buckets) { 74 | for(const bucket of buckets.values()) { 75 | bucket.sort((a, b) => a.date < b.date) 76 | bucket.shift() 77 | 78 | for(const report of bucket) { 79 | debug(`Deleting ${report.path}/${report.name}_${report.preset} and associated files`) 80 | if (!this.dryRun) { 81 | reports.delete(report) 82 | } 83 | } 84 | } 85 | } 86 | 87 | async purgeEmptyDirs() { 88 | const reportDir = this.reports.baseDir 89 | 90 | for (const entry of fs.readdirSync(reportDir)) { 91 | const subdir = path.join(reportDir, entry) 92 | 93 | if (!fs.statSync(subdir).isDirectory()) { 94 | continue 95 | } 96 | 97 | if (fs.readdirSync(subdir).length > 0) { 98 | continue 99 | } 100 | 101 | try { 102 | debug(`Deleting empty directory ${subdir}`) 103 | if (!this.dryRun) { 104 | await fs.rmdir(subdir) 105 | } 106 | } catch (e) { 107 | warn(`Could not delete empty directory ${subdir} - error: ${e}`) 108 | } 109 | } 110 | } 111 | } 112 | 113 | 114 | module.exports = { 115 | ReportCleanup, 116 | } 117 | -------------------------------------------------------------------------------- /src/report-cleanup.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const chai = require('chai') 6 | chai.use(require('chai-fs')) 7 | const expect = chai.expect 8 | const Moment = require('moment') 9 | const fs = require('fs-extra') 10 | const join = require('path').join 11 | const tmpdir = require('os').tmpdir 12 | 13 | const { Reports } = require('./reports') 14 | const { ReportCleanup } = require('./report-cleanup') 15 | 16 | const Fixture = `${__dirname}/test-fixtures/reports/store` 17 | const dirWithoutContent = '2018-08-02T00_00_00.034Z' 18 | const dirWithContent = '2018-08-02T14_10_57.975Z' 19 | 20 | 21 | 22 | /** 23 | * make sure that a week starts on a sunday for these tests. 24 | */ 25 | Moment.locale('en') 26 | 27 | 28 | function countBucketEntries(buckets) { 29 | return Array.from(buckets.values()).reduce((acc, cur) => acc.concat(cur), []).length 30 | } 31 | 32 | 33 | class MockReport { 34 | constructor({url, name, preset, date, path}) { 35 | this.url = url || 'https://kumbier.it' 36 | this.name = name || 'KITS' 37 | this.preset = preset || 'mobile-fast' 38 | this.date = date || '2018-07-15T23:12:05.023Z' 39 | this.path = path || '/tmp/reports' 40 | } 41 | } 42 | 43 | 44 | describe('ReportCleanup', function () { 45 | this.beforeEach(() => { 46 | this._baseDir = fs.mkdtempSync(join(tmpdir(), 'mocha-')) 47 | fs.copySync(Fixture, this._baseDir) 48 | }) 49 | 50 | this.afterEach(() => { 51 | if (this.reports) { 52 | this.reports.destroy() 53 | } 54 | try { 55 | fs.removeSync(this._baseDir) 56 | } catch (e) {} // eslint-disable-line no-empty 57 | }) 58 | 59 | it('_keyGenWeekly() generates correct bucket keys', () => { 60 | const sut = new ReportCleanup({}) 61 | const sunday = new MockReport({date: '2019-01-06T00:00:00.000Z'}) 62 | const saturday = new MockReport({date: '2019-01-12T23:59:59.999Z'}) 63 | const nextSunday = new MockReport({date: '2019-01-13T00:00:00.000Z'}) 64 | 65 | expect(sut._keyGenWeekly(sunday)).to.equal(sut._keyGenWeekly(saturday)) 66 | expect(sut._keyGenWeekly(saturday)).not.to.equal(sut._keyGenWeekly(nextSunday)) 67 | }) 68 | 69 | 70 | it('_keyGenDaily() generates correct bucket keys', () => { 71 | const sut = new ReportCleanup({}) 72 | const saturdayStart = new MockReport({date: '2019-01-12T00:00:00.000Z'}) 73 | const saturdayEnd = new MockReport({date: '2019-01-12T23:59:59.999Z'}) 74 | const sunday = new MockReport({date: '2019-01-13T00:00:00.000Z'}) 75 | 76 | expect(sut._keyGenWeekly(saturdayStart)).to.equal(sut._keyGenWeekly(saturdayEnd)) 77 | expect(sut._keyGenWeekly(saturdayEnd)).not.to.equal(sut._keyGenWeekly(sunday)) 78 | }) 79 | 80 | describe('bucketize()', () => { 81 | it('puts reports of same name and same preset into daily buckets', () => { 82 | const sut = new ReportCleanup({}) 83 | const reports = [ 84 | new MockReport({date: '2019-01-12T00:00:00.000Z'}), 85 | new MockReport({date: '2019-01-12T23:59:59.999Z'}), 86 | new MockReport({date: '2019-01-13T00:00:00.000Z'}) 87 | ] 88 | const buckets = sut.bucketize(reports, sut._keyGenDaily) 89 | expect(buckets.size).to.equal(2) 90 | expect(countBucketEntries(buckets)).to.equal(reports.length) 91 | }) 92 | 93 | it('puts reports of same name and different presets into daily buckets', () => { 94 | const sut = new ReportCleanup({}) 95 | const reports = [ 96 | new MockReport({date: '2019-01-12T00:00:00.000Z', preset: 'fast'}), 97 | new MockReport({date: '2019-01-12T00:01:00.000Z', preset: 'slow'}), 98 | new MockReport({date: '2019-01-12T23:59:59.999Z', preset: 'fast'}), 99 | new MockReport({date: '2019-01-13T00:00:00.000Z', preset: 'fast'}), 100 | new MockReport({date: '2019-01-13T00:00:00.000Z', preset: 'slow'}) 101 | ] 102 | const buckets = sut.bucketize(reports, sut._keyGenDaily) 103 | expect(buckets.size).to.equal(4) 104 | expect(countBucketEntries(buckets)).to.equal(reports.length) 105 | }) 106 | 107 | it('puts reports of different names and different presets into daily buckets', () => { 108 | const sut = new ReportCleanup({}) 109 | const reports = [ 110 | new MockReport({date: '2019-01-12T00:00:00.000Z', name: 'KITS', preset: 'fast'}), 111 | new MockReport({date: '2019-01-12T00:01:00.000Z', name: 'KITS', preset: 'slow'}), 112 | new MockReport({date: '2019-01-12T23:59:59.999Z', name: 'KITS', preset: 'fast'}), 113 | new MockReport({date: '2019-01-13T00:00:00.000Z', name: 'KITS', preset: 'fast'}), 114 | new MockReport({date: '2019-01-13T00:00:00.000Z', name: 'KITS', preset: 'slow'}), 115 | new MockReport({date: '2019-01-12T00:00:00.000Z', name: 'EXAMPLE', preset: 'fast'}), 116 | new MockReport({date: '2019-01-12T00:01:00.000Z', name: 'EXAMPLE', preset: 'slow'}), 117 | new MockReport({date: '2019-01-12T23:59:59.999Z', name: 'EXAMPLE', preset: 'fast'}), 118 | new MockReport({date: '2019-01-13T00:00:00.000Z', name: 'EXAMPLE', preset: 'fast'}), 119 | new MockReport({date: '2019-01-13T00:00:00.000Z', name: 'EXAMPLE', preset: 'slow'}) 120 | ] 121 | const buckets = sut.bucketize(reports, sut._keyGenDaily) 122 | expect(buckets.size).to.equal(8) 123 | expect(countBucketEntries(buckets)).to.equal(reports.length) 124 | }) 125 | }) 126 | 127 | /** 128 | * The current cache cannot be prefilled. FIXME 129 | */ 130 | xit('removes empty directories in the reportDir', async () => { 131 | this.reports = await Reports.setup(this._baseDir, false) 132 | fs.unlinkSync(join(this._baseDir, dirWithoutContent, '.gitkeep')) 133 | const sut = new ReportCleanup({reports: this.reports, dryRun: false}) 134 | expect(this._baseDir).be.a.directory().and.include.subDirs([dirWithoutContent]) 135 | expect(this._baseDir).be.a.directory().and.include.subDirs([dirWithContent]) 136 | await sut.purgeEmptyDirs() 137 | 138 | /* 139 | ok, so: for whatever stupid reason windows does not sync the filesystem operations 140 | fast enough, so that this test will fail, if we omit this line. We are unable to 141 | force synchronization, because node does not expose that for directories, but only 142 | for files. This is a workaround and we are not proud of it. but it works. FIXME. 143 | plz. 144 | */ 145 | await fs.readdir(this._baseDir) 146 | expect(this._baseDir).be.a.directory().and.not.include.subDirs([dirWithoutContent]) 147 | expect(this._baseDir).be.a.directory().and.include.subDirs([dirWithContent]) 148 | }) 149 | }) -------------------------------------------------------------------------------- /src/report.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const crypto = require('crypto') 6 | const debug = require('debug') 7 | const { readFileSync, existsSync, unlinkSync } = require('fs-extra') 8 | const path = require('path') 9 | const zlib = require('zlib') 10 | 11 | 12 | class Report { 13 | constructor({ url, name, preset, date, path }) { 14 | if (!url || !name || !preset || !date || !path) { 15 | throw Error('Missing required option in configuration object - required: { url, name, preset, date, path }, given: ' + JSON.stringify(arguments[0], null, 2)) 16 | } 17 | 18 | const options = { 19 | url, 20 | name, 21 | preset, 22 | date, 23 | path 24 | } 25 | const id = this._calculateId(options) 26 | Object.assign(this, options, { id }) 27 | } 28 | 29 | static fromMeta({ url, name, preset, runStartedAt, reportsDir }) { 30 | if (!url || !name || !preset || !runStartedAt || !reportsDir) { 31 | throw Error('Missing required option in configuration object - required: { url, name, preset, runStartedAt, reportsDir }, given: ' + JSON.stringify(arguments[0], null, 2)) 32 | } 33 | 34 | return new Report({ 35 | url, 36 | name, 37 | preset, 38 | date: runStartedAt, 39 | path: path.join(reportsDir, runStartedAt.replace(/[><:|?*]/g, '_')) 40 | }) 41 | } 42 | 43 | static fromFile(dir, configFile) { 44 | const content = readFileSync(path.join(dir, configFile)) 45 | const config = JSON.parse(zlib.gunzipSync(content)) 46 | config.reportsDir = path.dirname(dir) 47 | return Report.fromMeta(config) 48 | } 49 | 50 | _calculateId(options) { 51 | const hash = crypto.createHash('md5') 52 | hash.update(JSON.stringify(options)) 53 | return hash.digest('hex') 54 | } 55 | 56 | get html() { 57 | return this._getTarget('report.html') 58 | } 59 | 60 | get artifacts() { 61 | return this._getTarget('artifacts.json') 62 | } 63 | 64 | get json() { 65 | return this._getTarget('report.json') 66 | } 67 | 68 | get config() { 69 | return this._getTarget('config.json') 70 | } 71 | 72 | _getTarget(type) { 73 | let target = path.join(this.path, `${this.name}_${this.preset}_${type}`) 74 | target = existsSync(target + '.gz') ? target + '.gz' : target 75 | 76 | if (!existsSync(target)) { 77 | return null 78 | } 79 | 80 | return target 81 | } 82 | 83 | withoutInternals() { 84 | const secured = Object.assign({}, this) 85 | delete secured.path 86 | return secured 87 | } 88 | 89 | delete() { 90 | [ 91 | this._getTarget('report.html'), 92 | this._getTarget('artifacts.json'), 93 | this._getTarget('report.json'), 94 | this._getTarget('config.json') 95 | ] 96 | .filter(file => !!file) 97 | .map(file => { 98 | try { 99 | unlinkSync(file) 100 | } catch (e) { 101 | debug('LIGHTMON:WARN')(`Could not delete file - error on ${file} was ${e}`) 102 | } 103 | }) 104 | } 105 | } 106 | 107 | module.exports = { 108 | Report 109 | } 110 | -------------------------------------------------------------------------------- /src/report.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const { Report } = require('./report') 6 | const { join, resolve } = require('path') 7 | const chai = require('chai') 8 | chai.use(require('chai-fs')) 9 | const expect = chai.expect 10 | const fs = require('fs-extra') 11 | const tmpdir = require('os').tmpdir 12 | 13 | const dirFixture = resolve(__dirname, './test-fixtures/reports/store') 14 | const subDir = '2018-08-03T09_10_00.013Z' 15 | const configFileFixture = 'KITS_mobile-fast_config.json.gz' 16 | 17 | 18 | describe('Report', function () { 19 | this.beforeEach(async function() { 20 | this._baseDir = fs.mkdtempSync(join(tmpdir(), 'mocha-')) 21 | fs.copySync(dirFixture, this._baseDir) 22 | }) 23 | 24 | this.afterEach(function() { 25 | try { 26 | fs.removeSync(this._baseDir) 27 | } catch (e) {} // eslint-disable-line no-empty 28 | }) 29 | 30 | 31 | it('has an identical canonical id for identical metadata', function () { 32 | const conf = { 33 | url: 'https://kumbier.it', 34 | name: 'KITS', 35 | preset: 'mobile-fast', 36 | date: '2018-07-15T23:12:05.023Z', 37 | path: '/some/path/2018-07-15T23_12_05.023Z' 38 | } 39 | const conf1 = new Report(conf) 40 | const conf2 = new Report(conf) 41 | 42 | expect(conf1.id).not.to.be.undefined 43 | expect(conf2.id).not.to.be.undefined 44 | expect(conf1.id).to.equal(conf2.id) 45 | }) 46 | 47 | 48 | it('can instantiate itself from meta-information', function () { 49 | const meta = { 50 | url: 'https://kumbier.it', 51 | name: 'KITS', 52 | preset: 'mobile-fast', 53 | reportsDir: dirFixture, 54 | runStartedAt: '2018-07-15T23:12:05.023Z' 55 | } 56 | const conf = { 57 | url: 'https://kumbier.it', 58 | name: 'KITS', 59 | preset: 'mobile-fast', 60 | date: '2018-07-15T23:12:05.023Z', 61 | path: join(dirFixture, '2018-07-15T23_12_05.023Z') 62 | } 63 | const actual = Report.fromMeta(meta) 64 | const expected = new Report(conf) 65 | expect(actual.id).to.equal(expected.id) 66 | }) 67 | 68 | 69 | it('can instantiate itself from a configfile', function () { 70 | const report = Report.fromFile(join(dirFixture, subDir), configFileFixture) 71 | const expected = new Report({ 72 | url: 'https://kumbier.it', 73 | name: 'KITS', 74 | preset: 'mobile-fast', 75 | date: '2018-08-03T09:10:00.013Z', 76 | path: join(dirFixture, subDir) 77 | }) 78 | 79 | expect(report.id).to.equal(expected.id) 80 | }) 81 | 82 | 83 | it('has a secure version of itself', function () { 84 | const report = Report.fromFile(join(dirFixture, subDir), configFileFixture) 85 | expect(report.withoutInternals()).to.not.haveOwnProperty('path') 86 | }) 87 | 88 | 89 | it('can delete itself', function() { 90 | const report = Report.fromFile(join(this._baseDir, subDir), configFileFixture) 91 | const configFile = join(this._baseDir, subDir, configFileFixture) 92 | 93 | expect(fs.existsSync(configFile)).to.be.true 94 | 95 | report.delete() 96 | 97 | expect(fs.existsSync(configFile)).to.be.false 98 | }) 99 | }) 100 | -------------------------------------------------------------------------------- /src/reports.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const config = require('../config/default') 6 | const debug = require('debug') 7 | const path = require('path') 8 | const { spawn } = require('child_process'); 9 | const fs = require('fs-extra') 10 | const Sqlite3 = require('better-sqlite3') 11 | 12 | const {Report} = require('./report') 13 | 14 | const dbInit = [ 15 | `CREATE TABLE IF NOT EXISTS reports ( 16 | id TEXT PRIMARY KEY, 17 | url TEXT NOT NULL, 18 | name TEXT NOT NULL, 19 | preset TEXT NOT NULL, 20 | date TEXT NOT NULL, 21 | path TEXT NOT NULL, 22 | lastseen TEXT 23 | )`, 24 | 'CREATE INDEX IF NOT EXISTS idx_url ON reports (url)', 25 | 'CREATE INDEX IF NOT EXISTS idx_url_and_preset ON reports (url, preset)', 26 | 'CREATE INDEX IF NOT EXISTS idx_url_and_preset_and_date ON reports (url, preset, date)' 27 | ] 28 | 29 | 30 | class ReportsCache { 31 | constructor(cachePath, readonly = false) { 32 | try { 33 | this._db = new Sqlite3(cachePath, {readonly}) 34 | } catch (e) { 35 | console.error(`ERROR: Could not open cache database at ${cachePath}`) 36 | console.error(e) 37 | process.exit(1) 38 | } 39 | 40 | // set the current time for lastseen on initialization 41 | this.updateCurrentLastseen() 42 | 43 | // this enables better concurrency while the file watcher syncs the database cache 44 | this._db.pragma('journal_mode=WAL') 45 | 46 | // Ensure database structure and indexes are set 47 | for (const stmt of dbInit) { 48 | this._db.prepare(stmt).run() 49 | } 50 | } 51 | 52 | updateCurrentLastseen(value = (new Date()).toISOString()) { 53 | const x = Date.parse(value) 54 | if (isNaN(x)) { 55 | throw new Error(`Not a valid Date: "${value}"`) 56 | } 57 | this._currentLastseen = (new Date(value)).toISOString() 58 | } 59 | 60 | upsert(report) { 61 | let values = { 62 | id: report.id.toString(), 63 | url: report.url.toString(), 64 | name: report.name.toString(), 65 | preset: report.preset.toString(), 66 | date: report.date.toString(), 67 | path: report.path.toString(), 68 | lastseen: this._currentLastseen 69 | } 70 | this._db.prepare('INSERT INTO reports VALUES ($id, $url, $name, $preset, $date, $path, $lastseen) ' + 71 | 'ON CONFLICT(id) DO UPDATE SET lastseen=$lastseen').run(values) 72 | } 73 | 74 | delete(report) { 75 | return this._db.prepare('DELETE FROM reports WHERE id = ?').run(report.id) 76 | } 77 | 78 | deleteByMeta(name, preset, created) { 79 | if (!name || !preset || !created) { 80 | throw new Error(`deleteByMeta requires three parameters - at least one is missing: (name: ${name}, preset: ${preset}, created: ${created})`) 81 | } 82 | return this._db.prepare('DELETE FROM reports WHERE name=? AND preset=? AND date=?').run(name, preset, created) 83 | } 84 | 85 | get(id) { 86 | let result = this._db.prepare('SELECT id, url, name, preset, date, path FROM reports WHERE id = ?').get(id) 87 | if (!result) 88 | return 89 | return new Report(result) 90 | } 91 | 92 | /** 93 | * INSECURE! An internal method, which accepts a where-clause as a string. ONLY use this method with static data. 94 | * Allowing unsecured input into the parameter results in an SQL injection vulnerability. 95 | * @param where 96 | * @returns Result[] 97 | * @private 98 | */ 99 | _all(where = '1=1') { 100 | let result = this._db.prepare(`SELECT id, url, name, preset, date, path FROM reports WHERE ${where}`).all() 101 | return result.map(row => new Report(row)) 102 | } 103 | 104 | all() { 105 | return this._all() 106 | } 107 | 108 | uniqueUrls() { 109 | return this._db.prepare('SELECT DISTINCT url FROM reports ORDER BY url').all().map(row => row.url) 110 | } 111 | 112 | presetsForUrl(url) { 113 | return this._db.prepare('SELECT DISTINCT preset FROM reports WHERE url=? ORDER BY preset').all(url).map(row => row.preset) 114 | } 115 | 116 | timesForUrlAndPreset(url, preset) { 117 | let dict = {} 118 | for (const row of this._db.prepare('SELECT date, id FROM reports WHERE url=? AND preset=? ORDER BY date DESC').all(url, preset)) { 119 | dict[row.date] = row.id 120 | } 121 | return dict 122 | } 123 | 124 | youngerThan(date) { 125 | return this._db.prepare('SELECT id, url, name, preset, date, path FROM reports WHERE date>?') 126 | .all(date).map(row => new Report(row)) 127 | } 128 | 129 | metadataForUrlAndPresetAndTime(url, preset, time) { 130 | return this._db.prepare('SELECT id, url, name, preset, date, path FROM reports WHERE url = ? AND preset = ? AND date = ?') 131 | .all(url, preset, time).map(row => new Report(row)) 132 | } 133 | 134 | outdated() { 135 | return this._all(`lastseen<'${this._currentLastseen}'`) 136 | } 137 | } 138 | 139 | 140 | class ReportsCacheDisabled { 141 | constructor() { 142 | } 143 | 144 | upsert() { 145 | } 146 | 147 | updateCurrentLastseen() { 148 | } 149 | 150 | delete() { 151 | } 152 | 153 | get() { 154 | } 155 | 156 | all() { 157 | } 158 | 159 | outdated() { 160 | } 161 | } 162 | 163 | 164 | class Reports { 165 | constructor(baseDir, setupWatcher = true, cache = null, shouldRestartCacheSync = true) { 166 | this._baseDir = baseDir 167 | fs.ensureDirSync(baseDir) 168 | this._reportsCache = cache === null ? new ReportsCache(path.join(config.cacheDir, 'lightmon-cache.sqlite3')) : cache 169 | if (setupWatcher) { 170 | this._setupCacheSync() 171 | } 172 | this._watcherRestarts = 0 173 | this._shouldRestartCacheSync = shouldRestartCacheSync 174 | } 175 | 176 | static async setup(baseDir, cache = null) { 177 | const reports = new Reports(baseDir, false, cache) 178 | await reports._setupCacheSync(baseDir) 179 | return reports 180 | } 181 | 182 | _setupCacheSync(dir = this._baseDir) { 183 | this._watcher = spawn('node', [path.join(__dirname, '..', 'bin', 'sync-cache'), '--report-dir', dir, '--verbose'], { 184 | detached: false, 185 | stdio: ['pipe', 'inherit', 'inherit'] 186 | }) 187 | //this._watcher.stdout.on('data', (data) => debug('LIGHTMON:SYNC:INFO')(`${data.trim()}`)) 188 | //this._watcher.stderr.on('data', (data) => debug('LIGHTMON:SYNC:ERROR')(`${data.trim()}`)) 189 | this._watcher.on('close', (code) => { 190 | debug('LIGHTMON:WARNING')(`Sync childprocess exited with code ${code}`) 191 | if (this._watcherRestarts > 3) { 192 | throw new Error('I would have to restart the sync process for more than 3 times, something is wrong. Exiting.') 193 | } 194 | if (this._shouldRestartCacheSync) { 195 | this._watcherRestarts++ 196 | debug('LIGHTMON:WARNING')('Restarting cache sync process...') 197 | this._setupCacheSync(dir) 198 | } else { 199 | debug('LIGHTMON:WARNING')('shouldRestartCacheSync is false - not restarting sync process.') 200 | } 201 | }) 202 | } 203 | 204 | single(id) { 205 | return this._reportsCache.get(id) 206 | } 207 | 208 | all() { 209 | return this._reportsCache.all() 210 | } 211 | 212 | uniqueUrls() { 213 | return this._reportsCache.uniqueUrls() 214 | } 215 | 216 | presetsForUrl(url) { 217 | return this._reportsCache.presetsForUrl(url) 218 | } 219 | 220 | timesForUrlAndPreset(url, preset) { 221 | return this._reportsCache.timesForUrlAndPreset(url, preset) 222 | } 223 | 224 | metadataForUrlAndPresetAndTime(url, preset, time) { 225 | return this._reportsCache.metadataForUrlAndPresetAndTime(url, preset, time) 226 | } 227 | 228 | /** 229 | * Removes disk location of reports to mitigate information leakage 230 | */ 231 | withoutInternals() { 232 | return this.all().map(r => r.withoutInternals()) 233 | } 234 | 235 | olderThan(date) { 236 | return this.all().filter(report => { 237 | return new Date(report.date) < date 238 | }) 239 | } 240 | 241 | youngerThan(date) { 242 | return this._reportsCache.youngerThan(date) 243 | } 244 | 245 | delete(report) { 246 | report.delete() 247 | this._reportsCache.delete(report) 248 | } 249 | 250 | get baseDir() { 251 | return this._baseDir 252 | } 253 | 254 | async destroy() { 255 | try { 256 | this._watcher.kill() 257 | // eslint-disable-next-line no-empty 258 | } catch (e) { 259 | console.error(e) 260 | } 261 | } 262 | } 263 | 264 | 265 | module.exports = { 266 | Reports, 267 | ReportsCache, 268 | ReportsCacheDisabled 269 | } -------------------------------------------------------------------------------- /src/reports.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const { Report } = require('./report') 6 | 7 | const fs = require('fs-extra') 8 | const { dirname, join, resolve } = require('path') 9 | const tmpdir = require('os').tmpdir 10 | const chai = require('chai') 11 | chai.use(require('chai-fs')) 12 | const expect = chai.expect 13 | 14 | const { Reports, ReportsCache } = require('./reports') 15 | 16 | const dirFixture = resolve(__dirname, './test-fixtures/reports/store') 17 | const anotherReportDir = join(dirFixture, '..', 'other', '2018-08-03T09:20:00.059Z') 18 | 19 | 20 | // FIXME: without a cache, the reports is unfilled at the beginning and should instead use 21 | // a truly non-cached version. 22 | xdescribe('Reports', function () { 23 | this.beforeEach(async function () { 24 | this._baseDir = fs.mkdtempSync(join(tmpdir(), 'mocha-')) 25 | fs.copySync(dirFixture, this._baseDir) 26 | this.reports = await Reports.setup(this._baseDir, false) 27 | }) 28 | 29 | 30 | this.afterEach(async () => { 31 | if (this.reports) { 32 | await this.reports.destroy() 33 | } 34 | try { 35 | fs.removeSync(this._baseDir) 36 | } catch (e) {} // eslint-disable-line no-empty 37 | }) 38 | 39 | 40 | it('can list the existing reports', async function () { 41 | expect(this.reports.all().length).to.equal(7) 42 | }) 43 | 44 | 45 | it('is notified on new reports', function (done) { 46 | Reports.setup(this._baseDir, false).then(reports => { 47 | const before = reports.all().length 48 | 49 | fs.copySync(dirname(anotherReportDir), this._baseDir) 50 | 51 | setTimeout(() => { 52 | expect(reports.all().length).to.equal(before + 4) 53 | done() 54 | }, 50) 55 | }) 56 | }) 57 | 58 | 59 | it('can get a report with a specific id', async function() { 60 | const validId = this.reports.all()[0].id 61 | const retrieved = this.reports.single(validId) 62 | expect(retrieved.id).to.equal(validId) 63 | }) 64 | 65 | 66 | it('returns results older than', async function() { 67 | let retrieved = this.reports.olderThan(new Date('2018-08-04')) 68 | expect(retrieved.length).to.equal(7) 69 | 70 | retrieved = this.reports.olderThan(new Date('2018-08-03')) 71 | expect(retrieved.length).to.equal(3) 72 | 73 | retrieved = this.reports.olderThan(new Date('2018-08-02')) 74 | expect(retrieved.length).to.equal(0) 75 | 76 | retrieved = this.reports.olderThan(new Date('2018-08-01')) 77 | expect(retrieved.length).to.equal(0) 78 | }) 79 | 80 | 81 | it('deletes report with all associated files', async function() { 82 | const before = this.reports.all().length 83 | const someReport = this.reports.all()[0] 84 | 85 | this.reports.delete(someReport) 86 | 87 | expect(before - 1).to.equal(this.reports.all().length) 88 | expect(this.reports.single(someReport.id)).to.be.undefined 89 | }) 90 | }) 91 | 92 | describe('ReportsCache', function() { 93 | beforeEach(() => { 94 | this._cache = new ReportsCache(':memory:') 95 | this._someReport = new Report({ url: 'https://kumbier.it', name: 'KITS', preset: 'desktop-fast', date: '2020-08-28T15:56:12Z', path: '/path/2020-08-28T15_56_12Z' }) 96 | }) 97 | 98 | it('inserts and retrieves a report', () => { 99 | this._cache.upsert(this._someReport) 100 | let retrievedReport = this._cache.get(this._someReport.id) 101 | expect(this._someReport.id).to.equal(retrievedReport.id) 102 | }) 103 | 104 | it('deletes a report', () => { 105 | this._cache.upsert(this._someReport) 106 | let retrievedReport = this._cache.get(this._someReport.id) 107 | expect(retrievedReport).not.to.be.undefined 108 | 109 | this._cache.delete(this._someReport) 110 | 111 | expect(this._cache.get(this._someReport.id)).to.be.undefined 112 | }) 113 | 114 | it('ignores already existing reports', () => { 115 | this._cache.upsert(this._someReport) 116 | this._cache.upsert(this._someReport) 117 | 118 | let retrievedReports = this._cache.all() 119 | 120 | expect(retrievedReports.length).to.equal(1) 121 | }) 122 | 123 | it('retrieves all reports', () => { 124 | this._cache.upsert(this._someReport) 125 | const anotherReport = new Report({ url: 'https://kumbier.it', name: 'KITS', preset: 'desktop-fast', date: '2020-08-28T16:01:00Z', path: '/path/2020-08-28T16_01_00Z' }) 126 | this._cache.upsert(anotherReport) 127 | 128 | let retrievedIds = this._cache.all().map(report => report.id).sort() 129 | 130 | expect(retrievedIds).to.deep.equal([anotherReport.id, this._someReport.id].sort()) 131 | }) 132 | 133 | it('retrieves outdated reports', () => { 134 | const anotherReport = new Report({ url: 'https://kumbier.it', name: 'KITS', preset: 'desktop-fast', date: '2020-08-28T16:01:00Z', path: '/path/2020-08-28T16_01_00Z' }) 135 | this._cache.updateCurrentLastseen('2020-09-01T00:00:00.000Z') 136 | this._cache.upsert(this._someReport) 137 | this._cache.updateCurrentLastseen('2020-09-01T01:00:00.000Z') 138 | this._cache.upsert(anotherReport) 139 | 140 | let retrievedIds = this._cache.outdated().map(report => report.id) 141 | 142 | expect(retrievedIds).to.deep.equal([this._someReport.id]) 143 | }) 144 | }) -------------------------------------------------------------------------------- /src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_desktop-fast_config.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_desktop-fast_config.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_desktop-fast_report.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_desktop-fast_report.html.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_desktop-fast_report.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_desktop-fast_report.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_desktop-slow_config.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_desktop-slow_config.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_desktop-slow_report.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_desktop-slow_report.html.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_desktop-slow_report.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_desktop-slow_report.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_mobile-fast_config.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_mobile-fast_config.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_mobile-fast_report.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_mobile-fast_report.html.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_mobile-fast_report.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_mobile-fast_report.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_mobile-slow_config.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_mobile-slow_config.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_mobile-slow_report.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_mobile-slow_report.html.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_mobile-slow_report.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/other/2018-08-03T09_20_00.059Z/KITS_mobile-slow_report.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-02T00_00_00.034Z/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-02T00_00_00.034Z/.gitkeep -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_desktop-slow_config.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_desktop-slow_config.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_desktop-slow_report.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_desktop-slow_report.html.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_desktop-slow_report.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_desktop-slow_report.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_mobile-fast_config.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_mobile-fast_config.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_mobile-fast_report.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_mobile-fast_report.html.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_mobile-fast_report.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_mobile-fast_report.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_mobile-slow_config.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_mobile-slow_config.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_mobile-slow_report.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_mobile-slow_report.html.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_mobile-slow_report.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-02T14_10_57.975Z/KITS_mobile-slow_report.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_desktop-fast_config.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_desktop-fast_config.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_desktop-fast_report.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_desktop-fast_report.html.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_desktop-fast_report.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_desktop-fast_report.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_desktop-slow_config.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_desktop-slow_config.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_desktop-slow_report.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_desktop-slow_report.html.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_desktop-slow_report.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_desktop-slow_report.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_mobile-fast_config.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_mobile-fast_config.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_mobile-fast_report.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_mobile-fast_report.html.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_mobile-fast_report.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_mobile-fast_report.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_mobile-slow_config.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_mobile-slow_config.json.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_mobile-slow_report.html.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_mobile-slow_report.html.gz -------------------------------------------------------------------------------- /src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_mobile-slow_report.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Verivox/lighthouse-monitor/4155437668f282d6b64615aaa412c19b0b25d09a/src/test-fixtures/reports/store/2018-08-03T09_10_00.013Z/KITS_mobile-slow_report.json.gz -------------------------------------------------------------------------------- /src/webserver/healthz.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const express = require('express') 6 | 7 | 8 | class Healthz { 9 | constructor({ Webserver, reports, expectedLastReportInSec }) { 10 | this.router = new express.Router() 11 | Webserver.middleware.add(this) 12 | this._expectedLast = !expectedLastReportInSec ? null : expectedLastReportInSec * 1000 13 | this._reports = reports 14 | } 15 | 16 | middleware() { 17 | this.router.route('/healthz').get(this._check.bind(this)) 18 | return this.router 19 | } 20 | 21 | _check(req, res) { 22 | if (!this._expectedLast) { 23 | return res.status(400).send(JSON.stringify({'status':'expectedLastReportInSec not configured, no status available'}, null, 2)) 24 | } 25 | const from = (new Date(Date.now() - this._expectedLast)).toISOString() 26 | const results = this._reports.youngerThan(from) 27 | if (results.length > 0) { 28 | return res.send(JSON.stringify({'status':'ok'}, null, 2)) 29 | } 30 | return res.status(500).send(JSON.stringify({'status':'missing reports'}, null, 2)) 31 | } 32 | } 33 | 34 | 35 | module.exports = { 36 | Healthz 37 | } 38 | -------------------------------------------------------------------------------- /src/webserver/html-report.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const config = require('../../config/default') 6 | const express = require('express') 7 | const { Reports } = require('../reports') 8 | const mime = require('mime/lite') 9 | 10 | 11 | class HtmlReport { 12 | constructor({ Webserver }) { 13 | this.router = new express.Router() 14 | this.reports = new Reports(config.reportDir) 15 | Webserver.middleware.add(this) 16 | } 17 | 18 | middleware() { 19 | this.router.route('/report').get(this._all.bind(this)) 20 | this.router.route('/report/url').get(this._url.bind(this)) 21 | this.router.route('/report/url/:url/').get(this._presetByUrl.bind(this)) 22 | this.router.route('/report/url/:url/:preset/').get(this._reportByUrlAndPreset.bind(this)) 23 | this.router.route('/report/url/:url/:preset/:timestamp').get(this._reportByUrlPresetTimestamp.bind(this)) 24 | this.router.route('/report/:id').get(this._get.bind(this)) 25 | this.router.route('/report/:id/artifacts').get(this._getArtifact.bind(this)) 26 | this.router.route('/report/:id/html').get(this._getHtml.bind(this)) 27 | this.router.route('/report/:id/download').get(this._download.bind(this)) 28 | this.router.route('/report/:id/json').get(this._getJson.bind(this)) 29 | this.router.route('/report/:id/config').get(this._getConfig.bind(this)) 30 | return this.router 31 | } 32 | 33 | _all(req, res) { 34 | const reportMonsterObj = {} 35 | for (let report of this.reports.withoutInternals()) { 36 | reportMonsterObj[report.id] = report 37 | } 38 | const json = JSON.stringify(reportMonsterObj, null, 2) 39 | return res.send(json) 40 | } 41 | 42 | _url(req, res) { 43 | return res.send( 44 | this.reports.uniqueUrls() 45 | ) 46 | } 47 | 48 | _presetByUrl(req, res) { 49 | return res.send( 50 | this.reports.presetsForUrl(req.params.url) 51 | ) 52 | } 53 | 54 | _reportByUrlAndPreset(req, res) { 55 | return res.send( 56 | this.reports.timesForUrlAndPreset(req.params.url, req.params.preset) 57 | ) 58 | } 59 | 60 | _reportByUrlPresetTimestamp(req, res) { 61 | return res.send( 62 | this.reports.metadataForUrlAndPresetAndTime(req.params.url, req.params.preset, req.params.timestamp) 63 | .map(report => report.withoutInternals())[0] 64 | ) 65 | } 66 | 67 | _get(req, res) { 68 | const id = req.params.id 69 | const report = this.reports.single(id) 70 | 71 | return (!report) ? 72 | res.status(404).send('Resource not found') : 73 | res.type('json') 74 | .send(report.withoutInternals()) 75 | } 76 | 77 | _getHtml(req, res) { 78 | return this.__getFile(req, res, 'html') 79 | } 80 | 81 | _download(req, res) { 82 | res.set('Content-disposition', 'attachment; filename=report.html') 83 | return this.__getFile(req, res, 'html') 84 | } 85 | 86 | _getArtifact(req, res) { 87 | res.set('Content-disposition', 'attachment; filename=artifact.json') 88 | return this.__getFile(req, res, 'artifacts') 89 | } 90 | 91 | _getJson(req, res) { 92 | res.set('Content-disposition', 'attachment; filename=report.json') 93 | return this.__getFile(req, res, 'json') 94 | } 95 | 96 | _getConfig(req, res) { 97 | res.set('Content-disposition', 'attachment; filename=config.json') 98 | return this.__getFile(req, res, 'config') 99 | } 100 | 101 | __getFile(req, res, type) { 102 | const id = req.params.id 103 | const report = this.reports.single(id) 104 | 105 | if (!report || !report[type]) { 106 | return res.status(404).send('Resource not found') 107 | } 108 | 109 | if (report[type].endsWith('.gz')) { 110 | res.set('Content-Encoding', 'gzip') 111 | res.set('Content-Type', this._getType(report[type])) 112 | } 113 | 114 | return res.sendFile(report[type]) 115 | } 116 | 117 | _getType(filename) { 118 | const fileWithoutGz = filename.endsWith('.gz') ? filename.substr(0, filename.length - 3) : filename 119 | const ext = fileWithoutGz.substr(fileWithoutGz.lastIndexOf('.') + 1) 120 | return mime.getType(ext) 121 | } 122 | } 123 | 124 | 125 | module.exports = { 126 | HtmlReport 127 | } 128 | -------------------------------------------------------------------------------- /src/webserver/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const { Healthz } = require('./healthz') 6 | const { HtmlReport } = require('./html-report') 7 | const { Metrics } = require('./metrics') 8 | const { StaticFiles } = require('./static-files') 9 | const { Webserver } = require('./webserver') 10 | 11 | module.exports = { 12 | Healthz, 13 | HtmlReport, 14 | Metrics, 15 | StaticFiles, 16 | Webserver, 17 | } -------------------------------------------------------------------------------- /src/webserver/metrics.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const express = require('express') 6 | const path = require('path') 7 | 8 | 9 | class Metrics { 10 | constructor({ Webserver, prometheusMetricsFile, jsonMetricsFile }) { 11 | this._prometheusFile = prometheusMetricsFile 12 | this._jsonFile = jsonMetricsFile 13 | this.router = new express.Router() 14 | 15 | this.router.route('/metrics/').get(this._prometheusFormat.bind(this)) 16 | this.router.route('/metrics.json').get(this._jsonFormat.bind(this)) 17 | 18 | Webserver.middleware.add(this) 19 | } 20 | 21 | middleware() { 22 | return this.router 23 | } 24 | 25 | _prometheusFormat(req, res) { 26 | const metrics = path.join(__dirname, '..', '..', this._prometheusFile) 27 | res.sendFile(metrics) 28 | } 29 | 30 | _jsonFormat(req, res) { 31 | res.sendFile(path.join(__dirname, '..', '..', this._jsonFile)) 32 | } 33 | } 34 | 35 | 36 | module.exports = { 37 | Metrics 38 | } -------------------------------------------------------------------------------- /src/webserver/static-files.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const path = require('path') 6 | const express = require('express') 7 | 8 | 9 | class StaticFiles { 10 | constructor({ Webserver, publicFolder }) { 11 | this.publicPath = path.resolve(path.join(__dirname, '..', '..', publicFolder)) 12 | Webserver.middleware.add(this) 13 | } 14 | 15 | middleware() { 16 | return express.static(this.publicPath) 17 | } 18 | } 19 | 20 | 21 | module.exports = { 22 | StaticFiles, 23 | } 24 | -------------------------------------------------------------------------------- /src/webserver/webserver.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Part of Lightmon: https://github.com/verivox/lightmon 3 | * Licensed under MIT from the Verivox GmbH 4 | */ 5 | const debug = require('debug') 6 | const info = debug('LIGHTMON:INFO') 7 | const express = require('express') 8 | 9 | 10 | class Webserver { 11 | constructor({ port = 3000 }) { 12 | this._app = express() 13 | this._port = port 14 | } 15 | 16 | registerList(middlewares) { 17 | for (const middle of middlewares) { 18 | this.register(middle) 19 | } 20 | } 21 | 22 | register(mid) { 23 | if (typeof mid === 'function') { 24 | this._app.use(mid) 25 | } else { 26 | this._app.use(mid.middleware().bind(mid)) 27 | } 28 | } 29 | 30 | start() { 31 | this.registerList(Webserver.middleware) 32 | this._app.listen(this._port, this._onStartup.bind(this)) 33 | } 34 | 35 | _onStartup() { 36 | info(`Listening on Port: ${this._port}`) 37 | } 38 | 39 | run() { 40 | info('Starting Webserver') 41 | this.start() 42 | } 43 | } 44 | 45 | Webserver.middleware = new Set() 46 | 47 | 48 | module.exports = { 49 | default: Webserver, 50 | Webserver, 51 | } 52 | -------------------------------------------------------------------------------- /tools/prometheus/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM prom/prometheus 2 | 3 | COPY entrypoint.sh /entrypoint.sh 4 | 5 | USER root 6 | 7 | ENTRYPOINT [ "/entrypoint.sh" ] -------------------------------------------------------------------------------- /tools/prometheus/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # shellcheck disable=SC2039 3 | 4 | set -e 5 | 6 | if ! grep "docker.host.internal" /etc/hosts 7 | then 8 | echo -e "$(/bin/ip route|awk '/default/ { print $3 }')\tdocker.host.internal" >> /etc/hosts 9 | fi 10 | 11 | /bin/prometheus \ 12 | --config.file=/etc/prometheus/prometheus.yml \ 13 | --storage.tsdb.path=/prometheus \ 14 | --web.console.libraries=/etc/prometheus/console_libraries \ 15 | --web.console.templates=/etc/prometheus/consoles \ 16 | --log.level=debug -------------------------------------------------------------------------------- /tools/prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 5s 3 | evaluation_interval: 5s 4 | 5 | ## A scrape configuration containing exactly one endpoint to scrape: 6 | ## Here it's Prometheus itself. 7 | scrape_configs: 8 | - job_name: 'prometheus' 9 | static_configs: 10 | - targets: ['localhost:9090'] 11 | 12 | ## This will scrape metrics file on the dock host port 3000 13 | - job_name: 'lightmon' 14 | static_configs: 15 | - targets: ['docker.host.internal:3000'] 16 | 17 | ## if you use the push gateway instead, this is how to import it 18 | - job_name: 'pushgw' 19 | static_configs: 20 | - targets: ['docker.host.internal:9091'] 21 | honor_labels: true -------------------------------------------------------------------------------- /tools/prometheus/start-docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" 4 | 5 | pushd "$THIS_DIR" > /dev/null 6 | 7 | set -e 8 | # set -x 9 | 10 | docker build -t local/prometheus:local . 11 | 12 | docker run \ 13 | -v "$THIS_DIR/prometheus.yml:/etc/prometheus/prometheus.yml" \ 14 | -v "$THIS_DIR/reports:/reports" \ 15 | -p 9090:9090 \ 16 | --name prometheus \ 17 | --rm \ 18 | -it \ 19 | local/prometheus:local 20 | 21 | popd > /dev/null --------------------------------------------------------------------------------