├── .dockerignore ├── .github └── workflows │ ├── push.yml │ └── test.yml ├── .gitignore ├── .nvmrc ├── Dockerfile ├── LICENSE ├── README.md ├── architecture.svg ├── docker-compose.yml ├── ecosystem.config.js ├── entrypoint.sh ├── package.json ├── readme.png ├── src ├── api │ └── index.js ├── constants │ └── index.js ├── index.js ├── routes │ └── index.js ├── static │ ├── 404.html │ ├── img │ │ ├── lighthouse-404.png │ │ └── lighthouse-logo.png │ ├── reportTemplate.html │ └── statsTemplate.html ├── store │ ├── audit.js │ ├── index.js │ └── schedule.js └── utils │ ├── common.js │ ├── index.js │ ├── lighthouse.js │ ├── responseBuilder.js │ └── schedule.js └── yarn.lock /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | architecture.svg 3 | ecosystem.config.js -------------------------------------------------------------------------------- /.github/workflows/push.yml: -------------------------------------------------------------------------------- 1 | name: Push Image 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | jobs: 8 | build_image: 9 | name: Build Lighthouse Image 10 | runs-on: ubuntu-18.04 11 | 12 | steps: 13 | - uses: actions/checkout@v2 14 | with: 15 | ref: ${{ github.ref }} 16 | 17 | - name: Login to GitHub Container Registry 18 | uses: docker/login-action@v1 19 | with: 20 | registry: ghcr.io 21 | username: ${{ github.repository_owner }} 22 | password: ${{ secrets.PERSONAL_ACCESS_TOKEN }} 23 | 24 | - name: Build image 25 | run: DOCKER_BUILDKIT=1 docker build -t ghcr.io/wanteddev/lighthouse:${{github.event.release.tag_name}} --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from ghcr.io/wanteddev/lighthouse:latest . 26 | 27 | - name: Push image 28 | run: docker push ghcr.io/wanteddev/lighthouse:${{github.event.release.tag_name}} 29 | 30 | - name: Update latest image (to be used in caching) 31 | run: | 32 | docker tag ghcr.io/wanteddev/lighthouse:${{github.event.release.tag_name}} ghcr.io/wanteddev/lighthouse:latest 33 | docker push ghcr.io/wanteddev/lighthouse:latest 34 | if: "github.event.release.prerelease != true" 35 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test Audit 2 | on: workflow_dispatch 3 | 4 | jobs: 5 | audit: 6 | runs-on: ubuntu-latest 7 | steps: 8 | - uses: actions/checkout@master 9 | - name: Create Report Wanted Job List 10 | uses: ./ 11 | with: 12 | url: https://www.wanted.co.kr/wdlist?country=kr&job_sort=job.latest_order&years=-1&locations=seoul 13 | desktop: 'false' 14 | mobile: 'true' 15 | 16 | - name: Create Report Wanted Job Detail 17 | uses: ./ 18 | with: 19 | url: 'https://www.wanted.co.kr/wd/32385' 20 | desktop: 'true' 21 | mobile: 'true' 22 | 23 | - uses: actions/upload-artifact@master 24 | with: 25 | name: report 26 | path: './report' 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | *.http 3 | .env 4 | *.log 5 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | v15.9.0 -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:15.9.0-slim 2 | 3 | RUN apt-get update \ 4 | # Install latest chrome dev package, which installs the necessary libs to 5 | # make the bundled version of Chromium that Puppeteer installs work. 6 | && apt-get install -y gnupg2 \ 7 | && apt-get install -y wget \ 8 | && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ 9 | && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ 10 | && apt-get update \ 11 | && apt-get install -y google-chrome-unstable \ 12 | && rm -rf /var/lib/apt/lists/* \ 13 | && wget --quiet https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh -O /usr/sbin/wait-for-it.sh \ 14 | && chmod +x /usr/sbin/wait-for-it.sh 15 | 16 | WORKDIR /home/app 17 | 18 | COPY ./package.json /home/app/package.json 19 | COPY ./yarn.lock /home/app/yarn.lock 20 | 21 | RUN yarn --frozen-lockfile 22 | RUN yarn global add pm2 23 | 24 | COPY . /home/app 25 | 26 | CMD [ "node", "src/index.js" ] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Lighthouse Auditing Bot 2 | 3 | ## Summary 4 | This project creates a containerized auditing environment for Google's [Lighthouse](https://developers.google.com/web/tools/lighthouse) tool. 5 | Bringing all the benefits of running your tests in a stable environment without the overhead of updating CI/CD pipelines and code 6 | 7 | ![](/readme.png) 8 | 9 | ## Features 10 | * __Ad-hoc Auditing__ - Quickly run an audit of a website with the `/lighthouse {url}` command, or simply type `/lighthouse` to launch a dialog with all available options 11 | * __Job Scheduling__ - With the `/lighthouse jobs` command, you can schedule an auditing job to be run whenever necessary 12 | * __Customizeable HTML Reports__ - Always be able to view the full detailed report from Lighthouse as an HTML file, which is provided by a template in this project, and customize parts of the template (in `src/static/reportTemplate.html`) to your heart's content! 13 | * __Trend Charts__ - Track changes in each of the audit categories over time for a given URL by running the `/lighthouse stats {url}` command and clicking the link to an intuitive dashboard (also provided as an HTML template that can be customized in `src/static/statsTemplate.html`) 14 | 15 | ## Development 16 | ### Pre-requisites & Notes 17 | 18 | * **Node.js v12+** 19 | 20 | Regardless of the method you are deploying with, this application relies on a variety of environment variables to be able to function properly. Either use the `export` method, or inject your docker container with env variables depending on what method you are deploying this chatbot with. 21 | 22 | | Variable Name | Example Value | Explanation | 23 | | ----------------: | :------------------------- | :---------------------------------- | 24 | | PORT | 3001 | The port being used by this chatbot | 25 | | TOKEN | xoxb-921212312-125361390560-xxxxxxxxxx | The OAuth token value received after installing the Slack App | 26 | | MONGO_USERNAME | root | Auth username for a mongodb server | 27 | | MONGO_PASSWORD | test_passwd | Auth password for a mongodb server | 28 | | MONGO_SERVER | 192.168.1.10:27017 | The endpoint for a mongodb server | 29 | | CHATBOT_SERVER | http://192.168.1.10:3001 | IP to be used by this chatbot (needed to set URL endpoints in Message Attachments) | 30 | | TZ (optional) | Asia/Seoul | The timezone value that will be used on server (important for job scheduling) | 31 | 32 | 33 | ### Developing with Docker 34 | **0. Build local Lighthouse bot image** 35 | ``` 36 | docker build -t wanteddev/lighthouse-bot . 37 | ``` 38 | 39 | **1. Run mongodb (as a separate container)** 40 | ``` 41 | docker run -d -p 27017:27017 -e MONGO_INITDB_ROOT_USERNAME="root" -e MONGO_INITDB_ROOT_PASSWORD="test_passwd" --name lighthouse-mongo mongo:latest 42 | ``` 43 | 44 | **2. Run Lighthouse bot container** 45 | Note: Ensure you have the environment variables set when running the container 46 | 47 | Example `run` command: 48 | ``` 49 | docker run -d -p 3001:3001 -v $PWD/src:/home/app/src -e TZ="Asia/Seoul" -e PORT=3001 -e TOKEN="xoxb-921212312-125361390560-xxxxxxxxxx" -e MONGO_USERNAME="root" -e MONGO_PASSWORD="test_passwd" -e MONGO_SERVER="192.168.1.129:27017" -e CHATBOT_SERVER="http://192.168.1.129:3001" --name lighthouse-bot wanteddev/lighthouse-bot 50 | ``` 51 | 52 | 53 | ### Developing with PM2 54 | **0. Follow the [installation guide for MongoDB](https://docs.mongodb.com/manual/installation/) to set up your MongoDB instance** 55 | 56 | **1. Set values for all required environment variables** 57 | ``` 58 | export PORT=3001 59 | export TOKEN=xoxb-921212312-125361390560-xxxxxxxxxx 60 | export MONGO_USERNAME=root 61 | export MONGO_PASSWORD=test_passwd 62 | export MONGO_SERVER=192.168.1.10:27017 63 | export CHATBOT_SERVER=http://192.168.1.10:3001 64 | export TZ=Asia/Seoul 65 | ``` 66 | 67 | **2. Globally install [PM2](https://pm2.keymetrics.io)** 68 | 69 | ``` 70 | yarn global add pm2 71 | ``` 72 | 73 | **3. Install dependencies** 74 | ``` 75 | yarn 76 | ``` 77 | 78 | **4. Run chatbot with pm2** 79 | ``` 80 | pm2 start ecosystem.config.js 81 | ``` 82 | **5. [Register a slash command](https://docs.mattermost.com/developer/slash-commands.html#custom-slash-command) in Mattermost that sends a `GET` request to the `/lighthouse` endpoint** 83 | ![](documentation/img/lighthouse-slashcmd.png) 84 | 85 | ## Deployment 86 | Deploying this chatbot is done in the same way as the [development environment setup](#development), with the exception that you would set the `NODE_ENV` variable to `production`, as well as not do any volume binding to the host when running the chatbot with Docker. 87 | 88 | ## Slack App registration 89 | As we're open-sourcing this project, the responsibility to deploy and manage each application and data stored in it falls upon whoever ultimately deploys it. 90 | We're sharing the configurations used in our private Slack App so that others may reference it and apply it to their own auditing environments! 91 | 92 | #### Interactive Components 93 | In order to use the inputs and options inside of the custom auditing functionality, you need to activate the "Interactive Components" functionality, and set the Request URL value as below: 94 | **Request URL**: `https://${chatbotUrl}/receive_submission` 95 | #### Slash Commands 96 | As the main method of accessing the chatbot server, we have registered a Slash command as follows: 97 | * Command: `/lighthouse` 98 | * Request URL: `https://${chatbotUrl}/lighthouse` 99 | * Short Description: `Use Lighthouse Audit Commands` 100 | * Usage Hint: `help` 101 | 102 | #### Bots 103 | Some of the settings we've activated for our chatbot include: 104 | * `Always Show My Bot as Online` 105 | * `Show Tabs` > `Messages Tab` (So that you can run audits in a DM) 106 | 107 | #### Permissions 108 | In order to post messages, this chatbot requires the following 3 permissions: 109 | * `chat:write`: Basic messaging permissions 110 | * `chat:write.public`: Ability to post messages in channels the bot is not a member of 111 | * `commands`: Ability to add slash commands that people can use 112 | 113 | ## Troubleshooting 114 | * The audit command fails with an `error while loading shared libraries: libX11-xcb.so.1: cannot open shared object file: No such file or directory` error on Ubuntu 115 | * Run the command below to install dependencies needed to launch Puppeteer from your host 116 | 117 | ``` 118 | sudo apt install -y gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget libgbm-dev 119 | ``` 120 | 121 | ### Current Tasks 122 | - [x] Implementing trend charts for audit results to be accessed with `/lighthouse stats {url}` 123 | - [x] Styling audit trends dashboard 124 | - [x] Add usernames to schedule schemas so that they can be easily viewed through `/lighthouse schedule list` 125 | - [x] Implementing `/lighthouse schedule info {id}` to get full details of a given job 126 | - [ ] Add unit testing with Jest or AVA 127 | - [ ] Write documentation to make command usage clearer 128 | - [ ] Add more comprehensive logging 129 | - [x] Investigate the possibility of using workers to run audits so that multiple audits can run simultaneously 130 | - [x] Add a configurable number of past audits to be fetched from `stats` command 131 | - e.g. `/lighthouse stats https://google.com limit 15` 132 | 133 | ## Blog 134 | - [로컬에서 lighthouse 테스트 하는 방법 (한국어)](https://medium.com/wantedjobs/wanteddev-lighthouse-%EC%89%AC%EC%9A%B4-%EC%82%AC%EC%9A%A9%EB%B2%95-feat-pm2-mongo-db-cc07d9d3b520) 135 | -------------------------------------------------------------------------------- /architecture.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 9 | 10 | dependency-cruiser output 11 | 12 | 13 | cluster_src 14 | 15 | src 16 | 17 | 18 | cluster_src/api 19 | 20 | api 21 | 22 | 23 | cluster_src/constants 24 | 25 | constants 26 | 27 | 28 | cluster_src/routes 29 | 30 | routes 31 | 32 | 33 | cluster_src/store 34 | 35 | store 36 | 37 | 38 | cluster_src/utils 39 | 40 | utils 41 | 42 | 43 | 44 | fs 45 | 46 | fs 47 | 48 | 49 | 50 | path 51 | 52 | path 53 | 54 | 55 | 56 | src/api/index.js 57 | 58 | 59 | index.js 60 | 61 | 62 | 63 | 64 | 65 | src/constants/index.js 66 | 67 | 68 | index.js 69 | 70 | 71 | 72 | 73 | 74 | src/api/index.js->src/constants/index.js 75 | 76 | 77 | 78 | 79 | 80 | src/utils/index.js 81 | 82 | 83 | index.js 84 | 85 | 86 | 87 | 88 | 89 | src/api/index.js->src/utils/index.js 90 | 91 | 92 | 93 | 94 | 95 | src/utils/common.js 96 | 97 | 98 | common.js 99 | 100 | 101 | 102 | 103 | 104 | src/utils/index.js->src/utils/common.js 105 | 106 | 107 | 108 | 109 | 110 | src/utils/lighthouse.js 111 | 112 | 113 | lighthouse.js 114 | 115 | 116 | 117 | 118 | 119 | src/utils/index.js->src/utils/lighthouse.js 120 | 121 | 122 | 123 | 124 | 125 | src/utils/responseBuilder.js 126 | 127 | 128 | responseBuilder.js 129 | 130 | 131 | 132 | 133 | 134 | src/utils/index.js->src/utils/responseBuilder.js 135 | 136 | 137 | 138 | 139 | 140 | src/utils/schedule.js 141 | 142 | 143 | schedule.js 144 | 145 | 146 | 147 | 148 | 149 | src/utils/index.js->src/utils/schedule.js 150 | 151 | 152 | 153 | 154 | 155 | src/index.js 156 | 157 | 158 | index.js 159 | 160 | 161 | 162 | 163 | 164 | src/index.js->path 165 | 166 | 167 | 168 | 169 | 170 | src/index.js->src/constants/index.js 171 | 172 | 173 | 174 | 175 | 176 | src/index.js->src/utils/index.js 177 | 178 | 179 | 180 | 181 | 182 | src/routes/index.js 183 | 184 | 185 | index.js 186 | 187 | 188 | 189 | 190 | 191 | src/index.js->src/routes/index.js 192 | 193 | 194 | 195 | 196 | 197 | src/store/index.js 198 | 199 | 200 | index.js 201 | 202 | 203 | 204 | 205 | 206 | src/index.js->src/store/index.js 207 | 208 | 209 | 210 | 211 | 212 | src/routes/index.js->fs 213 | 214 | 215 | 216 | 217 | 218 | src/routes/index.js->src/api/index.js 219 | 220 | 221 | 222 | 223 | 224 | src/routes/index.js->src/constants/index.js 225 | 226 | 227 | 228 | 229 | 230 | src/routes/index.js->src/utils/index.js 231 | 232 | 233 | 234 | 235 | 236 | src/routes/index.js->src/store/index.js 237 | 238 | 239 | 240 | 241 | 242 | src/store/index.js->src/constants/index.js 243 | 244 | 245 | 246 | 247 | 248 | src/store/index.js->src/utils/index.js 249 | 250 | 251 | 252 | 253 | 254 | src/store/audit.js 255 | 256 | 257 | audit.js 258 | 259 | 260 | 261 | 262 | 263 | src/store/index.js->src/store/audit.js 264 | 265 | 266 | 267 | 268 | 269 | src/store/schedule.js 270 | 271 | 272 | schedule.js 273 | 274 | 275 | 276 | 277 | 278 | src/store/index.js->src/store/schedule.js 279 | 280 | 281 | 282 | 283 | 284 | src/store/audit.js->src/utils/index.js 285 | 286 | 287 | 288 | 289 | 290 | src/store/schedule.js->src/utils/index.js 291 | 292 | 293 | 294 | 295 | 296 | src/utils/lighthouse.js->fs 297 | 298 | 299 | 300 | 301 | 302 | src/utils/lighthouse.js->src/utils/common.js 303 | 304 | 305 | 306 | 307 | 308 | worker_threads 309 | 310 | worker_threads 311 | 312 | 313 | 314 | src/utils/lighthouse.js->worker_threads 315 | 316 | 317 | 318 | 319 | 320 | src/utils/responseBuilder.js->src/utils/common.js 321 | 322 | 323 | 324 | 325 | 326 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | mongo: 5 | env_file: .env 6 | networks: 7 | - app 8 | image: mongo 9 | environment: 10 | - MONGO_INITDB_ROOT_USERNAME=${MONGO_ROOT_USER} 11 | - MONGO_INITDB_ROOT_PASSWORD=${MONGO_ROOT_PASSWORD} 12 | - MONGO_INITDB_DATABASE=lighthouse 13 | lighthouse-bot: 14 | build: . 15 | environment: 16 | - PORT=3001 17 | - WEBSOCKET_URL=${WEBSOCKET_URL} 18 | - TOKEN=${TOKEN} 19 | - MONGO_USERNAME=${MONGO_ROOT_USER} 20 | - MONGO_PASSWORD=${MONGO_ROOT_PASSWORD} 21 | - MONGO_SERVER=${MONGO_SERVER} 22 | - CHATBOT_SERVER=${CHATBOT_SERVER} 23 | - TZ=Asia/Seoul 24 | networks: 25 | - app 26 | env_file: .env 27 | depends_on: 28 | - mongo 29 | ports: 30 | - "3001:3001" 31 | 32 | volumes: 33 | my-db: 34 | 35 | networks: 36 | app: 37 | -------------------------------------------------------------------------------- /ecosystem.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | module.exports = { 4 | apps: [{ 5 | name: 'lighthouse-bot', 6 | script: 'src/index.js', 7 | instances: 1, 8 | autorestart: true, 9 | watch: process.env.NODE_ENV !== 'production' ? path.resolve(__dirname) : false, 10 | node_args: ['--experimental-worker'], 11 | max_memory_restart: '1G', 12 | env: { 13 | PORT: 3001, 14 | TOKEN: 'xoxb-393798891191-dsadasdas-xxxxxxxxxxxxx', 15 | MONGO_USERNAME: 'root', 16 | MONGO_PASSWORD: 'test_passwd', 17 | MONGO_SERVER: '127.0.0.1:27017', 18 | CHATBOT_SERVER: 'https://localhost:3001', 19 | TZ: 'Asia/Seoul' 20 | } 21 | }] 22 | }; 23 | -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | createReport() { 5 | # Check if we're being triggered by a pull request. 6 | PULL_REQUEST_NUMBER=$(jq .number "$GITHUB_EVENT_PATH") 7 | 8 | # check platform 9 | DEVICE=$1 10 | 11 | # simply check the provided live URL. 12 | REPORT_URL=$INPUT_URL 13 | 14 | # Prepare directory for audit results and sanitize URL to a valid and unique filename. 15 | OUTPUT_FOLDER="report/${DEVICE}" 16 | # shellcheck disable=SC2001 17 | OUTPUT_FILENAME=$(echo "$REPORT_URL" | sed 's/[^a-zA-Z0-9]/_/g') 18 | OUTPUT_PATH="$GITHUB_WORKSPACE/$OUTPUT_FOLDER/$OUTPUT_FILENAME" 19 | mkdir -p "$OUTPUT_FOLDER" 20 | 21 | # Clarify in logs which URL we're auditing. 22 | printf "* Beginning audit of %s ...\n\n" "$REPORT_URL" 23 | 24 | # Run Lighthouse! 25 | lighthouse "${REPORT_URL}" --port=9222 --chrome-flags="--headless --disable-gpu --no-sandbox --no-zygote" --emulated-form-factor "${DEVICE}" --output "html" --output "json" --output-path "${OUTPUT_PATH}" 26 | 27 | # Parse individual scores from JSON output. 28 | # Unorthodox jq syntax because of dashes -- https://github.com/stedolan/jq/issues/38 29 | SCORE_PERFORMANCE=$(jq '.categories["performance"].score' "$OUTPUT_PATH".report.json) 30 | SCORE_ACCESSIBILITY=$(jq '.categories["accessibility"].score' "$OUTPUT_PATH".report.json) 31 | SCORE_PRACTICES=$(jq '.categories["best-practices"].score' "$OUTPUT_PATH".report.json) 32 | SCORE_SEO=$(jq '.categories["seo"].score' "$OUTPUT_PATH".report.json) 33 | SCORE_PWA=$(jq '.categories["pwa"].score' "$OUTPUT_PATH".report.json) 34 | 35 | # Print scores to standard output (0 to 100 instead of 0 to 1). 36 | # Using hacky bc b/c bash hates floating point arithmetic... 37 | printf "\n* Completed audit of %s ! Scores are printed below:\n\n" "$REPORT_URL" 38 | printf "+-------------------------------+\n" 39 | printf "| Performance: %.0f\t|\n" "$(echo "$SCORE_PERFORMANCE*100" | bc -l)" 40 | printf "| Accessibility: %.0f\t|\n" "$(echo "$SCORE_ACCESSIBILITY*100" | bc -l)" 41 | printf "| Best Practices: %.0f\t|\n" "$(echo "$SCORE_PRACTICES*100" | bc -l)" 42 | printf "| SEO: %.0f\t|\n" "$(echo "$SCORE_SEO*100" | bc -l)" 43 | printf "| Progressive Web App: %.0f\t|\n" "$(echo "$SCORE_PWA*100" | bc -l)" 44 | printf "+-------------------------------+\n\n" 45 | printf "* Detailed results are saved here, use https://github.com/actions/upload-artifact to retrieve them:\n" 46 | printf " %s\n" "$OUTPUT_PATH.report.html" 47 | printf " %s\n" "$OUTPUT_PATH.report.json" 48 | } 49 | 50 | if [ "$INPUT_MOBILE" == "true" ]; then 51 | createReport mobile 52 | fi 53 | if [ "$INPUT_DESKTOP" == "true" ]; then 54 | createReport desktop 55 | fi 56 | 57 | exit 0 -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lighthouse-bot", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "src/index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "start": "pm2 start ecosystem.config.js --no-daemon", 9 | "dev": "pm2 start ecosystem.config.js --node-args=\"--inspect=9229\" --no-daemon" 10 | }, 11 | "keywords": [], 12 | "author": "", 13 | "license": "Apache-2.0", 14 | "dependencies": { 15 | "axios": "^0.21.1", 16 | "express": "^4.17.1", 17 | "lighthouse": "^7.1.0", 18 | "mongoose": "^5.9.19", 19 | "node-schedule": "^1.3.2", 20 | "path": "^0.12.7", 21 | "puppeteer": "^7.1.0", 22 | "run-middleware": "^0.9.10" 23 | }, 24 | "devDependencies": { 25 | "eslint": "^7.3.0" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /readme.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanteddev/lighthouse/4b032ad1537ba5e3a13074a5b50129b41a999a6a/readme.png -------------------------------------------------------------------------------- /src/api/index.js: -------------------------------------------------------------------------------- 1 | const axios = require('axios'); 2 | const constants = require('../constants'); 3 | const utils = require('../utils'); 4 | 5 | const TOKEN = utils.common.checkEnvVar(constants.TOKEN); 6 | const API_ENDPOINT = constants.API_ENDPOINT; 7 | 8 | /** 9 | * Open a modal to a given user 10 | * @param {Object} payload - object with required request body params 11 | * @param {string} payload.trigger_id - The trigger ID received from a slash command (req) 12 | * @param {Object} payload.view - The contents of the modal using the blocks API (req) 13 | * @returns {Object} data 14 | */ 15 | async function openModal(payload) { 16 | const data = await doPost(`${API_ENDPOINT}/views.open`, payload); 17 | return data; 18 | } 19 | 20 | /** 21 | * Send an ephemeral post to Slack 22 | * @param {Object} payload - object with required request body params 23 | * @param {string} payload.text - Contents of message to be sent (req) 24 | * @param {string} payload.user - ID of user to sent message to (req) 25 | * @param {Array} payload.attachments - post attachments (req) 26 | * @param {string} payload.channel - ID of channel to post to (req) 27 | * @returns {Object} data 28 | */ 29 | async function sendEphemeralPostToUser(payload) { 30 | const data = await doPost(`${API_ENDPOINT}/chat.postEphemeral`, payload); 31 | return data; 32 | } 33 | 34 | /** Send a post to Slack 35 | * @param {Object} payload - object with required request body params 36 | * @param {string} payload.text - Contents of message to be sent (req) 37 | * @param {Array} payload.attachments - post attachments (optional) 38 | * @param {string} payload.channel - ID of channel to post to (req) 39 | * @returns {Object} data 40 | */ 41 | async function sendPostToChannel(payload) { 42 | const data = await doPost(`${API_ENDPOINT}/chat.postMessage`, payload); 43 | return data; 44 | } 45 | 46 | async function doPost(url, data) { 47 | const options = { 48 | url, 49 | data, 50 | method: 'POST', 51 | headers: { 52 | 'Content-Type': 'application/json;charset=utf-8', 53 | 'Authorization': `Bearer ${TOKEN}`, 54 | }, 55 | json: true, 56 | }; 57 | 58 | return await axios(options) 59 | .then((response) => { 60 | return response.data; 61 | }) 62 | .catch((error) => { 63 | utils.common.logger.error(error); 64 | return error; 65 | }); 66 | } 67 | 68 | module.exports = { 69 | openModal, 70 | sendEphemeralPostToUser, 71 | sendPostToChannel, 72 | }; 73 | -------------------------------------------------------------------------------- /src/constants/index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | // These are key-values for environment variables 3 | // Do not change their values, instead set environment variables 4 | // in PM2 or Docker 5 | PORT: 'PORT', 6 | TOKEN: 'TOKEN', 7 | MONGO_USERNAME: 'MONGO_USERNAME', 8 | MONGO_PASSWORD: 'MONGO_PASSWORD', 9 | MONGO_SERVER: 'MONGO_SERVER', 10 | CHATBOT_SERVER: 'CHATBOT_SERVER', 11 | 12 | // Constant variables 13 | API_ENDPOINT: 'https://slack.com/api', 14 | }; 15 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | const express = require('express'); 2 | const path = require('path'); 3 | 4 | const utils = require('./utils'); 5 | const constants = require('./constants'); 6 | const store = require('./store'); 7 | const routes = require('./routes'); 8 | 9 | const PORT = utils.common.checkEnvVar(constants.PORT); 10 | 11 | const app = express(); 12 | require('run-middleware')(app); 13 | app.use(express.json()); 14 | app.use(express.urlencoded({extended: true})); 15 | 16 | const static_path = path.join(__dirname, "./static"); 17 | app.use(express.static(static_path)); 18 | 19 | app.use('/', routes); 20 | 21 | app.listen(PORT, async function() { 22 | utils.common.logger.info(`bot listening on port ${PORT}!`); 23 | // On server startup, load all stored schedules and queue them to be run 24 | const list = await store.schedule.getScheduleList(); 25 | for (let schedule of list) { 26 | utils.common.logger.debug(`scheduling job with id=${schedule._id}`); 27 | utils.schedule.scheduleJob(schedule, async function() { 28 | const options = { 29 | throttling: schedule.throttling, 30 | performance: schedule.performance, 31 | accessibility: schedule.accessibility, 32 | 'best-practices': schedule['best-practices'], 33 | pwa: schedule.pwa, 34 | seo: schedule.seo, 35 | }; 36 | 37 | app.runMiddleware('/init_audit', { 38 | method: 'POST', 39 | body: { 40 | audit_url: schedule.audit_url, 41 | user_id: schedule.user_id, 42 | channel: schedule.channel, 43 | options, 44 | } 45 | }); 46 | }); 47 | } 48 | }); 49 | -------------------------------------------------------------------------------- /src/routes/index.js: -------------------------------------------------------------------------------- 1 | const express = require('express'); 2 | const fs = require('fs'); 3 | 4 | const router = express.Router(); 5 | 6 | const api = require('../api'); 7 | const constants = require('../constants'); 8 | const utils = require('../utils'); 9 | const store = require('../store'); 10 | 11 | const CHATBOT_SERVER = utils.common.checkEnvVar(constants.CHATBOT_SERVER); 12 | 13 | router.post('/lighthouse', async function(req, res) { 14 | const req_data = req.body; 15 | const { 16 | text, 17 | user_id, 18 | channel_id, 19 | trigger_id, 20 | } = req_data; 21 | 22 | let retext; 23 | const onlyUrl = text.match(/(https?:\/\/[^ ]*)\b/); 24 | if (onlyUrl) { 25 | retext = onlyUrl[1]; 26 | } else { 27 | retext = text; 28 | } 29 | 30 | const req_options = retext.split(' '); 31 | const url_pattern = /^https?:\/\//; 32 | switch(req_options[0]) { 33 | case 'help': 34 | res.send({ 35 | text: '*Lighthouse Audit Bot - Slash Command Help*\n\n' 36 | + '* `/lighthouse` - Launch dialog to run ad-hoc audits with full control over options\n' 37 | + '* `/lighthouse {url}` - Run a quick audit with default settings on a website\n' 38 | + '* `/lighthouse jobs` - Launch dialog to create an audit job with full control over options\n' 39 | + '* `/lighthouse jobs ls` - Show full list of schedules created\n' 40 | + '* `/lighthouse jobs info {id}` - Show configuration of a given job\n' 41 | + '* `/lighthouse jobs rm {id}` - Removes a scheduled audit job (You may input several IDs in the same command)\n' 42 | + '* `/lighthouse stats {url}` - Returns a link to the audit trends dashboard of a given URL (**MUST** have performed at least 2 audits on the URL beforehand)' 43 | }); 44 | return; 45 | case 'stats': 46 | const url = req_options[1]; 47 | if (!url) { 48 | res.send({ 49 | response_type: 'ephemeral', 50 | text: 'Please input the URL that you\'d like to view stats of with `/lighthouse stats {url}`' 51 | }); 52 | return; 53 | } 54 | 55 | // check if more than 2 audits have run in the past 56 | const audits = await store.audit.getAuditReportsByUrl(url); 57 | if (audits.length < 2) { 58 | res.send({ 59 | response_type: 'ephemeral', 60 | text: `Please ensure you have run at least 2 audit runs on the URL ${url}` 61 | }); 62 | return; 63 | } 64 | 65 | res.send({ 66 | text: `<${CHATBOT_SERVER}/view_stats?url=${url}|Click here> to view all auditing statistics for ${url}` 67 | }); 68 | break; 69 | case 'jobs': 70 | switch (req_options[1]) { 71 | case 'ls': 72 | // generate schedule list and return to user 73 | const list = await store.schedule.getScheduleList(); 74 | let text = 'No scheduled jobs found'; 75 | let attachments = []; 76 | if (list.length > 0) { 77 | text = ''; 78 | for(let schedule of list) { 79 | const details = utils.response.generateScheduleDetails(schedule); 80 | attachments.push(details); 81 | // text += `| ${schedule._id} | @${schedule.username} | ${schedule.audit_url} | ${schedule.schedule} |\n`; 82 | } 83 | } 84 | res.send({ 85 | response_type: 'ephemeral', 86 | text, 87 | attachments, 88 | }); 89 | break; 90 | case 'rm': 91 | // remove a scheduled job if a valid ID is provided 92 | if (!req_options[2]) { 93 | res.send({ 94 | response_type: 'ephemeral', 95 | text: 'Please input the ID of the schedule you\'d like to remove as `/lighthouse jobs rm {id}`\nYou may retrieve this value by running the `/lighthouse jobs ls` command' 96 | }); 97 | return; 98 | } 99 | 100 | let id_idx = 2; 101 | let deleted_items = []; 102 | while(req_options[id_idx]) { 103 | try { 104 | await store.schedule.deleteScheduleWithId(req_options[id_idx]); 105 | utils.schedule.removeJob(req_options[id_idx]); 106 | utils.common.logger.info(`removed scheduled job with id=${req_options[id_idx]}`); 107 | deleted_items.push({isDeleted: true, text: `Deleted: ${req_options[id_idx]}`}); 108 | } catch(error) { 109 | utils.common.logger.error(error); 110 | deleted_items.push({isDeleted: false, text: `Error: Failed to remove scheduled job with ID \`${req_options[id_idx]}\`. Please make sure the ID you selected is valid with the \`/lighthouse jobs ls\` command.`}); 111 | } 112 | id_idx++; 113 | } 114 | 115 | let response = 'Scheduled jobs deletion:\n'; 116 | for (const item of deleted_items) { 117 | response += `* ${item.text}\n`; 118 | } 119 | 120 | res.send({ 121 | response_type: 'ephemeral', 122 | text: response 123 | }); 124 | break; 125 | 126 | case 'info': 127 | const id = req_options[2]; 128 | if (!id) { 129 | res.send({ 130 | response_type: 'ephemeral', 131 | text: 'Please input the ID of the schedule you\'d like to view details of as `/lighthouse info {id}`\nYou may retrieve this value by running the `/lighthouse jobs ls` command' 132 | }); 133 | return; 134 | } 135 | 136 | try { 137 | const schedule = await store.schedule.getSchedule(id); 138 | const response = utils.response.generateScheduleDetails(schedule, true); 139 | utils.common.logger.info(`retrieved information on schedule with id="${schedule._id}" for user_id=${req_data.user_id}`); 140 | res.send({ 141 | attachments: [response] 142 | }); 143 | } catch(error) { 144 | utils.common.logger.error(error); 145 | res.send({ 146 | response_type: 'ephemeral', 147 | text: `Failed to fetch information for job with ID \`${id}\`.\nPlease make sure the ID you selected is valid with the \`/lighthouse jobs ls\` command.` 148 | }); 149 | } 150 | break; 151 | default: 152 | // if none found, launch create schedule dialog 153 | utils.common.logger.info(`launching job scheduling dialog for user_id=${req_data.user_id}`); 154 | const view = utils.response.generateAuditDialog(true); 155 | view.private_metadata = JSON.stringify({channel: channel_id, is_schedule: true}); 156 | const payload = { 157 | trigger_id, 158 | view, 159 | }; 160 | 161 | await api.openModal(payload); 162 | } 163 | break; 164 | default: 165 | if (req_options[0] && url_pattern.test(req_options[0])) { 166 | // Run quick audit if value URL is found in command 167 | const opts = { 168 | performance: true, 169 | accessibility: true, 170 | 'best-practices': true, 171 | seo: true, 172 | pwa: false, 173 | throttling: true, 174 | }; 175 | res.send(); 176 | await runAudit(req_options[0], user_id, channel_id, opts); 177 | } else { 178 | 179 | // Launch audit dialog w/ options 180 | utils.common.logger.info(`launching audit dialog for user_id=${req_data.user_id}`); 181 | const view = utils.response.generateAuditDialog(); 182 | view.private_metadata = JSON.stringify({channel: channel_id, is_schedule: false}); 183 | const payload = { 184 | trigger_id, 185 | view, 186 | }; 187 | await api.openModal(payload); 188 | } 189 | } 190 | res.send(); 191 | }); 192 | 193 | /******************************** 194 | * Schedule Creation 195 | *********************************/ 196 | router.post('/receive_submission', async function(req, res) { 197 | const res_data = JSON.parse(req.body.payload); 198 | const values = res_data.view.state.values; 199 | const {channel, is_schedule} = JSON.parse(res_data.view.private_metadata); 200 | 201 | let submission = { 202 | throttling: false, 203 | performance: false, 204 | seo: false, 205 | pwa: false, 206 | 'best-practices': false, 207 | audit_url: '', 208 | schedule: '', 209 | user_id: res_data.user.id, 210 | username: res_data.user.username, 211 | channel, 212 | auth_header: undefined, 213 | cookie_name: undefined, 214 | cookie_value: undefined, 215 | }; 216 | 217 | console.log(JSON.stringify(values)); 218 | for (const key in values) { 219 | if (values[key].audit_options && values[key].audit_options.selected_options && values[key].audit_options.selected_options.length > 0) { 220 | values[key].audit_options.selected_options.forEach(option => { 221 | submission[option.value] = true; 222 | }); 223 | } else { 224 | for (const optionKey of Object.keys(values[key])) { 225 | submission[optionKey] = values[key][optionKey].value; 226 | } 227 | } 228 | } 229 | 230 | try { 231 | // Ad-hoc run 232 | if (!is_schedule) { 233 | const options = { 234 | throttling: submission.throttling, 235 | performance: submission.performance, 236 | accessibility: submission.accessibility, 237 | 'best-practices': submission['best-practices'], 238 | pwa: submission.pwa, 239 | seo: submission.seo, 240 | auth_header: submission.auth_header, 241 | cookie_name: submission.cookie_name, 242 | cookie_value: submission.cookie_value, 243 | }; 244 | res.send(); 245 | await runAudit(submission.audit_url, submission.user_id, submission.channel, options); 246 | return; 247 | } 248 | 249 | // Schedule run 250 | const schedule = await store.schedule.createSchedule(submission); 251 | 252 | utils.schedule.scheduleJob(schedule, async function() { 253 | const options = { 254 | throttling: schedule.throttling, 255 | performance: schedule.performance, 256 | accessibility: schedule.accessibility, 257 | 'best-practices': schedule['best-practices'], 258 | pwa: schedule.pwa, 259 | seo: schedule.seo, 260 | auth_header: schedule.auth_header, 261 | cookie_name: schedule.cookie_name, 262 | cookie_value: schedule.cookie_value, 263 | }; 264 | await runAudit(schedule.audit_url, schedule.user_id, schedule.channel, options); 265 | }); 266 | 267 | let text = 'Successfully scheduled a new job!'; 268 | let attachments = []; 269 | const attachment = utils.response.generateScheduleDetails(schedule); 270 | attachments.push(attachment); 271 | 272 | await api.sendEphemeralPostToUser({ 273 | text, 274 | user: res_data.user.id, 275 | channel, 276 | attachments, 277 | }); 278 | } catch(error) { 279 | utils.common.logger.error(error); 280 | await api.sendPostToChannel({ 281 | channel, 282 | text: 'An error has occurred, please try again or contact an administrator for support.' 283 | }); 284 | } 285 | 286 | res.send(); 287 | }); 288 | 289 | /******************************** 290 | * Audit Run 291 | *********************************/ 292 | router.post('/run_audit', async function(req, res) { 293 | const body = req.body; 294 | const {audit_url} = body.submission; 295 | 296 | const validation_error = validateOptions(body.submission); 297 | if (validation_error) { 298 | utils.common.logger.error(validation_error); 299 | res.send({error: validation_error}); 300 | return; 301 | } else { 302 | res.send(); // make sure dialog gets dismissed 303 | } 304 | 305 | await runAudit(audit_url, body.user_id, body.channel, body.submission); 306 | }); 307 | 308 | // Using as middleware in order to add schedules from app root 309 | // TODO: investigate more adequate pattern to make runAudit re-usable with current project structure 310 | router.post('/init_audit', async function(req, res) { 311 | const {audit_url, user_id, channel, options} = req.body; 312 | res.send(); 313 | await runAudit(audit_url, user_id, channel, options); 314 | }); 315 | 316 | async function runAudit(url, user_id, channel, options) { 317 | let time = utils.common.generateCurrentTime(); 318 | try { 319 | utils.common.logger.debug(`Running audit report for url=${url}`); 320 | await api.sendEphemeralPostToUser({ 321 | user: user_id, 322 | channel, 323 | text: `Running audit report for ${url}!\nPlease wait for the audit to be completed`, 324 | attachments: [], 325 | }); 326 | const lhs = await utils.lighthouse.runLighthouseAudit(url, options); 327 | const audit = await store.audit.createAudit(user_id, JSON.stringify(lhs), url); 328 | const report_url = `${CHATBOT_SERVER}/view_report/${audit._id}`; 329 | const report = utils.response.generateReportAttachment(lhs, url, time, report_url); 330 | 331 | const payload = { 332 | channel, 333 | attachments: [ 334 | report, 335 | ], 336 | }; 337 | await api.sendPostToChannel(payload); 338 | } catch(error) { 339 | utils.common.logger.error('Failed to run audit'); 340 | utils.common.logger.error(error); 341 | await api.sendPostToChannel({ 342 | channel, 343 | text: `Failed to run audit, please try again or contact an administrator.` 344 | }); 345 | } 346 | } 347 | 348 | function validateOptions(options) { 349 | if ( 350 | options.performance === '0' && 351 | options.accessibility === '0' && 352 | options['best-practices'] === '0' && 353 | options.pwa === '0' && 354 | options.seo === '0' 355 | ) { 356 | return 'Please make sure you have at least one category enabled'; 357 | } 358 | 359 | if (options.auth_script && !options.await_selector) { 360 | return 'Please input an await selector when using an Authentication Script'; 361 | } 362 | return null; 363 | } 364 | 365 | /******************************** 366 | * Report Viewer 367 | *********************************/ 368 | router.get('/view_report/:id', async function(req, res) { 369 | const id = req.params.id; 370 | res.setHeader('Content-Type', 'text/html'); 371 | try { 372 | const report = await store.audit.getAuditReport(id); 373 | const html = utils.lighthouse.generateHtmlReport(report); 374 | res.send(html); 375 | } catch(error) { 376 | utils.common.logger.error(error); 377 | const html = fs.readFileSync(__dirname + '/../static/404.html', 'utf8'); 378 | res.send(html); 379 | } 380 | }); 381 | 382 | router.get('/view_stats', async function(req, res) { 383 | const {url, number} = req.query; 384 | res.setHeader('Content-Type', 'text/html'); 385 | try { 386 | if (!url) { 387 | throw new Error('No URL parameter found when trying to render stats'); 388 | } 389 | 390 | const data = { 391 | url, 392 | performance: [], 393 | accessibility: [], 394 | 'best-practices': [], 395 | pwa: [], 396 | seo: [], 397 | }; 398 | const audits = await store.audit.getAuditReportsByUrl(url, parseInt(number)); 399 | for (let audit of audits) { 400 | const report = JSON.parse(audit.report); 401 | const date = new Date(audit.created_date * 1000).toLocaleTimeString([], {year: 'numeric', month: '2-digit', day: '2-digit'}); 402 | 403 | // Add null checks 404 | if (report.categories.performance) { 405 | data.performance.push({time: date, value: report.categories.performance.score * 100}); 406 | } 407 | if (report.categories.accessibility) { 408 | data.accessibility.push({time: date, value: report.categories.accessibility.score * 100}); 409 | } 410 | if (report.categories['best-practices']) { 411 | data['best-practices'].push({time: date, value: report.categories['best-practices'].score * 100}); 412 | } 413 | if (report.categories.pwa) { 414 | data.pwa.push({time: date, value: report.categories.pwa.score * 100}); 415 | } 416 | if (report.categories.seo) { 417 | data.seo.push({time: date, value: report.categories.seo.score * 100}); 418 | } 419 | } 420 | 421 | const html = utils.lighthouse.generateHtmlStats(data); 422 | res.send(html); 423 | } catch(error) { 424 | utils.common.logger.error(error); 425 | const html = fs.readFileSync(__dirname + '/../static/404.html', 'utf8'); 426 | res.send(html); 427 | } 428 | }); 429 | 430 | module.exports = router; 431 | -------------------------------------------------------------------------------- /src/static/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Lighthouse report not found 5 | 6 | 36 | 37 | 38 |
39 |

404 - not found

40 | 41 |

The resource you requested could not be found

42 |
43 | 44 | -------------------------------------------------------------------------------- /src/static/img/lighthouse-404.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanteddev/lighthouse/4b032ad1537ba5e3a13074a5b50129b41a999a6a/src/static/img/lighthouse-404.png -------------------------------------------------------------------------------- /src/static/img/lighthouse-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wanteddev/lighthouse/4b032ad1537ba5e3a13074a5b50129b41a999a6a/src/static/img/lighthouse-logo.png -------------------------------------------------------------------------------- /src/static/reportTemplate.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Lighthouse Report 8 | 9 | 10 | 11 | 12 | 13 | 14 |
15 | 16 |
17 | 18 | 21 | 22 | 62 | 63 | -------------------------------------------------------------------------------- /src/static/statsTemplate.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Lighthouse Performance Graph 7 | 68 | 69 | 70 | 71 | 72 | 73 | 79 |
80 | 81 |
82 | 83 | 203 | 204 | -------------------------------------------------------------------------------- /src/store/audit.js: -------------------------------------------------------------------------------- 1 | const mongoose = require('mongoose'); 2 | const utils = require('../utils'); 3 | 4 | const schema = new mongoose.Schema({ 5 | created_date: Number, // unix timestamp 6 | audit_url: String, 7 | user_id: String, // id of person who ran an audit 8 | report: String, // json formatted string of report object 9 | }); 10 | 11 | const AuditModel = mongoose.model('Audit', schema); 12 | 13 | async function createAudit(user_id, report, audit_url) { 14 | const new_audit = new AuditModel({ 15 | created_date: utils.common.generateTimestamp(), 16 | audit_url, 17 | user_id, 18 | report, 19 | }); 20 | 21 | const data = await new_audit.save(); 22 | return data; 23 | } 24 | 25 | async function getAuditReport(id) { 26 | const audit = await AuditModel.findById(id); 27 | const report = JSON.parse(audit.report); 28 | return report; 29 | } 30 | 31 | async function getAuditReportsByUrl(url, reportNumber) { 32 | const audits = await AuditModel.find({audit_url: url}).sort({_id: -1}).limit(reportNumber || 5); 33 | return audits.reverse(); 34 | } 35 | 36 | module.exports = { 37 | createAudit, 38 | getAuditReport, 39 | getAuditReportsByUrl 40 | }; 41 | -------------------------------------------------------------------------------- /src/store/index.js: -------------------------------------------------------------------------------- 1 | const mongoose = require('mongoose'); 2 | const utils = require('../utils'); 3 | const constants = require('../constants'); 4 | 5 | const schedule = require('./schedule'); 6 | const audit = require('./audit'); 7 | 8 | const mongoUsername = utils.common.checkEnvVar(constants.MONGO_USERNAME); 9 | const mongoPassword = utils.common.checkEnvVar(constants.MONGO_PASSWORD); 10 | const mongoServer = utils.common.checkEnvVar(constants.MONGO_SERVER); 11 | 12 | mongoose.connect(`mongodb://${mongoUsername}:${mongoPassword}@${mongoServer}/admin`, {useNewUrlParser: true, useUnifiedTopology: true}).then( 13 | function() { 14 | utils.common.logger.info('Successfully connected to database!'); 15 | }, 16 | function(error) { 17 | utils.common.logger.error(error); 18 | } 19 | ); 20 | 21 | module.exports = { 22 | schedule, 23 | audit, 24 | }; 25 | -------------------------------------------------------------------------------- /src/store/schedule.js: -------------------------------------------------------------------------------- 1 | const mongoose = require('mongoose'); 2 | const utils = require('../utils'); 3 | 4 | const schema = new mongoose.Schema({ 5 | created_date: Number, // unix timestamp 6 | user_id: String, // id of person who registered a given schedule 7 | username: String, 8 | channel: String, 9 | 10 | // Audit options 11 | schedule: String, 12 | audit_url: String, 13 | performance: Boolean, 14 | accessibility: Boolean, 15 | 'best-practices': Boolean, 16 | seo: Boolean, 17 | pwa: Boolean, 18 | throttling: Boolean, 19 | auth_header: String, 20 | cookie_name: String, 21 | cookie_value: String, 22 | }); 23 | 24 | const ScheduleModel = mongoose.model('Schedule', schema); 25 | 26 | async function createSchedule(payload) { 27 | const new_schedule = new ScheduleModel({ 28 | created_date: utils.common.generateTimestamp(), 29 | user_id: payload.user_id, 30 | channel: payload.channel, 31 | username: payload.username, 32 | schedule: payload.schedule, 33 | audit_url: payload.audit_url, 34 | performance: payload.performance, 35 | accessibility: payload.accessibility, 36 | 'best-practices': payload['best-practices'], 37 | seo: payload.seo, 38 | pwa: payload.pwa, 39 | throttling: payload.throttling, 40 | auth_header: payload.auth_header, 41 | cookie_name: payload.cookie_name, 42 | cookie_value: payload.cookie_value, 43 | }); 44 | 45 | const data = await new_schedule.save(); 46 | return data; 47 | } 48 | 49 | async function getSchedule(id) { 50 | const data = await ScheduleModel.findById(id); 51 | return data; 52 | } 53 | 54 | async function getScheduleList() { 55 | const list = await ScheduleModel.find(); 56 | return list; 57 | } 58 | 59 | async function deleteScheduleWithId(id) { 60 | const data = await ScheduleModel.findByIdAndDelete(id); 61 | return data; 62 | } 63 | 64 | module.exports = { 65 | createSchedule, 66 | getSchedule, 67 | getScheduleList, 68 | deleteScheduleWithId, 69 | }; 70 | -------------------------------------------------------------------------------- /src/utils/common.js: -------------------------------------------------------------------------------- 1 | const logger = { 2 | debug: function(message) { 3 | const timestamp = generateCurrentTime(); 4 | console.log('\x1b[36m%s\x1b[0m', `[DEBU]: ${timestamp} - ${message}`); 5 | }, 6 | 7 | error: function(message) { 8 | const timestamp = generateCurrentTime(); 9 | console.error('\x1b[31m%s\x1b[0m', `[ERRO]: ${timestamp} - ${message}`); 10 | }, 11 | 12 | info: function(message) { 13 | const timestamp = generateCurrentTime(); 14 | console.log('\x1b[32m%s\x1b[0m', `[INFO]: ${timestamp} - ${message}`); 15 | }, 16 | }; 17 | 18 | function checkEnvVar(variable) { 19 | if (process.env[variable]) { 20 | return process.env[variable]; 21 | } 22 | 23 | logger.error(`Error: the environment variable ${variable} has not been set!`); 24 | process.exit(1); 25 | } 26 | 27 | function generateCurrentTime() { 28 | return new Date().toLocaleTimeString([], {year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', second: '2-digit', hourCycle: 'h23'}); 29 | } 30 | 31 | function generateTimestamp() { 32 | return Math.floor(new Date() / 1000); 33 | } 34 | 35 | function getScoreElement(score, type) { 36 | let color = '#0BCE6B'; 37 | let emoji = ':white_check_mark:'; 38 | if (score >= 0 && score < 0.5) { 39 | color = '#FF4F42'; 40 | emoji = ':x:'; 41 | } else if (score >= 0.5 && score < 0.9) { 42 | color = '#FFA400'; 43 | emoji = ':warning:'; 44 | } 45 | return type === 'color' ? color : emoji; 46 | } 47 | 48 | module.exports = { 49 | logger, 50 | checkEnvVar, 51 | generateTimestamp, 52 | generateCurrentTime, 53 | getScoreElement, 54 | }; 55 | -------------------------------------------------------------------------------- /src/utils/index.js: -------------------------------------------------------------------------------- 1 | const common = require('./common'); 2 | const response = require('./responseBuilder'); 3 | const lighthouse = require('./lighthouse'); 4 | const schedule = require('./schedule'); 5 | 6 | module.exports = { 7 | common, 8 | response, 9 | lighthouse, 10 | schedule, 11 | }; 12 | -------------------------------------------------------------------------------- /src/utils/lighthouse.js: -------------------------------------------------------------------------------- 1 | const puppeteer = require('puppeteer'); 2 | const lighthouse = require('lighthouse'); 3 | const {Worker, isMainThread, parentPort, workerData} = require('worker_threads'); 4 | 5 | const fs = require('fs'); 6 | 7 | const {replaceStrings} = require('lighthouse/lighthouse-core/report/report-generator'); 8 | const htmlReportAssets = require('lighthouse/lighthouse-core/report/html/html-report-assets.js'); 9 | 10 | const {logger} = require('./common'); 11 | 12 | // Auto-runs puppeteer when run as a worker 13 | // main thread should pass down URL and lighthouse options as workerData 14 | if (!isMainThread) { 15 | const {url, options} = workerData; 16 | launchPuppeteer(url, options); 17 | } 18 | 19 | async function launchPuppeteer(url, options) { 20 | try { 21 | const browser = await puppeteer.launch({ 22 | args: [ 23 | '--no-sandbox', 24 | '--disable-setuid-sandbox', 25 | // This will write shared memory files into /tmp instead of /dev/shm, 26 | // because Docker’s default for /dev/shm is 64MB 27 | '--disable-dev-shm-usage' 28 | ] 29 | }); 30 | const page = await browser.newPage(); 31 | 32 | if (options.auth_header) { 33 | await page.setExtraHTTPHeaders({ 34 | 'Authorization': options.auth_header, 35 | }) 36 | } 37 | 38 | if (options.cookie_name && options.cookie_value) { 39 | await page.setCookie({ name: options.cookie_name, value: options.cookie_value, url }); 40 | } 41 | await page.goto(url, { 42 | waitUntil: 'networkidle0', 43 | }); 44 | await page.waitForSelector('body', {visible: true}); 45 | await page.close(); 46 | // Lighthouse will open URL. Puppeteer observes `targetchanged` and sets up network conditions. 47 | // Possible race condition. 48 | let opts = { 49 | port: (new URL(browser.wsEndpoint())).port, 50 | output: 'json', 51 | onlyCategories: [], 52 | screenEmulation: { 53 | disabled: true, 54 | }, 55 | }; 56 | 57 | if (options.performance) opts.onlyCategories.push('performance'); 58 | if (options.accessibility) opts.onlyCategories.push('accessibility'); 59 | if (options['best-practices']) opts.onlyCategories.push('best-practices'); 60 | if (options.pwa) opts.onlyCategories.push('pwa'); 61 | if (options.seo) opts.onlyCategories.push('seo'); 62 | 63 | // as throttling is enabled by default in lighthouse, disable it if explicitly unchecked 64 | if (options.throttling === false) { 65 | // Values referenced in 66 | // https://github.com/GoogleChrome/lighthouse/blob/master/lighthouse-core/config/constants.js 67 | opts.throttlingMethod = 'provided'; 68 | opts.emulatedUserAgent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4420.0 Safari/537.36 Chrome-Lighthouse'; 69 | opts.throttling = { 70 | rttMs: 40, 71 | throughputKbps: 10 * 1024, 72 | cpuSlowdownMultiplier: 1, 73 | requestLatencyMs: 0, 74 | downloadThroughputKbps: 0, 75 | uploadThroughputKbps: 0, 76 | }; 77 | opts.formFactor = 'desktop' 78 | opts.screenEmulation = { 79 | mobile: false, 80 | width: 1350, 81 | height: 940, 82 | deviceScaleFactor: 1, 83 | disabled: false, 84 | } 85 | } 86 | 87 | const {lhr} = await lighthouse(url, opts); 88 | // Return response back to main thread 89 | parentPort.postMessage(lhr); 90 | await browser.close(); 91 | return; 92 | } catch(error) { 93 | logger.error(error); 94 | } 95 | } 96 | 97 | // This function spawns a worker thread that will handle launching puppeteer and returning results 98 | async function runLighthouseAudit(url, options) { 99 | return new Promise((resolve, reject) => { 100 | const worker = new Worker(__filename, {workerData: {url, options}}); 101 | worker.on('message', resolve); 102 | worker.on('error', reject); 103 | worker.on('exit', (code) => { 104 | if (code !== 0) 105 | reject(new Error(`Worker stopped with exit code ${code}`)); 106 | }); 107 | }); 108 | } 109 | 110 | function generateHtmlReport(lhr) { 111 | const REPORT_TEMPLATE = fs.readFileSync(__dirname + '/../static/reportTemplate.html', 'utf8'); 112 | const sanitizedJson = JSON.stringify(lhr) 113 | .replace(/`, 283 | color, 284 | fields 285 | }; 286 | } 287 | 288 | function generateCheckbox(text, value) { 289 | return { 290 | text: { 291 | type: 'plain_text', 292 | text, 293 | }, 294 | value, 295 | }; 296 | } 297 | 298 | function generateAuditField(audit) { 299 | const emoji = getScoreElement(audit.score, 'emoji'); 300 | 301 | return { 302 | short: true, 303 | title: audit.title, 304 | value: `${emoji} \`${audit.displayValue}\`` 305 | }; 306 | } 307 | 308 | module.exports = { 309 | generateAuditDialog, 310 | generateReportAttachment, 311 | generateScheduleDetails, 312 | }; 313 | -------------------------------------------------------------------------------- /src/utils/schedule.js: -------------------------------------------------------------------------------- 1 | const schedule = require('node-schedule'); 2 | 3 | function scheduleJob(s, job) { 4 | schedule.scheduleJob(s._id.toString(), s.schedule, () => { 5 | job(); 6 | }); 7 | } 8 | 9 | function removeJob(id) { 10 | schedule.cancelJob(id); 11 | } 12 | 13 | module.exports = { 14 | scheduleJob, 15 | removeJob 16 | }; 17 | --------------------------------------------------------------------------------