├── .gitignore ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── frontend ├── .eslintrc.yaml ├── .gcloudignore ├── LICENSE ├── Readme.md ├── app.js ├── app.yaml ├── bin │ └── www ├── logger │ └── index.js ├── package-lock.json ├── package.json ├── public │ ├── scripts │ │ └── material-components-web.js │ └── stylesheets │ │ ├── material-components-web.css │ │ └── style.css ├── routes │ ├── index.js │ └── websites.js └── views │ ├── error.jade │ ├── index.jade │ ├── layout.jade │ └── website.jade ├── image-diff ├── .eslintrc.yaml ├── .gcloudignore ├── LICENSE ├── Readme.md ├── diff.js ├── index.js ├── package-lock.json ├── package.json └── test │ ├── one.jpg │ ├── test-diff.js │ └── two.jpg ├── screenshot ├── .eslintrc.yaml ├── .gcloudignore ├── LICENSE ├── Readme.md ├── app.yaml ├── logger │ └── index.js ├── package-lock.json ├── package.json └── server.js └── task-scheduler ├── .eslintrc.yaml ├── .gcloudignore ├── LICENSE ├── README.md ├── app.yaml ├── cron.yaml ├── lib └── create-screenshot-event.js ├── logger └── index.js ├── package-lock.json ├── package.json └── server.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | education, socio-economic status, nationality, personal appearance, race, 10 | religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at [INSERT EMAIL ADDRESS]. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows [Google's Open Source Community 28 | Guidelines](https://opensource.google.com/conduct/). 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Web page visual history 2 | 3 | A set of microservices to track visual changes of web pages. 4 | 5 | **This code is provided as example only. It is provided without guarantee or level of support.** This is not an official Google product (experimental or otherwise), it is just code that happens to be owned by Google. 6 | 7 | ## Dependencies 8 | 9 | * Google Cloud Platform 10 | * App Engine 11 | * Cloud Functions 12 | * Cloud Datastore 13 | * Cloud Storage 14 | * Cloud Pub/Sub 15 | * App Engine cron 16 | 17 | ## List of micro services 18 | 19 | Each micro service is stored in its dedicated folder. 20 | 21 | * `frontend`: A frontend to vizualise which websites are tracked and see their screenshots 22 | * `task-scheduler`: Every 5 minutes, looks for screenshots to take and schedule them as tasks. 23 | * `screenshot`: Takes screenshots of the given URL, stores it in Cloud Storage 24 | * `image-diff`: Compares an image with its reference image in Cloud Storage 25 | 26 | ## Architecture 27 | 28 | screen shot 2018-08-17 at 6 30 37 pm 29 | 30 | Every 5 minutes, the `task-scheduler` microservice runs, queries the Cloud Datastore database for webpages ot screenshot. 31 | For each webpage, it sends a message to a Pub/Sub topic. 32 | This topic is responsible for pushing these messages to the `screenshot` microservice. 33 | This `screenshot` service receives the message as a regular HTTP POST request, screenshots requested webpage and stores the result in a Cloud Storage bucket. 34 | The creation of a new file in the bucket triggers the a Cloud Function (`image-diff`). 35 | The `image-diff` function compares the new image with a reference image from the `references` folder. If a difference is found, it stores the image in a `keyframe` folder and updates the reference image. 36 | 37 | The `frontend` service is a web frontend that allows to browse the data: Its main page lists the currently tracked webpages, clicking on a webpage shows all the saved keyframes for this webpage. Users can start tracking a new webpage by entering its URL, which will add a new entity to the Cloud Datastore database. 38 | 39 | ## Potential Improvements 40 | 41 | * tooling to deploy from scratch (Deployment Manager?) 42 | * app to clean up the "staging" bucket 43 | * User can delete a particular keyframe, from the UI. 44 | * Basic styling of the frontend (use https://material.io/components/web/) 45 | * Hangouts Chat notifier function 46 | * User can delete a website 47 | * Add a back action on the website details page 48 | * make screenshot screen resolution configurable 49 | 50 | ## License 51 | 52 | [Apache 2.0](./LICENSE) 53 | -------------------------------------------------------------------------------- /frontend/.eslintrc.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Google LLC 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License 13 | extends: eslint:recommended 14 | env: 15 | node: true 16 | parserOptions: 17 | ecmaVersion: 8 18 | -------------------------------------------------------------------------------- /frontend/.gcloudignore: -------------------------------------------------------------------------------- 1 | .gcloudignore 2 | node_modules/ -------------------------------------------------------------------------------- /frontend/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /frontend/Readme.md: -------------------------------------------------------------------------------- 1 | # Website history web frontend 2 | 3 | Express.js web frontend to display website screenshots 4 | 5 | ## Dependencies 6 | 7 | - Google App Engine 8 | - Cloud Datastore 9 | - Cloud Storage 10 | 11 | Make all images in the GCS bucket public, with: `gsutil iam ch allUsers:objectViewer gs://[BUCKET_NAME]` 12 | 13 | ## Development 14 | 15 | Run locally with pretty printed bunyan logs with `npm run dev`. 16 | 17 | ## Deploy 18 | 19 | `npm run deploy` 20 | 21 | ## Notes 22 | 23 | To run this locally you will need to set the following ENVARS 24 | 25 | * GCLOUD\_PROJECT 26 | - Name of your project 27 | * GOOGLE\_APPLICATION\_CREDENTIALS 28 | - Path to local service account crednetials 29 | - more details at https://cloud.google.com/docs/authentication/getting-started 30 | * BUCKET_NAME 31 | - name of the bucket 32 | 33 | ## TODO 34 | 35 | -------------------------------------------------------------------------------- /frontend/app.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | 'use strict'; 18 | 19 | const bodyParser = require('body-parser'); 20 | const cookieParser = require('cookie-parser'); 21 | const express = require('express'); 22 | // const favicon = require('serve-favicon'); 23 | const path = require('path'); 24 | 25 | const index = require('./routes/index'); 26 | const logger = require('./logger'); 27 | const websites = require('./routes/websites'); 28 | 29 | const app = express(); 30 | 31 | // view engine setup 32 | app.set('views', path.join(__dirname, 'views')); 33 | app.set('view engine', 'jade'); 34 | 35 | // uncomment after placing your favicon in /public 36 | //app.use(favicon(path.join(__dirname, 'public', 'favicon.ico'))); 37 | app.use(bodyParser.json()); 38 | app.use(bodyParser.urlencoded({ extended: false })); 39 | app.use(cookieParser()); 40 | app.use(express.static(path.join(__dirname, 'public'))); 41 | 42 | app.use('/', index); 43 | app.use('/website', websites); 44 | 45 | // catch 404 and forward to error handler 46 | app.use(function(req, res, next) { 47 | var err = new Error(`${req.originalUrl} Not Found`); 48 | err.status = 404; 49 | logger.warn(err); 50 | next(err); 51 | }); 52 | 53 | // error handler 54 | app.use(function(err, req, res) { 55 | // set locals, only providing error in development 56 | res.locals.message = err.message; 57 | res.locals.error = req.app.get('env') === 'development' ? err : {}; 58 | 59 | // log the error 60 | if (err.status && err.status !== 404) logger.error(err); 61 | // render the error page 62 | res.status(err.status || 500); 63 | res.render('error'); 64 | }); 65 | 66 | module.exports = app; 67 | -------------------------------------------------------------------------------- /frontend/app.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Google LLC 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License 13 | 14 | runtime: nodejs8 15 | instance_class: F4_1G 16 | 17 | handlers: 18 | - url: /stylesheets 19 | static_dir: public/stylesheets 20 | secure: always 21 | redirect_http_response_code: 301 22 | - url: /scripts 23 | static_dir: public/scripts 24 | secure: always 25 | redirect_http_response_code: 301 26 | - url: /.* 27 | script: 'auto' 28 | secure: always 29 | redirect_http_response_code: 301 30 | 31 | env_variables: 32 | BUCKET_NAME: screenshots-microservice-demo -------------------------------------------------------------------------------- /frontend/bin/www: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | /** 4 | * Copyright 2018, Google LLC 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | // Stackdriver APM 19 | require('@google-cloud/trace-agent').start(); 20 | require('@google-cloud/debug-agent').start({allowExpressions: true}); 21 | require('@google-cloud/profiler').start(); 22 | 23 | /** 24 | * Module dependencies. 25 | */ 26 | var debug = require('debug')('frontend:server'); 27 | var http = require('http'); 28 | 29 | var app = require('../app'); 30 | const logger = require('../logger'); 31 | 32 | /** 33 | * Get port from environment and store in Express. 34 | */ 35 | 36 | var port = normalizePort(process.env.PORT || '3000'); 37 | app.set('port', port); 38 | 39 | /** 40 | * Create HTTP server. 41 | */ 42 | 43 | var server = http.createServer(app); 44 | 45 | /** 46 | * Listen on provided port, on all network interfaces. 47 | */ 48 | 49 | server.listen(port); 50 | server.on('error', onError); 51 | server.on('listening', onListening); 52 | 53 | /** 54 | * Normalize a port into a number, string, or false. 55 | */ 56 | 57 | function normalizePort(val) { 58 | var port = parseInt(val, 10); 59 | 60 | if (isNaN(port)) { 61 | // named pipe 62 | return val; 63 | } 64 | 65 | if (port >= 0) { 66 | // port number 67 | return port; 68 | } 69 | 70 | return false; 71 | } 72 | 73 | /** 74 | * Event listener for HTTP server "error" event. 75 | */ 76 | 77 | function onError(error) { 78 | if (error.syscall !== 'listen') { 79 | throw error; 80 | } 81 | 82 | var bind = typeof port === 'string' 83 | ? 'Pipe ' + port 84 | : 'Port ' + port; 85 | 86 | // handle specific listen errors with friendly messages 87 | switch (error.code) { 88 | case 'EACCES': 89 | logger.error(bind + ' requires elevated privileges'); 90 | process.exit(1); 91 | break; 92 | case 'EADDRINUSE': 93 | logger.error(bind + ' is already in use'); 94 | process.exit(1); 95 | break; 96 | default: 97 | throw error; 98 | } 99 | } 100 | 101 | /** 102 | * Event listener for HTTP server "listening" event. 103 | */ 104 | 105 | function onListening() { 106 | var addr = server.address(); 107 | var bind = typeof addr === 'string' 108 | ? 'pipe ' + addr 109 | : 'port ' + addr.port; 110 | debug('Listening on ' + bind); 111 | } 112 | -------------------------------------------------------------------------------- /frontend/logger/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018, Google LLC 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | 'use strict'; 17 | 18 | // [START logging_bunyan_quickstart] 19 | const bunyan = require('bunyan'); 20 | 21 | // Imports the Google Cloud client library for Bunyan 22 | const LoggingBunyan = require('@google-cloud/logging-bunyan'); 23 | 24 | // Creates a Bunyan Stackdriver Logging client 25 | const loggingBunyan = new LoggingBunyan(); 26 | 27 | // Create a Bunyan logger that streams to Stackdriver Logging 28 | // Logs will be written to: "projects/YOUR_PROJECT_ID/logs/bunyan_log" 29 | const logger = bunyan.createLogger({ 30 | // The JSON payload of the log as it appears in Stackdriver Logging 31 | // will contain "name": "my-service" 32 | name: 'frontend', 33 | streams: [ 34 | // Log to the console at 'info' and above 35 | {stream: process.stdout, level: 'info'}, 36 | // And log to Stackdriver Logging, logging at 'info' and above 37 | loggingBunyan.stream('info'), 38 | ], 39 | }); 40 | 41 | module.exports = logger; 42 | -------------------------------------------------------------------------------- /frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "frontend", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "start": "node ./bin/www", 7 | "deploy": "gcloud app deploy --quiet", 8 | "test": "eslint app.js routes/ logger/ bin/www", 9 | "dev": "npm start | bunyan" 10 | }, 11 | "author": { 12 | "name": "Google LLC" 13 | }, 14 | "license": "Apache-2.0", 15 | "dependencies": { 16 | "@google-cloud/datastore": "^1.4.0", 17 | "@google-cloud/debug-agent": "^2.6.0", 18 | "@google-cloud/logging-bunyan": "^0.7.0", 19 | "@google-cloud/profiler": "^0.1.14", 20 | "@google-cloud/storage": "^1.6.0", 21 | "@google-cloud/trace-agent": "^2.10.1", 22 | "body-parser": "^1.18.3", 23 | "bunyan": "^1.8.12", 24 | "cookie-parser": "~1.4.3", 25 | "debug": "~2.6.3", 26 | "express": "^4.16.3", 27 | "jade": "~1.11.0", 28 | "material-components-web": "^0.35.0", 29 | "serve-favicon": "~2.4.2" 30 | }, 31 | "devDependencies": { 32 | "eslint": "^4.19.1" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /frontend/public/scripts/material-components-web.js: -------------------------------------------------------------------------------- 1 | ../../node_modules/material-components-web/dist/material-components-web.js -------------------------------------------------------------------------------- /frontend/public/stylesheets/material-components-web.css: -------------------------------------------------------------------------------- 1 | ../../node_modules/material-components-web/dist/material-components-web.css -------------------------------------------------------------------------------- /frontend/public/stylesheets/style.css: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018, Google LLC 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | body { 17 | padding: 50px; 18 | font: 14px "Lucida Grande", Helvetica, Arial, sans-serif; 19 | } 20 | 21 | a { 22 | color: #00B7FF; 23 | } 24 | 25 | #header { 26 | text-align: center; 27 | } 28 | 29 | #form-title { 30 | text-align: center; 31 | } 32 | 33 | #form-input { 34 | padding: 2em; 35 | } 36 | 37 | #tracking-card { 38 | padding: 2em; 39 | } 40 | -------------------------------------------------------------------------------- /frontend/routes/index.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 'use strict'; 17 | 18 | const express = require('express'); 19 | const Datastore = require('@google-cloud/datastore'); 20 | 21 | const logger = require('../logger'); 22 | 23 | const router = express.Router(); 24 | const datastore = new Datastore(); 25 | 26 | /* GET home page. */ 27 | router.get('/', async (req, res, next) => { 28 | const query = datastore.createQuery('Website'); 29 | let results; 30 | try { 31 | results = await datastore.runQuery(query); 32 | res.render('index', { websites: results[0] }); 33 | } 34 | catch (err) { 35 | next(err); 36 | return; 37 | } 38 | logger.info('Successfuly got Website data from datastore'); 39 | }); 40 | 41 | module.exports = router; 42 | -------------------------------------------------------------------------------- /frontend/routes/websites.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 'use strict'; 17 | 18 | const express = require('express'); 19 | const Datastore = require('@google-cloud/datastore'); 20 | const Storage = require('@google-cloud/storage'); 21 | 22 | const logger = require('../logger'); 23 | 24 | const router = express.Router(); 25 | const storage = new Storage(); 26 | const datastore = new Datastore(); 27 | 28 | /* displays a new message, creates it if needed */ 29 | router.get('/', async (req, res, next) => { 30 | let url = req.query.url; 31 | if(!url) { 32 | logger.info('400: URL not provided'); 33 | return res.status(400).send('Please provide a URL.'); 34 | } 35 | 36 | let website; 37 | 38 | logger.info(`Looking for website ${url}`); 39 | 40 | // Look for this website 41 | // TODO: use some memcache 42 | const query = datastore 43 | .createQuery('Website') 44 | .filter('url', '=', url); 45 | let results; 46 | try { 47 | results = await datastore.runQuery(query); 48 | } 49 | catch (err) { 50 | next(err); 51 | return; 52 | } 53 | if(results[0] && results[0].length > 0) { 54 | // website found 55 | website = results[0][0]; 56 | logger.info(`Website found: ${website.url}`); 57 | } else { 58 | logger.info(`Website not found: ${url}`); 59 | // website does not exist, create it 60 | 61 | //make sure it starts with an http or https protocol 62 | if(!url.startsWith('http://') && !url.startsWith('https://')) { 63 | url = 'http://' + url; 64 | } 65 | 66 | const taskKey = datastore.key('Website'); 67 | const newWebsite = { 68 | key: taskKey, 69 | data: { 70 | url: url, 71 | }, 72 | }; 73 | try { 74 | await datastore.save(newWebsite); 75 | } 76 | catch (err) { 77 | next(err); 78 | return; 79 | } 80 | logger.info(`New website saved: ${url}`); 81 | website = newWebsite.data; 82 | } 83 | 84 | // retrieve keyframes 85 | // TODO should we have a sane default if this fails? 86 | const bucketName = process.env.BUCKET_NAME; 87 | logger.info(`bucketName: ${bucketName}`); 88 | const urlPath = url.replace(/[^a-z0-9]/gi, '_').toLowerCase(); 89 | const pathPrefix = 'keyframes'; 90 | const options = { 91 | prefix: `${pathPrefix}/${urlPath}/`, 92 | delimiter: '/' 93 | }; 94 | let gcsResults; 95 | try { 96 | gcsResults = await storage.bucket(bucketName).getFiles(options) 97 | } 98 | catch (err) { 99 | next(err); 100 | return; 101 | } 102 | const files = gcsResults[0]; 103 | files.map( file => { 104 | file.url = ` https://storage.googleapis.com/${bucketName}/${file.name}`; 105 | }) 106 | logger.info(`${files.length} keyframes found for ${url}`) 107 | 108 | res.render('website', { website, keyframes: files }); 109 | }); 110 | 111 | module.exports = router; 112 | -------------------------------------------------------------------------------- /frontend/views/error.jade: -------------------------------------------------------------------------------- 1 | extends layout 2 | 3 | block content 4 | h1(class="mdc-typography--headline1")= message 5 | h2(class="mdc-typography--headline2")= error.status 6 | pre #{error.stack} 7 | -------------------------------------------------------------------------------- /frontend/views/index.jade: -------------------------------------------------------------------------------- 1 | extends layout 2 | 3 | block content 4 | div(id="header") 5 | h1(class="mdc-typography--headline1") web page watcher 6 | 7 | div(class="mdc-layout-grid") 8 | div(class="mdc-layout-grid__inner") 9 | 10 | div(class="mdc-layout-grid__cell--span-6") 11 | form(id="form-input" action="/website" method="GET" class="mdc-text-field") 12 | input(type="text" name="url" class="mdc-text-field__input") 13 | input(type="submit" class="mdc-button") 14 | 15 | div(id="tracking-card" class="mdc-layout-grid__cell--span-12 mdc-card") 16 | ol(class="mdc-list") 17 | each website in websites 18 | li(class="mdc-list-item") 19 | a(href="/website?url=#{website.url}")=website.url 20 | -------------------------------------------------------------------------------- /frontend/views/layout.jade: -------------------------------------------------------------------------------- 1 | doctype html 2 | html 3 | head 4 | title= title 5 | link(rel='stylesheet', href='/stylesheets/material-components-web.css') 6 | link(rel='stylesheet', href='/stylesheets/style.css') 7 | body(class="mdc-typography") 8 | block content 9 | script(src="/scripts/material-components-web.js") 10 | script 11 | | mdc.autoInit(); 12 | -------------------------------------------------------------------------------- /frontend/views/website.jade: -------------------------------------------------------------------------------- 1 | extends layout 2 | 3 | block content 4 | h1(class="mdc-typography--headline2")= website.url 5 | h2(class="mdc-typography--headline3") History 6 | each keyframe in keyframes 7 | img(src=keyframe.url) -------------------------------------------------------------------------------- /image-diff/.eslintrc.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Google LLC 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License 13 | extends: eslint:recommended 14 | env: 15 | node: true 16 | es6: true 17 | -------------------------------------------------------------------------------- /image-diff/.gcloudignore: -------------------------------------------------------------------------------- 1 | .gcloudignore 2 | .git 3 | .gitignore 4 | node_modules 5 | -------------------------------------------------------------------------------- /image-diff/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /image-diff/Readme.md: -------------------------------------------------------------------------------- 1 | # Image Diff Function 2 | 3 | Simple function that compares any new image uploaded to GCS with a reference image 4 | 5 | ## Dependencies 6 | 7 | - a GCS bucket, create one with `gsutil mb gs://[GLOBALLY_UNIQUE_BUCKET_NAME]` (and update the deploy command in `package.json` to reflect this) 8 | - Enable the Cloud Functions API 9 | 10 | ## Deploy 11 | 12 | `npm run deploy` 13 | 14 | ## How it works 15 | 16 | This function listen for new files created in the given Cloud Storage bucket. 17 | If a new file is detected in a `screenshots//` folder, then it will compare this image with a potential reference image `references//ref.png`. 18 | If a difference is found, then the reference is updated and the image is copied to a `keyframes//` folder. -------------------------------------------------------------------------------- /image-diff/diff.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | const pixelmatch = require('pixelmatch'); 17 | const sharp = require('sharp'); 18 | 19 | function diff(oldImage, newImage, length, width, callback) { 20 | // Number of pixels that have to mismatch to consider the images different 21 | const diffPixelThreshold = 100 * 10; // at least a change equivalent to 100x10px 22 | 23 | const diffBuffer = Buffer.alloc(oldImage.length); 24 | Promise.all([sharp(oldImage).toBuffer(), sharp(newImage).toBuffer()]).then(images => { 25 | const diffPixelCount = pixelmatch(images[0], images[1], diffBuffer, length, width, { 26 | includeAA: true, // Ignore Anti-aliasing in diff 27 | threshold: 0.3, // be less sensitive than default (0.1) to determine if a pixel has changed. 28 | }); 29 | if (diffPixelCount < diffPixelThreshold) return callback(null, false); 30 | callback(null, diffPixelCount, diffBuffer); 31 | }); 32 | } 33 | 34 | module.exports = diff; 35 | -------------------------------------------------------------------------------- /image-diff/index.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | /* eslint no-console: "off" */ 18 | const traceApi = require('@google-cloud/trace-agent').start({ 19 | serviceContext: { 20 | service: 'image-diff' 21 | } 22 | }); 23 | const Storage = require('@google-cloud/storage'); 24 | const debug = require('@google-cloud/debug-agent').start(); 25 | require('@google-cloud/profiler').start(); 26 | 27 | 28 | const diff = require('./diff'); 29 | 30 | const storage = new Storage(); 31 | 32 | /** 33 | * Generic background Cloud Function to be triggered by Cloud Storage. 34 | * 35 | * @param {object} event The Cloud Functions event. 36 | * @param {function} callback The callback function. 37 | */ 38 | exports.imageDiff = (event, callback) => { 39 | const debugReady = debug.isReady(); 40 | debugReady.then(() => { 41 | 42 | const file = event.data; 43 | 44 | if(!file.name.startsWith('screenshots/')) { 45 | console.log(`${file.name} is not a screenshot`); 46 | return callback(); 47 | } 48 | 49 | // extract URL from file: 50 | var urlFolder; 51 | try { 52 | urlFolder = file.name.match(/\/([a-z0-9_]+)\//)[1] 53 | } catch(e) { 54 | console.error(`Cannot extract url folder for ${file.name}`); 55 | callback(); 56 | } 57 | console.log(`Evaluating ${urlFolder}`); 58 | 59 | // The received image: 60 | const imageFile = storage.bucket(file.bucket).file(file.name); 61 | // The reference for this URL: 62 | const referenceImageFile = storage.bucket(file.bucket).file(`references/${urlFolder}/ref.png`); 63 | 64 | // Check that the reference exists 65 | referenceImageFile.exists().then((data) => { 66 | const exists = data[0]; 67 | if(!exists) { 68 | console.log(`Could not find reference for ${urlFolder}`); 69 | storeAsKeyframeAndReference(urlFolder, imageFile); 70 | } else { 71 | 72 | // Compare images 73 | const compareSpan = traceApi.createChildSpan({name: 'compare'}); 74 | areImagesDifferent(referenceImageFile, imageFile, (err, numPixel) => { 75 | compareSpan.endSpan(); 76 | if(numPixel) { 77 | console.log(`Difference found for ${urlFolder}: ${numPixel} pixel differ.`); 78 | // images are different: 79 | storeAsKeyframeAndReference(urlFolder, imageFile, callback); 80 | } else { 81 | console.log(`No difference found for ${urlFolder}`); 82 | callback(); 83 | } 84 | }); 85 | } 86 | }); 87 | }); 88 | }; 89 | 90 | function storeAsKeyframeAndReference(urlFolder, file, callback) { 91 | const storeSpan = traceApi.createChildSpan({name: 'store'}); 92 | Promise.all([ 93 | file.copy(`references/${urlFolder}/ref.png`), 94 | file.copy(file.name.replace('screenshots/', 'keyframes/')) 95 | ]).then((/* copies */) => { 96 | storeSpan.endSpan(); 97 | callback(); 98 | }); 99 | } 100 | 101 | function areImagesDifferent(referenceFile, file, callback) { 102 | console.log(`Comparing images: ${referenceFile.name} and ${file.name}`); 103 | // Load both images 104 | Promise.all([ 105 | file.download(), 106 | referenceFile.download() 107 | ]).then((contents) => { 108 | var fileContent = contents[0][0]; 109 | var referenceFileContent = contents[1][0]; 110 | 111 | diff(fileContent, referenceFileContent, 1680, 1050, callback); 112 | }); 113 | } -------------------------------------------------------------------------------- /image-diff/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "imageDiff", 3 | "version": "1.0.0", 4 | "description": "Simple function to compare two images from GCS", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "npm run lint && npm run tap", 8 | "tap": "tap test/test-*.js", 9 | "lint": "eslint *.js test/", 10 | "deploy": "gcloud beta functions deploy imageDiff --trigger-resource screenshots-microservice-demo --trigger-event google.storage.object.finalize" 11 | }, 12 | "author": { 13 | "name": "Google LLC" 14 | }, 15 | "license": "Apache-2.0", 16 | "devDependencies": { 17 | "eslint": "^4.19.1", 18 | "tap": "^11.1.3" 19 | }, 20 | "dependencies": { 21 | "@google-cloud/debug-agent": "^2.6.0", 22 | "@google-cloud/profiler": "^0.1.14", 23 | "@google-cloud/storage": "^1.6.0", 24 | "@google-cloud/trace-agent": "^2.10.1", 25 | "pixelmatch": "^4.0.2", 26 | "sharp": "^0.20.1" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /image-diff/test/one.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/nodejs-serverless-microservices-demo/a1ffd1bd4e1b614229a109b4b7daa87b95fc4540/image-diff/test/one.jpg -------------------------------------------------------------------------------- /image-diff/test/test-diff.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | const fs = require('fs'); 17 | const path = require('path'); 18 | 19 | const tap = require('tap').test; 20 | 21 | const diff = require('../diff.js'); 22 | 23 | const one = fs.readFileSync(path.resolve(__dirname, 'one.jpg')); 24 | const two = fs.readFileSync(path.resolve(__dirname, 'two.jpg')); 25 | 26 | tap('the same image should return true', (t) => { 27 | t.plan(4); 28 | diff(one, one, 400, 400, (err, difference, buffer) => { 29 | t.notok(difference, 'there should be no pixel diff'); 30 | t.notok(buffer, 'there should be no diff buffer returned'); 31 | }); 32 | diff(two, two, 400, 400, (err, difference, buffer) => { 33 | t.notok(difference, 'there should be no pixel diff'); 34 | t.notok(buffer, 'there should be no diff buffer returned'); 35 | }); 36 | }); 37 | 38 | tap('different images should return true', (t) => { 39 | t.plan(4); 40 | diff(one, two, 400, 400, (err, difference, buffer) => { 41 | t.ok(difference, 'there should be a pixel difference'); 42 | t.ok(buffer && buffer.length, 'there should be a buffer returned with difference'); 43 | }); 44 | diff(two, one, 400, 400, (err, difference, buffer) => { 45 | t.ok(difference, 'there should be a pixel difference'); 46 | t.ok(buffer && buffer.length, 'there should be a buffer returned with difference'); 47 | }); 48 | }); 49 | -------------------------------------------------------------------------------- /image-diff/test/two.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/nodejs-serverless-microservices-demo/a1ffd1bd4e1b614229a109b4b7daa87b95fc4540/image-diff/test/two.jpg -------------------------------------------------------------------------------- /screenshot/.eslintrc.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Google LLC 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License 13 | extends: eslint:recommended 14 | env: 15 | node: true 16 | parserOptions: 17 | ecmaVersion: 8 18 | -------------------------------------------------------------------------------- /screenshot/.gcloudignore: -------------------------------------------------------------------------------- 1 | .gcloudignore 2 | node_modules/ -------------------------------------------------------------------------------- /screenshot/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /screenshot/Readme.md: -------------------------------------------------------------------------------- 1 | # screenshot 2 | 3 | A Node.js microservice that takes screenshots of a given URL and stores it in Google Cloud Storage. 4 | 5 | ## Dependencies 6 | 7 | - A Google Cloud Storage bucket 8 | 9 | ## Usage 10 | 11 | Start with `npm start` 12 | 13 | Capture a screenshot with `\`. 14 | 15 | ## Development 16 | 17 | Run locally with pretty printed bunyan logs with `npm run dev`. 18 | 19 | ## Deploy 20 | 21 | `npm run deploy` 22 | 23 | ## Notes 24 | 25 | To run this locally you will need to set the following ENVARS 26 | 27 | * GCLOUD\_PROJECT 28 | - Name of your project 29 | * GOOGLE\_APPLICATION\_CREDENTIALS 30 | - Path to local service account crednetials 31 | - more details at https://cloud.google.com/docs/authentication/getting-started 32 | * BUCKET_NAME 33 | - name of the bucket 34 | -------------------------------------------------------------------------------- /screenshot/app.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Google LLC 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License 13 | 14 | service: screenshot 15 | 16 | runtime: nodejs8 17 | 18 | instance_class: F4_1G 19 | 20 | env_variables: 21 | SCREENSHOT_BUCKET_NAME: screenshots-microservice-demo 22 | # see https://github.com/GoogleCloudPlatform/cloud-trace-nodejs#tracing-with-asyncawait 23 | GCLOUD_TRACE_NEW_CONTEXT: 1 24 | 25 | handlers: 26 | - url: /.* 27 | script: 'auto' 28 | secure: always 29 | redirect_http_response_code: 301 -------------------------------------------------------------------------------- /screenshot/logger/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018, Google LLC 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | 'use strict'; 17 | 18 | // [START logging_bunyan_quickstart] 19 | const bunyan = require('bunyan'); 20 | 21 | // Imports the Google Cloud client library for Bunyan 22 | const LoggingBunyan = require('@google-cloud/logging-bunyan'); 23 | 24 | // Creates a Bunyan Stackdriver Logging client 25 | const loggingBunyan = new LoggingBunyan(); 26 | 27 | // Create a Bunyan logger that streams to Stackdriver Logging 28 | // Logs will be written to: "projects/YOUR_PROJECT_ID/logs/bunyan_log" 29 | const logger = bunyan.createLogger({ 30 | // The JSON payload of the log as it appears in Stackdriver Logging 31 | // will contain "name": "my-service" 32 | name: 'screenshot', 33 | streams: [ 34 | // Log to the console at 'info' and above 35 | {stream: process.stdout, level: 'debug'}, 36 | // And log to Stackdriver Logging, logging at 'info' and above 37 | loggingBunyan.stream('info'), 38 | ], 39 | }); 40 | 41 | module.exports = logger; 42 | -------------------------------------------------------------------------------- /screenshot/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "screenshot", 3 | "engines": { 4 | "node": "8.x" 5 | }, 6 | "version": "1.0.0", 7 | "description": "Takes screenshot of given URL", 8 | "scripts": { 9 | "start": "node server.js", 10 | "deploy": "gcloud app deploy --quiet", 11 | "test": "eslint *.js logger/", 12 | "dev": "npm start | bunyan" 13 | }, 14 | "author": { 15 | "name": "Google LLC" 16 | }, 17 | "license": "Apache-2.0", 18 | "dependencies": { 19 | "@google-cloud/debug-agent": "^2.5.1", 20 | "@google-cloud/logging-bunyan": "^0.7.0", 21 | "@google-cloud/profiler": "^0.1.14", 22 | "@google-cloud/storage": "^1.6.0", 23 | "@google-cloud/trace-agent": "^2.10.1", 24 | "body-parser": "^1.18.2", 25 | "bunyan": "^1.8.12", 26 | "express": "^4.16.3", 27 | "puppeteer": "^1.2.0" 28 | }, 29 | "devDependencies": { 30 | "eslint": "^4.19.1" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /screenshot/server.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 'use strict'; 17 | 18 | // the following is a workaround: 19 | // Cloud Storage library tries to write in /home/ when uploading a buffer 20 | process.env.HOME = '/tmp'; 21 | 22 | // Stackdriver APM 23 | const traceApi = require('@google-cloud/trace-agent').start(); 24 | require('@google-cloud/debug-agent').start({allowExpressions: true}); 25 | require('@google-cloud/profiler').start(); 26 | 27 | const bodyParser = require('body-parser'); 28 | const express = require('express'); 29 | const puppeteer = require('puppeteer'); 30 | const Storage = require('@google-cloud/storage'); 31 | 32 | const logger = require('./logger'); 33 | 34 | const storage = new Storage(); 35 | 36 | const app = express(); 37 | app.use(bodyParser.json()); 38 | app.use(bodyParser.urlencoded({ extended: false })); 39 | 40 | // Ignore Favicon 41 | app.use((req, res, next) => { 42 | if (req.originalUrl === '/favicon.ico') { 43 | res.status(204).json({nope: true}); 44 | return; 45 | } 46 | next(); 47 | }); 48 | 49 | function getUrl(req) { 50 | let url; 51 | if(!req.body.message) { 52 | url = req.query.url; 53 | return url; 54 | } 55 | const parsedPayload = JSON.parse(Buffer.from(req.body.message.data, 'base64').toString('utf-8')) 56 | url = parsedPayload.url; 57 | return url; 58 | } 59 | 60 | async function startBrowser(url) { 61 | const spanChrome = traceApi.createChildSpan({name: 'launch-chrome'}); 62 | const browser = await puppeteer.launch({ 63 | headless: true, 64 | timeout: 90000, 65 | args: ['--no-sandbox', '--disable-setuid-sandbox'] 66 | }); 67 | spanChrome.endSpan(); 68 | const spanPage = traceApi.createChildSpan({name: 'new-page'}); 69 | const page = await browser.newPage(); 70 | spanPage.endSpan(); 71 | const spanVisit = traceApi.createChildSpan({name: 'goto-page'}); 72 | await page.goto(url); 73 | await page.setViewport({ 74 | width: 1680, 75 | height: 1050 76 | }); 77 | spanVisit.endSpan(); 78 | return [browser, page]; 79 | } 80 | 81 | async function hideCursor(page) { 82 | // Custom CSS to avoid capturing blinking cursors when input fields have focus 83 | const hideInputTextCSS = ` 84 | input { 85 | color: transparent; 86 | text-shadow: 0 0 0 black; 87 | } 88 | input:focus { 89 | outline: none; 90 | } 91 | `; 92 | await page.addStyleTag({ content: hideInputTextCSS }); 93 | } 94 | 95 | async function takeScreenshot(url) { 96 | const span = traceApi.createChildSpan({name: 'screenshot'}); 97 | let browser; 98 | let page; 99 | [browser, page] = await startBrowser(url); 100 | 101 | 102 | await hideCursor(page); 103 | 104 | logger.info(`URL: ${url} - starting screenshot`); 105 | const imageBuffer = await page.screenshot(); 106 | logger.info(`URL: ${url} - screenshot taken`); 107 | 108 | await browser.close(); 109 | span.endSpan(); 110 | return imageBuffer; 111 | } 112 | 113 | async function saveToBucket(imageBuffer, url) { 114 | // Uploads a local file to the bucket 115 | 116 | const span = traceApi.createChildSpan({name: 'save'}); 117 | logger.info(`URL: ${url} - saving screenshot to GCS bucket: ${process.env.SCREENSHOT_BUCKET_NAME}`); 118 | 119 | const bucketName = process.env.SCREENSHOT_BUCKET_NAME; 120 | const date = new Date(); 121 | const timestamp = date.getTime(); 122 | const filename = `${timestamp}.png`; 123 | const filepath = url.replace(/[^a-z0-9]/gi, '_').toLowerCase(); 124 | 125 | const bucket = storage.bucket(bucketName); 126 | 127 | const file = bucket.file(`screenshots/${filepath}/${filename}`); 128 | 129 | await file.save(imageBuffer); 130 | 131 | logger.info(`URL: ${url} - screenshot saved`); 132 | span.endSpan(); 133 | } 134 | 135 | app.use(async (req, res, next) => { 136 | const url = getUrl(req); 137 | 138 | if(!url) { 139 | return res.status(400).send('Please provide URL as GET parameter or in POST body, example: ?url=http://example.com'); 140 | } 141 | 142 | // make sure the URL starts with a protocol 143 | if(!url.startsWith('http')) {return res.status(400).send('URL must start with http:// or https://');} 144 | 145 | logger.debug(`URL: ${url}`); 146 | 147 | try { 148 | const imageBuffer = await takeScreenshot(url); 149 | await saveToBucket(imageBuffer, url); 150 | // returns the screenshot 151 | res.set('Content-Type', 'image/png') 152 | res.send(imageBuffer); 153 | } 154 | catch (err) { 155 | next(err); 156 | return; 157 | } 158 | }); 159 | 160 | // error handler 161 | app.use(function(err, req, res) { 162 | // set locals, only providing error in development 163 | res.locals.message = err.message; 164 | res.locals.error = req.app.get('env') === 'development' ? err : {}; 165 | 166 | // log the error 167 | logger.error(err); 168 | 169 | // render the error page 170 | res.status(err.status || 500); 171 | res.end(err.message); 172 | }); 173 | 174 | const server = app.listen(process.env.PORT || 8080, err => { 175 | if (err) return logger.error(err); 176 | const port = server.address().port; 177 | logger.info(`App listening on port ${port}`) 178 | }); -------------------------------------------------------------------------------- /task-scheduler/.eslintrc.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Google LLC 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License 13 | extends: eslint:recommended 14 | env: 15 | node: true 16 | parserOptions: 17 | ecmaVersion: 8 18 | -------------------------------------------------------------------------------- /task-scheduler/.gcloudignore: -------------------------------------------------------------------------------- 1 | .gcloudignore 2 | .git 3 | .gitignore 4 | node_modules/ -------------------------------------------------------------------------------- /task-scheduler/LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /task-scheduler/README.md: -------------------------------------------------------------------------------- 1 | # Schedule screenshot tasks 2 | 3 | When executed, this microservice looks for screenshots to grab and stores them as tasks in the "screenshot" queue. 4 | 5 | ## Required environment variables: 6 | 7 | - GOOGLE_CLOUD_PROJECT: Project ID 8 | - QUEUE_ID: the queue name to creates tasks in. 9 | - QUEUE_LOCATION: region of the queue 10 | - TARGET_SERVICE: name of the service that takes screenshots 11 | - TOPIC_NAME: Name of the Pub/Sub topic to send events to. 12 | - SUBSCRIPTION_NAME: Name of the Pub/Sub subscription 13 | 14 | ## Dependencies 15 | 16 | Create a Pub/Sub topic with `gcloud pubsub topics create $TOPIC_NAME`. 17 | 18 | Create a Pub/Sub subscription with: 19 | ``` 20 | gcloud beta pubsub subscriptions create $SUBSCRIPTION_NAME \ 21 | --topic $TOPIC_NAME \ 22 | --push-endpoint \ 23 | https://$TARGET_SERVICE-dot-$GOOGLE_CLOUD_PROJECT.appspot.com/ \ 24 | --ack-deadline 30 25 | ``` 26 | 27 | ## Development 28 | 29 | Run locally with pretty printed bunyan logs with `npm run dev`. 30 | 31 | ## Deployment 32 | 33 | Deploy the service with `npm run deploy`. 34 | 35 | ## Scheduling this microservice 36 | 37 | Schedule the execution of this microservice with `gcloud app deploy cron.yaml`. 38 | 39 | ## Development 40 | 41 | Make sure to have the proper env vars in place, then start local development with `npm start`. 42 | -------------------------------------------------------------------------------- /task-scheduler/app.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Google LLC 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License 13 | 14 | service: task-scheduler 15 | runtime: nodejs8 16 | 17 | # Replace these with yours 18 | env_variables: 19 | GOOGLE_CLOUD_PROJECT: 'nodejs-microservices-demo' 20 | QUEUE_ID: 'screenshots' 21 | QUEUE_LOCATION: 'us-central1' 22 | TARGET_SERVICE: 'screenshot' 23 | TOPIC_NAME: 'screenshots' 24 | SUBSCRIPTION_NAME: 'screenshots' -------------------------------------------------------------------------------- /task-scheduler/cron.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2018, Google LLC 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License 13 | 14 | cron: 15 | - description: "look for webpages to screenshot" 16 | url: / 17 | schedule: every 5 minutes 18 | target: task-scheduler -------------------------------------------------------------------------------- /task-scheduler/lib/create-screenshot-event.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018, Google LLC 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | const PubSub = require('@google-cloud/pubsub'); 17 | const logger = require('./../logger'); 18 | 19 | const pubsub = PubSub(); 20 | const topic = pubsub.topic(process.env.TOPIC_NAME); 21 | const publisher = topic.publisher(); 22 | 23 | module.exports = async (url) => { 24 | logger.info(`Sending task: URL: ${url}`); 25 | try { 26 | const data = JSON.stringify({ url }); 27 | const dataBuffer = Buffer.from(data); 28 | await publisher.publish(dataBuffer); 29 | logger.info(`Task sent: URL: ${url}`); 30 | } catch(err) { 31 | logger.error(err); 32 | return; 33 | } 34 | } -------------------------------------------------------------------------------- /task-scheduler/logger/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018, Google, Inc. 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | 'use strict'; 17 | 18 | // [START logging_bunyan_quickstart] 19 | const bunyan = require('bunyan'); 20 | 21 | // Imports the Google Cloud client library for Bunyan 22 | const LoggingBunyan = require('@google-cloud/logging-bunyan'); 23 | 24 | // Creates a Bunyan Stackdriver Logging client 25 | const loggingBunyan = new LoggingBunyan(); 26 | 27 | // Create a Bunyan logger that streams to Stackdriver Logging 28 | // Logs will be written to: "projects/YOUR_PROJECT_ID/logs/bunyan_log" 29 | const logger = bunyan.createLogger({ 30 | // The JSON payload of the log as it appears in Stackdriver Logging 31 | // will contain "name": "my-service" 32 | name: 'task-scheduler', 33 | streams: [ 34 | // Log to the console at 'info' and above 35 | {stream: process.stdout, level: 'debug'}, 36 | // And log to Stackdriver Logging, logging at 'info' and above 37 | loggingBunyan.stream('info'), 38 | ], 39 | }); 40 | 41 | module.exports = logger; 42 | -------------------------------------------------------------------------------- /task-scheduler/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "scheduler", 3 | "version": "1.0.0", 4 | "description": "Schedule screenshot task creation", 5 | "scripts": { 6 | "start": "node server.js", 7 | "deploy": "gcloud app deploy --quiet", 8 | "test": "eslint *.js logger/ lib/", 9 | "dev": "npm start | bunyan" 10 | }, 11 | "author": { 12 | "name": "Google LLC" 13 | }, 14 | "license": "Apache-2.0", 15 | "dependencies": { 16 | "@google-cloud/datastore": "^1.4.0", 17 | "@google-cloud/logging-bunyan": "^0.7.0", 18 | "@google-cloud/pubsub": "^0.18.0", 19 | "bunyan": "^1.8.12", 20 | "googleapis": "^27.0.0" 21 | }, 22 | "devDependencies": { 23 | "eslint": "^4.19.1" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /task-scheduler/server.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2018 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 'use strict'; 17 | 18 | const http = require("http"); 19 | 20 | const Datastore = require('@google-cloud/datastore'); 21 | 22 | const logger = require('./logger'); 23 | const createScreenshotTask = require('./lib/create-screenshot-event'); 24 | 25 | const datastore = new Datastore(); 26 | 27 | const server = http.createServer(async (req, res) => { 28 | logger.info('Looking for new tasks to schedule...'); 29 | 30 | const query = datastore.createQuery('Website'); 31 | 32 | let results; 33 | 34 | try { 35 | results = await datastore.runQuery(query); 36 | } 37 | catch (err) { 38 | logger.error(err); 39 | res.writeHead(500); 40 | res.end('internal server error'); 41 | return; 42 | } 43 | 44 | const websites = results[0]; 45 | 46 | logger.info(`Found ${websites.length} urls to screenshot, creating tasks...`) 47 | 48 | websites.forEach((website) => { 49 | // TODO return promise and await 50 | createScreenshotTask(`${website.url}`); 51 | }) 52 | 53 | logger.info(`${websites.length} tasks scheduled`); 54 | 55 | res.writeHead(200, {"Content-Type": "text/plain"}); 56 | res.end('done'); 57 | }); 58 | 59 | server.listen(process.env.PORT || 8080); 60 | --------------------------------------------------------------------------------