├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── composer.json ├── composer.lock ├── controllers ├── API.php ├── Main.php └── Stats.php ├── data └── .gitignore ├── jobs └── CheckFeed.php ├── lib ├── config.template.php ├── db_helpers.php └── helpers.php ├── public └── index.php ├── schema ├── 0001.sql ├── 0002.sql ├── 0003.sql ├── 0004.sql ├── 0005.php ├── 0005.sql ├── 0006.sql ├── 0007.sql ├── 0008.sql ├── 0009.sql └── schema.sql ├── scripts ├── cleanup.php ├── cron.php ├── debug.php ├── logs │ └── .gitignore ├── stats.php └── watchtower.php └── views └── index.php /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | vendor/ 3 | lib/config.php 4 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | By submitting code to this project, you agree to irrevocably release it under the same license as this project. See README.md for more details. -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Watchtower 2 | ========== 3 | 4 | Watchtower is a minimal API that watches web pages for changes and notifies subscribers. Its API is similar to [WebSub](https://www.w3.org/TR/websub/), as well as [Superfeedr subscriptions](https://documentation.superfeedr.com/subscribers.html). 5 | 6 | For HTML pages, Watchtower compares the text content with all tags removed in order to determine whether a page has changed. This prevents things like CSRF tokens from triggering a change event and redelivery of the page. For all other content types, the raw content is used to compare changes. 7 | 8 | 9 | API 10 | --- 11 | 12 | Every API request requires authenticating with an application API key included as a Bearer Token. 13 | 14 | ``` 15 | POST / HTTP/1.1 16 | Authorization: Bearer xxxxxxxxxx 17 | ``` 18 | 19 | ### Subscribing 20 | 21 | To subscribe to changes of a URL, send a POST request to the root URL with the following parameters 22 | 23 | `POST https://watchtower.example/` 24 | 25 | * `hub.mode` = `subscribe` 26 | * `hub.topic` the URL that you want to watch 27 | * `hub.callback` the subscriber's URL to be notified of changes 28 | 29 | Unlike WebSub, Watchtower does not make an initial verification request to check the callback URL. It assumes it's valid since the API request must also include the API key. 30 | 31 | Watchtower will create the subscription, then deliver the current contents of the URL to the subscriber. This happens asynchronously so it may take a few seconds after the API request. 32 | 33 | ### Unsubscribing 34 | 35 | To unsubscribe, send a POST request with the following parameters 36 | 37 | `POST https://watchtower.example/` 38 | 39 | * `hub.mode` = `unsubscribe` 40 | * `hub.topic` the URL that you want to watch 41 | * `hub.callback` the subscriber's URL to be notified of changes 42 | 43 | The subscription will be deactivated immediately. 44 | 45 | 46 | ### Web Hooks 47 | 48 | When Watchtower delivers a notification to the subscriber, it makes an HTTP POST with a content type header matching the content type of the topic, and the body of the POST is the full contents of the URL. It also includes an `Authorization` header with the API key, so that you can verify the authenticity of the API request. 49 | 50 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "require": { 3 | "j4mie/idiorm": "^1.5", 4 | "pda/pheanstalk": "^3.1", 5 | "p3k/caterpillar": "^0.1.2", 6 | "p3k/utils": "^1.0", 7 | "league/plates": "^3.3", 8 | "league/route": "^3.0", 9 | "zendframework/zend-diactoros": "^1.6", 10 | "p3k/http": "^0.1.5" 11 | }, 12 | "autoload": { 13 | "psr-4": { 14 | "Controllers\\": "controllers/", 15 | "Jobs\\": "jobs/" 16 | }, 17 | "files": [ 18 | "lib/config.php", 19 | "lib/helpers.php", 20 | "lib/db_helpers.php" 21 | ] 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /composer.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_readme": [ 3 | "This file locks the dependencies of your project to a known state", 4 | "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", 5 | "This file is @generated automatically" 6 | ], 7 | "content-hash": "2f66bc5819a50268350d92a01decfb6c", 8 | "packages": [ 9 | { 10 | "name": "container-interop/container-interop", 11 | "version": "1.2.0", 12 | "source": { 13 | "type": "git", 14 | "url": "https://github.com/container-interop/container-interop.git", 15 | "reference": "79cbf1341c22ec75643d841642dd5d6acd83bdb8" 16 | }, 17 | "dist": { 18 | "type": "zip", 19 | "url": "https://api.github.com/repos/container-interop/container-interop/zipball/79cbf1341c22ec75643d841642dd5d6acd83bdb8", 20 | "reference": "79cbf1341c22ec75643d841642dd5d6acd83bdb8", 21 | "shasum": "" 22 | }, 23 | "require": { 24 | "psr/container": "^1.0" 25 | }, 26 | "type": "library", 27 | "autoload": { 28 | "psr-4": { 29 | "Interop\\Container\\": "src/Interop/Container/" 30 | } 31 | }, 32 | "notification-url": "https://packagist.org/downloads/", 33 | "license": [ 34 | "MIT" 35 | ], 36 | "description": "Promoting the interoperability of container objects (DIC, SL, etc.)", 37 | "homepage": "https://github.com/container-interop/container-interop", 38 | "time": "2017-02-14T19:40:03+00:00" 39 | }, 40 | { 41 | "name": "indieweb/link-rel-parser", 42 | "version": "0.1.3", 43 | "source": { 44 | "type": "git", 45 | "url": "https://github.com/indieweb/link-rel-parser-php.git", 46 | "reference": "295420e4f16d9a9d262a3c25a7a583794428f055" 47 | }, 48 | "dist": { 49 | "type": "zip", 50 | "url": "https://api.github.com/repos/indieweb/link-rel-parser-php/zipball/295420e4f16d9a9d262a3c25a7a583794428f055", 51 | "reference": "295420e4f16d9a9d262a3c25a7a583794428f055", 52 | "shasum": "" 53 | }, 54 | "require": { 55 | "php": ">=5.3.0" 56 | }, 57 | "type": "library", 58 | "autoload": { 59 | "files": [ 60 | "src/IndieWeb/link_rel_parser.php" 61 | ] 62 | }, 63 | "notification-url": "https://packagist.org/downloads/", 64 | "license": [ 65 | "Apache-2.0" 66 | ], 67 | "authors": [ 68 | { 69 | "name": "Aaron Parecki", 70 | "homepage": "http://aaronparecki.com" 71 | }, 72 | { 73 | "name": "Tantek Çelik", 74 | "homepage": "http://tantek.com" 75 | } 76 | ], 77 | "description": "Parse rel values from HTTP headers", 78 | "homepage": "https://github.com/indieweb/link-rel-parser-php", 79 | "keywords": [ 80 | "http", 81 | "indieweb", 82 | "microformats2" 83 | ], 84 | "time": "2017-01-11T17:14:49+00:00" 85 | }, 86 | { 87 | "name": "j4mie/idiorm", 88 | "version": "v1.5.3", 89 | "source": { 90 | "type": "git", 91 | "url": "https://github.com/j4mie/idiorm.git", 92 | "reference": "f2f170c44af4761fef8ef34d6dbc237cd95df799" 93 | }, 94 | "dist": { 95 | "type": "zip", 96 | "url": "https://api.github.com/repos/j4mie/idiorm/zipball/f2f170c44af4761fef8ef34d6dbc237cd95df799", 97 | "reference": "f2f170c44af4761fef8ef34d6dbc237cd95df799", 98 | "shasum": "" 99 | }, 100 | "require": { 101 | "php": ">=5.2.0" 102 | }, 103 | "require-dev": { 104 | "phpunit/phpunit": "^4.8" 105 | }, 106 | "type": "library", 107 | "autoload": { 108 | "classmap": [ 109 | "idiorm.php" 110 | ] 111 | }, 112 | "notification-url": "https://packagist.org/downloads/", 113 | "license": [ 114 | "BSD-2-Clause", 115 | "BSD-3-Clause", 116 | "BSD-4-Clause" 117 | ], 118 | "authors": [ 119 | { 120 | "name": "Simon Holywell", 121 | "email": "treffynnon@php.net", 122 | "homepage": "http://simonholywell.com", 123 | "role": "Maintainer" 124 | }, 125 | { 126 | "name": "Jamie Matthews", 127 | "email": "jamie.matthews@gmail.com", 128 | "homepage": "http://j4mie.org", 129 | "role": "Developer" 130 | }, 131 | { 132 | "name": "Durham Hale", 133 | "email": "me@durhamhale.com", 134 | "homepage": "http://durhamhale.com", 135 | "role": "Maintainer" 136 | } 137 | ], 138 | "description": "A lightweight nearly-zero-configuration object-relational mapper and fluent query builder for PHP5", 139 | "homepage": "http://j4mie.github.com/idiormandparis", 140 | "keywords": [ 141 | "idiorm", 142 | "orm", 143 | "query builder" 144 | ], 145 | "time": "2017-03-21T01:31:25+00:00" 146 | }, 147 | { 148 | "name": "league/container", 149 | "version": "2.4.1", 150 | "source": { 151 | "type": "git", 152 | "url": "https://github.com/thephpleague/container.git", 153 | "reference": "43f35abd03a12977a60ffd7095efd6a7808488c0" 154 | }, 155 | "dist": { 156 | "type": "zip", 157 | "url": "https://api.github.com/repos/thephpleague/container/zipball/43f35abd03a12977a60ffd7095efd6a7808488c0", 158 | "reference": "43f35abd03a12977a60ffd7095efd6a7808488c0", 159 | "shasum": "" 160 | }, 161 | "require": { 162 | "container-interop/container-interop": "^1.2", 163 | "php": "^5.4.0 || ^7.0" 164 | }, 165 | "provide": { 166 | "container-interop/container-interop-implementation": "^1.2", 167 | "psr/container-implementation": "^1.0" 168 | }, 169 | "replace": { 170 | "orno/di": "~2.0" 171 | }, 172 | "require-dev": { 173 | "phpunit/phpunit": "4.*" 174 | }, 175 | "type": "library", 176 | "extra": { 177 | "branch-alias": { 178 | "dev-2.x": "2.x-dev", 179 | "dev-1.x": "1.x-dev" 180 | } 181 | }, 182 | "autoload": { 183 | "psr-4": { 184 | "League\\Container\\": "src" 185 | } 186 | }, 187 | "notification-url": "https://packagist.org/downloads/", 188 | "license": [ 189 | "MIT" 190 | ], 191 | "authors": [ 192 | { 193 | "name": "Phil Bennett", 194 | "email": "philipobenito@gmail.com", 195 | "homepage": "http://www.philipobenito.com", 196 | "role": "Developer" 197 | } 198 | ], 199 | "description": "A fast and intuitive dependency injection container.", 200 | "homepage": "https://github.com/thephpleague/container", 201 | "keywords": [ 202 | "container", 203 | "dependency", 204 | "di", 205 | "injection", 206 | "league", 207 | "provider", 208 | "service" 209 | ], 210 | "time": "2017-05-10T09:20:27+00:00" 211 | }, 212 | { 213 | "name": "league/plates", 214 | "version": "3.3.0", 215 | "source": { 216 | "type": "git", 217 | "url": "https://github.com/thephpleague/plates.git", 218 | "reference": "b1684b6f127714497a0ef927ce42c0b44b45a8af" 219 | }, 220 | "dist": { 221 | "type": "zip", 222 | "url": "https://api.github.com/repos/thephpleague/plates/zipball/b1684b6f127714497a0ef927ce42c0b44b45a8af", 223 | "reference": "b1684b6f127714497a0ef927ce42c0b44b45a8af", 224 | "shasum": "" 225 | }, 226 | "require": { 227 | "php": "^5.3 | ^7.0" 228 | }, 229 | "require-dev": { 230 | "mikey179/vfsstream": "^1.4", 231 | "phpunit/phpunit": "~4.0", 232 | "squizlabs/php_codesniffer": "~1.5" 233 | }, 234 | "type": "library", 235 | "extra": { 236 | "branch-alias": { 237 | "dev-master": "3.0-dev" 238 | } 239 | }, 240 | "autoload": { 241 | "psr-4": { 242 | "League\\Plates\\": "src" 243 | } 244 | }, 245 | "notification-url": "https://packagist.org/downloads/", 246 | "license": [ 247 | "MIT" 248 | ], 249 | "authors": [ 250 | { 251 | "name": "Jonathan Reinink", 252 | "email": "jonathan@reinink.ca", 253 | "role": "Developer" 254 | } 255 | ], 256 | "description": "Plates, the native PHP template system that's fast, easy to use and easy to extend.", 257 | "homepage": "http://platesphp.com", 258 | "keywords": [ 259 | "league", 260 | "package", 261 | "templates", 262 | "templating", 263 | "views" 264 | ], 265 | "time": "2016-12-28T00:14:17+00:00" 266 | }, 267 | { 268 | "name": "league/route", 269 | "version": "3.0.4", 270 | "source": { 271 | "type": "git", 272 | "url": "https://github.com/thephpleague/route.git", 273 | "reference": "274e3938c06ec0f478798a92b0deece65db5b5e1" 274 | }, 275 | "dist": { 276 | "type": "zip", 277 | "url": "https://api.github.com/repos/thephpleague/route/zipball/274e3938c06ec0f478798a92b0deece65db5b5e1", 278 | "reference": "274e3938c06ec0f478798a92b0deece65db5b5e1", 279 | "shasum": "" 280 | }, 281 | "require": { 282 | "league/container": "^2.4", 283 | "nikic/fast-route": "^1.0|^0.8|^0.7", 284 | "php": ">=5.4.0", 285 | "psr/container": "^1.0", 286 | "psr/http-message": "^1.0" 287 | }, 288 | "replace": { 289 | "orno/http": "~1.0", 290 | "orno/route": "~1.0" 291 | }, 292 | "require-dev": { 293 | "phpunit/phpunit": "^4.8" 294 | }, 295 | "type": "library", 296 | "extra": { 297 | "branch-alias": { 298 | "dev-master": "3.x-dev", 299 | "dev-2.x": "2.x-dev", 300 | "dev-1.x": "1.x-dev" 301 | } 302 | }, 303 | "autoload": { 304 | "psr-4": { 305 | "League\\Route\\": "src" 306 | } 307 | }, 308 | "notification-url": "https://packagist.org/downloads/", 309 | "license": [ 310 | "MIT" 311 | ], 312 | "authors": [ 313 | { 314 | "name": "Phil Bennett", 315 | "email": "philipobenito@gmail.com", 316 | "homepage": "http://www.philipobenito.com", 317 | "role": "Developer" 318 | } 319 | ], 320 | "description": "A fast routing and dispatch package built on top of FastRoute.", 321 | "homepage": "https://github.com/thephpleague/route", 322 | "keywords": [ 323 | "dispatcher", 324 | "league", 325 | "psr-7", 326 | "psr7", 327 | "route" 328 | ], 329 | "time": "2017-03-22T09:09:41+00:00" 330 | }, 331 | { 332 | "name": "mf2/mf2", 333 | "version": "v0.3.2", 334 | "source": { 335 | "type": "git", 336 | "url": "https://github.com/indieweb/php-mf2.git", 337 | "reference": "dc0d90d4ee30864bcf37cd3a8fc8db94f9134cc4" 338 | }, 339 | "dist": { 340 | "type": "zip", 341 | "url": "https://api.github.com/repos/indieweb/php-mf2/zipball/dc0d90d4ee30864bcf37cd3a8fc8db94f9134cc4", 342 | "reference": "dc0d90d4ee30864bcf37cd3a8fc8db94f9134cc4", 343 | "shasum": "" 344 | }, 345 | "require": { 346 | "php": ">=5.4.0" 347 | }, 348 | "require-dev": { 349 | "mf2/tests": "@dev", 350 | "phpdocumentor/phpdocumentor": "v2.8.4", 351 | "phpunit/phpunit": "4.8.*" 352 | }, 353 | "suggest": { 354 | "barnabywalters/mf-cleaner": "To more easily handle the canonical data php-mf2 gives you" 355 | }, 356 | "bin": [ 357 | "bin/fetch-mf2", 358 | "bin/parse-mf2" 359 | ], 360 | "type": "library", 361 | "autoload": { 362 | "files": [ 363 | "Mf2/Parser.php" 364 | ] 365 | }, 366 | "notification-url": "https://packagist.org/downloads/", 367 | "license": [ 368 | "CC0" 369 | ], 370 | "authors": [ 371 | { 372 | "name": "Barnaby Walters", 373 | "homepage": "http://waterpigs.co.uk" 374 | } 375 | ], 376 | "description": "A pure, generic microformats2 parser — makes HTML as easy to consume as a JSON API", 377 | "keywords": [ 378 | "html", 379 | "microformats", 380 | "microformats 2", 381 | "parser", 382 | "semantic" 383 | ], 384 | "time": "2017-05-27T15:27:47+00:00" 385 | }, 386 | { 387 | "name": "nikic/fast-route", 388 | "version": "v1.2.0", 389 | "source": { 390 | "type": "git", 391 | "url": "https://github.com/nikic/FastRoute.git", 392 | "reference": "b5f95749071c82a8e0f58586987627054400cdf6" 393 | }, 394 | "dist": { 395 | "type": "zip", 396 | "url": "https://api.github.com/repos/nikic/FastRoute/zipball/b5f95749071c82a8e0f58586987627054400cdf6", 397 | "reference": "b5f95749071c82a8e0f58586987627054400cdf6", 398 | "shasum": "" 399 | }, 400 | "require": { 401 | "php": ">=5.4.0" 402 | }, 403 | "type": "library", 404 | "autoload": { 405 | "psr-4": { 406 | "FastRoute\\": "src/" 407 | }, 408 | "files": [ 409 | "src/functions.php" 410 | ] 411 | }, 412 | "notification-url": "https://packagist.org/downloads/", 413 | "license": [ 414 | "BSD-3-Clause" 415 | ], 416 | "authors": [ 417 | { 418 | "name": "Nikita Popov", 419 | "email": "nikic@php.net" 420 | } 421 | ], 422 | "description": "Fast request router for PHP", 423 | "keywords": [ 424 | "router", 425 | "routing" 426 | ], 427 | "time": "2017-01-19T11:35:12+00:00" 428 | }, 429 | { 430 | "name": "p3k/caterpillar", 431 | "version": "0.1.2", 432 | "source": { 433 | "type": "git", 434 | "url": "https://github.com/aaronpk/Caterpillar.git", 435 | "reference": "3090a9dcdab2da9a3a21deae2841c9a3f286097f" 436 | }, 437 | "dist": { 438 | "type": "zip", 439 | "url": "https://api.github.com/repos/aaronpk/Caterpillar/zipball/3090a9dcdab2da9a3a21deae2841c9a3f286097f", 440 | "reference": "3090a9dcdab2da9a3a21deae2841c9a3f286097f", 441 | "shasum": "" 442 | }, 443 | "require": { 444 | "pda/pheanstalk": "3.*", 445 | "php": ">5.4.0" 446 | }, 447 | "type": "library", 448 | "autoload": { 449 | "psr-0": { 450 | "Caterpillar": "src/" 451 | } 452 | }, 453 | "notification-url": "https://packagist.org/downloads/", 454 | "license": [ 455 | "Apache 2.0" 456 | ], 457 | "authors": [ 458 | { 459 | "name": "Aaron Parecki", 460 | "homepage": "http://aaronparecki.com/" 461 | } 462 | ], 463 | "description": "Caterpillar is a background queue manager", 464 | "time": "2017-03-19T19:58:18+00:00" 465 | }, 466 | { 467 | "name": "p3k/http", 468 | "version": "0.1.5", 469 | "source": { 470 | "type": "git", 471 | "url": "https://github.com/aaronpk/p3k-http.git", 472 | "reference": "3740fe135e6d58457d7528e7c05a67b68e020a79" 473 | }, 474 | "dist": { 475 | "type": "zip", 476 | "url": "https://api.github.com/repos/aaronpk/p3k-http/zipball/3740fe135e6d58457d7528e7c05a67b68e020a79", 477 | "reference": "3740fe135e6d58457d7528e7c05a67b68e020a79", 478 | "shasum": "" 479 | }, 480 | "require": { 481 | "indieweb/link-rel-parser": "0.1.*", 482 | "mf2/mf2": "0.3.*" 483 | }, 484 | "type": "library", 485 | "autoload": { 486 | "psr-4": { 487 | "p3k\\": "src/p3k" 488 | } 489 | }, 490 | "notification-url": "https://packagist.org/downloads/", 491 | "license": [ 492 | "MIT" 493 | ], 494 | "authors": [ 495 | { 496 | "name": "Aaron Parecki", 497 | "homepage": "https://aaronparecki.com" 498 | } 499 | ], 500 | "description": "A simple wrapper API around the PHP curl functions", 501 | "homepage": "https://github.com/aaronpk/p3k-http", 502 | "time": "2017-04-29T17:43:29+00:00" 503 | }, 504 | { 505 | "name": "p3k/utils", 506 | "version": "1.0.0", 507 | "source": { 508 | "type": "git", 509 | "url": "https://github.com/aaronpk/p3k-utils.git", 510 | "reference": "4f2ab159bae3d79a21fb02f30a03eed3d57aab96" 511 | }, 512 | "dist": { 513 | "type": "zip", 514 | "url": "https://api.github.com/repos/aaronpk/p3k-utils/zipball/4f2ab159bae3d79a21fb02f30a03eed3d57aab96", 515 | "reference": "4f2ab159bae3d79a21fb02f30a03eed3d57aab96", 516 | "shasum": "" 517 | }, 518 | "require": { 519 | "php": ">=5.5" 520 | }, 521 | "require-dev": { 522 | "phpunit/phpunit": "^4.8.13", 523 | "predis/predis": "1.1.*" 524 | }, 525 | "type": "library", 526 | "autoload": { 527 | "files": [ 528 | "src/global.php", 529 | "src/url.php", 530 | "src/utils.php", 531 | "src/date.php", 532 | "src/cache.php" 533 | ] 534 | }, 535 | "notification-url": "https://packagist.org/downloads/", 536 | "license": [ 537 | "MIT" 538 | ], 539 | "authors": [ 540 | { 541 | "name": "Aaron Parecki", 542 | "homepage": "https://aaronparecki.com" 543 | } 544 | ], 545 | "description": "Some helpful functions used by https://p3k.io projects", 546 | "homepage": "https://github.com/aaronpk/p3k-utils", 547 | "time": "2017-04-30T16:14:20+00:00" 548 | }, 549 | { 550 | "name": "pda/pheanstalk", 551 | "version": "v3.1.0", 552 | "source": { 553 | "type": "git", 554 | "url": "https://github.com/pda/pheanstalk.git", 555 | "reference": "430e77c551479aad0c6ada0450ee844cf656a18b" 556 | }, 557 | "dist": { 558 | "type": "zip", 559 | "url": "https://api.github.com/repos/pda/pheanstalk/zipball/430e77c551479aad0c6ada0450ee844cf656a18b", 560 | "reference": "430e77c551479aad0c6ada0450ee844cf656a18b", 561 | "shasum": "" 562 | }, 563 | "require": { 564 | "php": ">=5.3.0" 565 | }, 566 | "require-dev": { 567 | "phpunit/phpunit": "~4.0" 568 | }, 569 | "type": "library", 570 | "extra": { 571 | "branch-alias": { 572 | "dev-master": "3.0-dev" 573 | } 574 | }, 575 | "autoload": { 576 | "psr-4": { 577 | "Pheanstalk\\": "src/" 578 | } 579 | }, 580 | "notification-url": "https://packagist.org/downloads/", 581 | "license": [ 582 | "MIT" 583 | ], 584 | "authors": [ 585 | { 586 | "name": "Paul Annesley", 587 | "email": "paul@annesley.cc", 588 | "homepage": "http://paul.annesley.cc/", 589 | "role": "Developer" 590 | } 591 | ], 592 | "description": "PHP client for beanstalkd queue", 593 | "homepage": "https://github.com/pda/pheanstalk", 594 | "keywords": [ 595 | "beanstalkd" 596 | ], 597 | "time": "2015-08-07T21:42:41+00:00" 598 | }, 599 | { 600 | "name": "pimple/pimple", 601 | "version": "v3.2.2", 602 | "source": { 603 | "type": "git", 604 | "url": "https://github.com/silexphp/Pimple.git", 605 | "reference": "4d45fb62d96418396ec58ba76e6f065bca16e10a" 606 | }, 607 | "dist": { 608 | "type": "zip", 609 | "url": "https://api.github.com/repos/silexphp/Pimple/zipball/4d45fb62d96418396ec58ba76e6f065bca16e10a", 610 | "reference": "4d45fb62d96418396ec58ba76e6f065bca16e10a", 611 | "shasum": "" 612 | }, 613 | "require": { 614 | "php": ">=5.3.0", 615 | "psr/container": "^1.0" 616 | }, 617 | "require-dev": { 618 | "symfony/phpunit-bridge": "^3.2" 619 | }, 620 | "type": "library", 621 | "extra": { 622 | "branch-alias": { 623 | "dev-master": "3.2.x-dev" 624 | } 625 | }, 626 | "autoload": { 627 | "psr-0": { 628 | "Pimple": "src/" 629 | } 630 | }, 631 | "notification-url": "https://packagist.org/downloads/", 632 | "license": [ 633 | "MIT" 634 | ], 635 | "authors": [ 636 | { 637 | "name": "Fabien Potencier", 638 | "email": "fabien@symfony.com" 639 | } 640 | ], 641 | "description": "Pimple, a simple Dependency Injection Container", 642 | "homepage": "http://pimple.sensiolabs.org", 643 | "keywords": [ 644 | "container", 645 | "dependency injection" 646 | ], 647 | "time": "2017-07-23T07:32:15+00:00" 648 | }, 649 | { 650 | "name": "psr/container", 651 | "version": "1.0.0", 652 | "source": { 653 | "type": "git", 654 | "url": "https://github.com/php-fig/container.git", 655 | "reference": "b7ce3b176482dbbc1245ebf52b181af44c2cf55f" 656 | }, 657 | "dist": { 658 | "type": "zip", 659 | "url": "https://api.github.com/repos/php-fig/container/zipball/b7ce3b176482dbbc1245ebf52b181af44c2cf55f", 660 | "reference": "b7ce3b176482dbbc1245ebf52b181af44c2cf55f", 661 | "shasum": "" 662 | }, 663 | "require": { 664 | "php": ">=5.3.0" 665 | }, 666 | "type": "library", 667 | "extra": { 668 | "branch-alias": { 669 | "dev-master": "1.0.x-dev" 670 | } 671 | }, 672 | "autoload": { 673 | "psr-4": { 674 | "Psr\\Container\\": "src/" 675 | } 676 | }, 677 | "notification-url": "https://packagist.org/downloads/", 678 | "license": [ 679 | "MIT" 680 | ], 681 | "authors": [ 682 | { 683 | "name": "PHP-FIG", 684 | "homepage": "http://www.php-fig.org/" 685 | } 686 | ], 687 | "description": "Common Container Interface (PHP FIG PSR-11)", 688 | "homepage": "https://github.com/php-fig/container", 689 | "keywords": [ 690 | "PSR-11", 691 | "container", 692 | "container-interface", 693 | "container-interop", 694 | "psr" 695 | ], 696 | "time": "2017-02-14T16:28:37+00:00" 697 | }, 698 | { 699 | "name": "psr/http-message", 700 | "version": "1.0.1", 701 | "source": { 702 | "type": "git", 703 | "url": "https://github.com/php-fig/http-message.git", 704 | "reference": "f6561bf28d520154e4b0ec72be95418abe6d9363" 705 | }, 706 | "dist": { 707 | "type": "zip", 708 | "url": "https://api.github.com/repos/php-fig/http-message/zipball/f6561bf28d520154e4b0ec72be95418abe6d9363", 709 | "reference": "f6561bf28d520154e4b0ec72be95418abe6d9363", 710 | "shasum": "" 711 | }, 712 | "require": { 713 | "php": ">=5.3.0" 714 | }, 715 | "type": "library", 716 | "extra": { 717 | "branch-alias": { 718 | "dev-master": "1.0.x-dev" 719 | } 720 | }, 721 | "autoload": { 722 | "psr-4": { 723 | "Psr\\Http\\Message\\": "src/" 724 | } 725 | }, 726 | "notification-url": "https://packagist.org/downloads/", 727 | "license": [ 728 | "MIT" 729 | ], 730 | "authors": [ 731 | { 732 | "name": "PHP-FIG", 733 | "homepage": "http://www.php-fig.org/" 734 | } 735 | ], 736 | "description": "Common interface for HTTP messages", 737 | "homepage": "https://github.com/php-fig/http-message", 738 | "keywords": [ 739 | "http", 740 | "http-message", 741 | "psr", 742 | "psr-7", 743 | "request", 744 | "response" 745 | ], 746 | "time": "2016-08-06T14:39:51+00:00" 747 | }, 748 | { 749 | "name": "slim/slim", 750 | "version": "3.9.0", 751 | "source": { 752 | "type": "git", 753 | "url": "https://github.com/slimphp/Slim.git", 754 | "reference": "575a8b53a0a489447915029c69680156cd355304" 755 | }, 756 | "dist": { 757 | "type": "zip", 758 | "url": "https://api.github.com/repos/slimphp/Slim/zipball/575a8b53a0a489447915029c69680156cd355304", 759 | "reference": "575a8b53a0a489447915029c69680156cd355304", 760 | "shasum": "" 761 | }, 762 | "require": { 763 | "container-interop/container-interop": "^1.2", 764 | "nikic/fast-route": "^1.0", 765 | "php": ">=5.5.0", 766 | "pimple/pimple": "^3.0", 767 | "psr/container": "^1.0", 768 | "psr/http-message": "^1.0" 769 | }, 770 | "provide": { 771 | "psr/http-message-implementation": "1.0" 772 | }, 773 | "require-dev": { 774 | "phpunit/phpunit": "^4.0", 775 | "squizlabs/php_codesniffer": "^2.5" 776 | }, 777 | "type": "library", 778 | "autoload": { 779 | "psr-4": { 780 | "Slim\\": "Slim" 781 | } 782 | }, 783 | "notification-url": "https://packagist.org/downloads/", 784 | "license": [ 785 | "MIT" 786 | ], 787 | "authors": [ 788 | { 789 | "name": "Rob Allen", 790 | "email": "rob@akrabat.com", 791 | "homepage": "http://akrabat.com" 792 | }, 793 | { 794 | "name": "Josh Lockhart", 795 | "email": "hello@joshlockhart.com", 796 | "homepage": "https://joshlockhart.com" 797 | }, 798 | { 799 | "name": "Gabriel Manricks", 800 | "email": "gmanricks@me.com", 801 | "homepage": "http://gabrielmanricks.com" 802 | }, 803 | { 804 | "name": "Andrew Smith", 805 | "email": "a.smith@silentworks.co.uk", 806 | "homepage": "http://silentworks.co.uk" 807 | } 808 | ], 809 | "description": "Slim is a PHP micro framework that helps you quickly write simple yet powerful web applications and APIs", 810 | "homepage": "https://slimframework.com", 811 | "keywords": [ 812 | "api", 813 | "framework", 814 | "micro", 815 | "router" 816 | ], 817 | "time": "2017-11-04T08:46:46+00:00" 818 | }, 819 | { 820 | "name": "zendframework/zend-diactoros", 821 | "version": "1.6.1", 822 | "source": { 823 | "type": "git", 824 | "url": "https://github.com/zendframework/zend-diactoros.git", 825 | "reference": "c8664b92a6d5bc229e48b0923486c097e45a7877" 826 | }, 827 | "dist": { 828 | "type": "zip", 829 | "url": "https://api.github.com/repos/zendframework/zend-diactoros/zipball/c8664b92a6d5bc229e48b0923486c097e45a7877", 830 | "reference": "c8664b92a6d5bc229e48b0923486c097e45a7877", 831 | "shasum": "" 832 | }, 833 | "require": { 834 | "php": "^5.6 || ^7.0", 835 | "psr/http-message": "^1.0" 836 | }, 837 | "provide": { 838 | "psr/http-message-implementation": "1.0" 839 | }, 840 | "require-dev": { 841 | "ext-dom": "*", 842 | "ext-libxml": "*", 843 | "phpunit/phpunit": "^5.7.16 || ^6.0.8", 844 | "zendframework/zend-coding-standard": "~1.0" 845 | }, 846 | "type": "library", 847 | "extra": { 848 | "branch-alias": { 849 | "dev-master": "1.6-dev", 850 | "dev-develop": "1.7-dev" 851 | } 852 | }, 853 | "autoload": { 854 | "psr-4": { 855 | "Zend\\Diactoros\\": "src/" 856 | } 857 | }, 858 | "notification-url": "https://packagist.org/downloads/", 859 | "license": [ 860 | "BSD-2-Clause" 861 | ], 862 | "description": "PSR HTTP Message implementations", 863 | "homepage": "https://github.com/zendframework/zend-diactoros", 864 | "keywords": [ 865 | "http", 866 | "psr", 867 | "psr-7" 868 | ], 869 | "time": "2017-10-12T15:24:51+00:00" 870 | } 871 | ], 872 | "packages-dev": [], 873 | "aliases": [], 874 | "minimum-stability": "stable", 875 | "stability-flags": [], 876 | "prefer-stable": false, 877 | "prefer-lowest": false, 878 | "platform": [], 879 | "platform-dev": [] 880 | } 881 | -------------------------------------------------------------------------------- /controllers/API.php: -------------------------------------------------------------------------------- 1 | hasHeader('Authorization')) { 14 | return json_response($response, ['error'=>'forbidden'], 403); 15 | } 16 | 17 | if(!preg_match('/Bearer (.+)/', $request->getHeaderLine('Authorization'), $match)) { 18 | return json_response($response, ['error'=>'forbidden'], 403); 19 | } 20 | 21 | $token = $match[1]; 22 | 23 | $user = db\find('users', ['token'=>$token]); 24 | if(!$user) { 25 | return json_response($response, ['error'=>'unauthorized'], 401); 26 | } 27 | 28 | $body = $request->getParsedBody(); 29 | 30 | // Check for required parameters 31 | $params = ['hub_mode', 'hub_topic', 'hub_callback']; 32 | foreach($params as $p) { 33 | if(!isset($body[$p]) || trim($body[$p]) == '') { 34 | return json_response($response, ['error'=>'invalid '.$p], 400); 35 | } 36 | } 37 | 38 | switch($body['hub_mode']) { 39 | case 'subscribe': 40 | 41 | $feed = db\find_or_create('feeds', [ 42 | 'url'=>$body['hub_topic'] 43 | ], [ 44 | 'tier'=>30, 45 | 'domain'=>parse_url($body['hub_topic'], PHP_URL_HOST) 46 | ], true); 47 | $subscriber = db\find_or_create('subscribers', [ 48 | 'user_id' => $user->id, 49 | 'feed_id' => $feed->id, 50 | 'callback_url' => $body['hub_callback'] 51 | ], [], true); 52 | $response_data = ['result'=>'subscribed']; 53 | 54 | // Queue a poll of this feed now, and force delivery to this subscriber 55 | q()->queue('\\Jobs\\CheckFeed', 'poll', [$feed->id, $subscriber->id]); 56 | 57 | break; 58 | case 'unsubscribe': 59 | 60 | $feed = db\find('feeds', ['url'=>$body['hub_topic']]); 61 | $response_data = ['result'=>'subscription_not_found']; 62 | if($feed) { 63 | $subscriber = db\find('subscribers', [ 64 | 'user_id' => $user->id, 65 | 'feed_id' => $feed->id, 66 | 'callback_url' => $body['hub_callback'] 67 | ]); 68 | if($subscriber) { 69 | $subscriber->delete(); 70 | $response_data = ['result'=>'unsubscribed']; 71 | } 72 | } 73 | 74 | break; 75 | default: 76 | return json_response($response, ['error'=>'invalid mode'], 400); 77 | } 78 | 79 | return json_response($response, $response_data); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /controllers/Main.php: -------------------------------------------------------------------------------- 1 | getBody()->write(view('index')); 11 | return $response; 12 | } 13 | 14 | } 15 | 16 | -------------------------------------------------------------------------------- /controllers/Stats.php: -------------------------------------------------------------------------------- 1 | getQueryParams(); 12 | 13 | $tiers = ORM::for_table('feeds') 14 | ->raw_query('SELECT tier, COUNT(1) AS num 15 | FROM feeds 16 | GROUP BY tier 17 | ORDER BY tier') 18 | ->find_many(); 19 | 20 | if(isset($params['config'])) { 21 | $text = 'graph_title Watchtower Polling Tiers 22 | graph_info Number of feeds in each polling tier 23 | graph_vlabel Feeds 24 | graph_category watchtower 25 | graph_args --lower-limit 0 26 | graph_scale yes 27 | 28 | '; 29 | foreach($tiers as $tier) { 30 | $code = 'tier'.$tier->tier; 31 | $text .= $code.'.label '.self::tier_label($tier->tier).' 32 | '.$code.'.type GAUGE 33 | '.$code.'.min 0 34 | '; 35 | } 36 | } else { 37 | $text = ''; 38 | foreach($tiers as $tier) { 39 | $code = 'tier'.$tier->tier; 40 | $text .= $code.'.value '.$tier->num."\n"; 41 | } 42 | } 43 | return text_response($response, $text."\n"); 44 | } 45 | 46 | public function feeds(ServerRequestInterface $request, ResponseInterface $response) { 47 | $params = $request->getQueryParams(); 48 | 49 | if(isset($params['config'])) { 50 | $text = 'graph_title Watchtower Feeds 51 | graph_info Number of feeds and unique domains 52 | graph_vlabel Number 53 | graph_category watchtower 54 | graph_args --lower-limit 0 55 | graph_scale yes 56 | 57 | feeds.label Feeds 58 | feeds.type GAUGE 59 | feeds.min 0 60 | domains.label Domains 61 | domains.type GAUGE 62 | domains.min 0'; 63 | } else { 64 | $feeds = ORM::for_table('feeds')->raw_query('SELECT COUNT(1) AS num FROM feeds')->find_one()->num; 65 | $domains = ORM::for_table('feeds')->raw_query('SELECT COUNT(DISTINCT(domain)) AS num FROM feeds')->find_one()->num; 66 | $text = 'feeds.value '.$feeds.' 67 | domains.value '.$domains; 68 | } 69 | return text_response($response, $text."\n"); 70 | } 71 | 72 | public function polls(ServerRequestInterface $request, ResponseInterface $response) { 73 | $params = $request->getQueryParams(); 74 | 75 | if(isset($params['config'])) { 76 | $text = 'graph_title Watchtower Polls 77 | graph_info Feed polls per minute 78 | graph_vlabel Polls per Minute 79 | graph_category watchtower 80 | graph_args --lower-limit 0 81 | graph_scale yes 82 | graph_period minute 83 | 84 | polls.label Polls per Minute 85 | polls.type DERIVE 86 | polls.min 0'; 87 | } else { 88 | $polls = ORM::for_table('stats')->where('key', 'fetches')->find_one()->value; 89 | $text = 'polls.value '.$polls; 90 | } 91 | return text_response($response, $text."\n"); 92 | } 93 | 94 | private static function tier_label($minutes) { 95 | if($minutes >= 60) { 96 | return floor($minutes / 60).' Hour'.(floor($minutes / 60) == 1 ? '' : 's'); 97 | } else { 98 | return $minutes.' Minute'.($minutes == 1 ? '' : 's'); 99 | } 100 | } 101 | 102 | } 103 | 104 | -------------------------------------------------------------------------------- /data/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /jobs/CheckFeed.php: -------------------------------------------------------------------------------- 1 | pending = 0; 47 | $feed->save(); 48 | 49 | // Check that this feed wasn't already recently checked 50 | if($feed->last_checked_at && (time()-strtotime($feed->last_checked_at)) < 15) { 51 | echo "Feed $feed_id was checked within the last minute, skipping\n"; 52 | return; 53 | } 54 | 55 | echo "Checking feed $feed_id $feed->url '$feed->content_type'\n"; 56 | 57 | ORM::for_table('stats')->raw_execute('UPDATE stats SET `value` = `value` + 1 WHERE `key` = "fetches"'); 58 | 59 | // Download the contents of the feed 60 | self::$http = new \p3k\HTTP('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36 p3k-http/0.1.5 p3k-watchtower/0.1'); 61 | self::$http->set_timeout(30); 62 | $headers = []; 63 | if($feed->http_etag) { 64 | $headers['If-None-Match'] = $feed->http_etag; 65 | } 66 | $data = self::$http->get($feed->url, $headers); 67 | 68 | $last_checks_since_last_change = $feed->checks_since_last_change; 69 | 70 | $changed = false; 71 | 72 | if($data['body'] == '' || $data['error']) { 73 | echo "Error fetching $feed->url\n"; 74 | $feed->checks_since_last_change++; 75 | } else { 76 | 77 | $content_type = self::parseHttpHeader($data['headers'], 'Content-Type') ?: 'unknown'; 78 | 79 | $feed->http_last_modified = self::parseHttpHeader($data['headers'], 'Last-Modified') ?: ''; 80 | $feed->http_etag = self::parseHttpHeader($data['headers'], 'Etag') ?: ''; 81 | $feed->content_length = self::parseHttpHeader($data['headers'], 'Content-Length'); 82 | $feed->content_type = $content_type; 83 | 84 | if(isset($data['rels']['hub']) && isset($data['rels']['self'])) { 85 | $feed->websub_hub = $data['rels']['hub'][0]; 86 | $feed->websub_topic = $data['rels']['self'][0]; 87 | // TODO: Queue a job to subscribe to the feed 88 | } 89 | 90 | $content_hash = md5($data['body']); 91 | 92 | // Check if the new content is different from the old content 93 | $previous_content_file = 'data/'.$feed_id.'.txt'; 94 | if(!file_exists($previous_content_file)) 95 | $previous_content = ''; 96 | else 97 | $previous_content = file_get_contents($previous_content_file); 98 | 99 | if(stripos($content_type, 'html')) { 100 | $previous_content = self::strip_html($previous_content); 101 | $current_content = self::strip_html($data['body']); 102 | $changed = $previous_content != $current_content; 103 | } else { 104 | $changed = $content_hash != $feed->content_hash; 105 | } 106 | 107 | $feed->content_hash = $content_hash; 108 | 109 | // If the new content different enough, deliver to the subscribers 110 | if($changed) { 111 | // Store the new content hash 112 | $feed->checks_since_last_change = 0; 113 | $feed->updated_at = date('Y-m-d H:i:s'); 114 | $feed->save(); 115 | 116 | // Deliver the content to each subscriber 117 | $subscribers = ORM::for_table('subscribers')->where('feed_id', $feed->id)->find_many(); 118 | foreach($subscribers as $subscriber) { 119 | self::deliver_to_subscriber($data['body'], $content_type, $subscriber); 120 | } 121 | } else { 122 | echo "No change\n"; 123 | $feed->checks_since_last_change++; 124 | $feed->save(); 125 | 126 | // Even if there was no change, deliver to the new subscriber right away 127 | if($subscriber_id) { 128 | $subscriber = db\get_by_id('subscribers', $subscriber_id); 129 | self::deliver_to_subscriber($data['body'], $content_type, $subscriber); 130 | } 131 | } 132 | 133 | file_put_contents($previous_content_file, $data['body']); 134 | } 135 | 136 | // If a feed changed after only 1 check, bump up two tiers 137 | if($changed && $last_checks_since_last_change == 0 && $feed->checks_since_last_change == 0) { 138 | $feed->tier = self::previousTier($feed->tier) ?: $feed->tier; 139 | $feed->tier = self::previousTier($feed->tier) ?: $feed->tier; 140 | echo "Changed immediately, bumping up to to $feed->tier\n"; 141 | } 142 | // If N checks happened with no changes, drop down one tier 143 | $n = 10; 144 | if($changed == false && $feed->checks_since_last_change >= $n) { 145 | $feed->tier = self::nextTier($feed->tier) ?: $feed->tier; 146 | $feed->checks_since_last_change = 0; 147 | echo "No changes in $n intervals, dropping down to $feed->tier\n"; 148 | } 149 | 150 | $feed->last_checked_at = date('Y-m-d H:i:s'); 151 | 152 | // Schedule the next check of this feed 153 | $feed->next_check_at = date('Y-m-d H:i:s', time()+($feed->tier*60)); 154 | $feed->save(); 155 | 156 | } 157 | 158 | private static function strip_html($html) { 159 | return preg_replace('/\s+/',"\n",strtolower(strip_tags($html))); 160 | } 161 | 162 | private static function deliver_to_subscriber($body, $content_type, $subscriber) { 163 | if($subscriber && $subscriber->callback_url) { 164 | 165 | $last_delivered = strtotime($subscriber->last_notified_at); 166 | if(!$subscriber->last_notified_at || (time()-$last_delivered) > 30) { 167 | // TODO: Move this into a separate delivery job? 168 | echo "Delivering to $subscriber->callback_url\n"; 169 | $user = db\get_by_id('users', $subscriber->user_id); 170 | $response = self::$http->post($subscriber->callback_url, $body, [ 171 | 'Content-Type: ' . $content_type, 172 | 'Authorization: Bearer ' . $user->token 173 | ]); 174 | $subscriber->last_http_status = $response['code']; 175 | if(floor($response['code'] / 200) != 2) { 176 | $subscriber->error_count++; 177 | } 178 | $subscriber->last_notified_at = date('Y-m-d H:i:s'); 179 | $subscriber->save(); 180 | } else { 181 | echo "Already delivered to $subscriber->callback_url in the last 30 seconds\n"; 182 | } 183 | } 184 | } 185 | 186 | } 187 | -------------------------------------------------------------------------------- /lib/config.template.php: -------------------------------------------------------------------------------- 1 | updated_at = date('Y-m-d H:i:s'); 11 | } 12 | 13 | function find_or_create($table, $where, $defaults=[], $autosave=false) { 14 | $item = ORM::for_table($table); 15 | 16 | // Where is an associative array of key/val combos 17 | foreach($where as $c=>$v) { 18 | $item = $item->where($c, $v); 19 | } 20 | 21 | $item = $item->find_one(); 22 | 23 | if(!$item) { 24 | $item = ORM::for_table($table)->create(); 25 | $item->created_at = date('Y-m-d H:i:s'); 26 | foreach($defaults as $k=>$v) { 27 | $item->{$k} = $v; 28 | } 29 | foreach($where as $k=>$v) { 30 | $item->{$k} = $v; 31 | } 32 | if($autosave) 33 | $item->save(); 34 | } 35 | return $item; 36 | } 37 | 38 | function find($table, $where) { 39 | $item = ORM::for_table($table); 40 | 41 | // Where is an associative array of key/val combos 42 | foreach($where as $c=>$v) { 43 | $item = $item->where($c, $v); 44 | } 45 | 46 | return $item->find_one(); 47 | } 48 | 49 | function create($table, $data) { 50 | $item = ORM::for_table($table)->create(); 51 | foreach($data as $k=>$v) { 52 | $item->{$k} = $v; 53 | } 54 | $item->save(); 55 | return $item; 56 | } 57 | 58 | function feed_from_url($url) { 59 | return ORM::for_table('feeds')->where('url', $url)->find_one(); 60 | } 61 | 62 | function get_by_id($table, $id) { 63 | return ORM::for_table($table)->where('id', $id)->find_one(); 64 | } 65 | 66 | function get_by_col($table, $col, $val) { 67 | return ORM::for_table($table)->where($col, $val)->find_one(); 68 | } 69 | -------------------------------------------------------------------------------- /lib/helpers.php: -------------------------------------------------------------------------------- 1 | 'SET NAMES utf8mb4']); 7 | 8 | function q() { 9 | static $caterpillar; 10 | if(!isset($caterpillar)) { 11 | $logdir = __DIR__.'/../scripts/logs/'; 12 | $caterpillar = new Caterpillar('watchtower', Config::$beanstalkServer, Config::$beanstalkPort, $logdir); 13 | } 14 | return $caterpillar; 15 | } 16 | 17 | function view($template, $data=[]) { 18 | global $templates; 19 | return $templates->render($template, $data); 20 | } 21 | 22 | function json_response($response, $data, $code=200) { 23 | $response->getBody()->write(json_encode($data)); 24 | return $response->withHeader('Content-Type', 'application/json')->withStatus($code); 25 | } 26 | 27 | function text_response($response, $data, $code=200) { 28 | $response->getBody()->write($data); 29 | return $response->withHeader('Content-Type', 'text/plain')->withStatus($code); 30 | } 31 | -------------------------------------------------------------------------------- /public/index.php: -------------------------------------------------------------------------------- 1 | share('response', Zend\Diactoros\Response::class); 13 | $container->share('request', function () { 14 | return Zend\Diactoros\ServerRequestFactory::fromGlobals( 15 | $_SERVER, $_GET, $_POST, $_COOKIE, $_FILES 16 | ); 17 | }); 18 | $container->share('emitter', Zend\Diactoros\Response\SapiEmitter::class); 19 | 20 | $route = new League\Route\RouteCollection(); 21 | 22 | $route->map('GET', '/', 'Controllers\\Main::index'); 23 | $route->map('POST', '/', 'Controllers\\API::index'); 24 | 25 | $route->map('GET', '/stats/tiers', 'Controllers\\Stats::tiers'); 26 | $route->map('GET', '/stats/feeds', 'Controllers\\Stats::feeds'); 27 | $route->map('GET', '/stats/polls', 'Controllers\\Stats::polls'); 28 | 29 | $response = $route->dispatch($container->get('request'), $container->get('response')); 30 | $container->get('emitter')->emit($response); 31 | -------------------------------------------------------------------------------- /schema/0001.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE feeds 2 | ADD COLUMN http_last_modified VARCHAR(100) NOT NULL DEFAULT '' AFTER checks_since_last_change, 3 | ADD COLUMN http_etag VARCHAR(255) NOT NULL DEFAULT '' AFTER http_last_modified, 4 | ADD COLUMN content_length INT(11) DEFAULT NULL AFTER content_type; 5 | 6 | -------------------------------------------------------------------------------- /schema/0002.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE feeds 2 | MODIFY COLUMN `url` varchar(2048) DEFAULT NULL; 3 | 4 | ALTER TABLE subscribers 5 | MODIFY COLUMN `callback_url` varchar(2048) DEFAULT NULL; 6 | 7 | ALTER TABLE users 8 | MODIFY COLUMN `url` varchar(2048) DEFAULT NULL; 9 | -------------------------------------------------------------------------------- /schema/0003.sql: -------------------------------------------------------------------------------- 1 | UPDATE feeds SET tier = 2880 WHERE tier > 1440; 2 | -------------------------------------------------------------------------------- /schema/0004.sql: -------------------------------------------------------------------------------- 1 | UPDATE feeds SET tier = 1440 WHERE tier >= 1440; 2 | -------------------------------------------------------------------------------- /schema/0005.php: -------------------------------------------------------------------------------- 1 | find_many(); 6 | foreach($feeds as $feed) { 7 | $feed->domain = parse_url($feed->url, PHP_URL_HOST); 8 | $feed->save(); 9 | } 10 | -------------------------------------------------------------------------------- /schema/0005.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE feeds 2 | ADD COLUMN `domain` varchar(255) DEFAULT NULL AFTER `url`; 3 | 4 | ALTER TABLE `feeds` ADD INDEX `domain` (`domain`); 5 | ALTER TABLE `feeds` ADD INDEX `tier` (`tier`); 6 | -------------------------------------------------------------------------------- /schema/0006.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE `stats` ( 2 | `key` varchar(30) NOT NULL, 3 | `value` int(11) DEFAULT NULL, 4 | PRIMARY KEY (`key`) 5 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 6 | 7 | INSERT INTO `stats` (`key`, `value`) VALUES ("fetches", 0); 8 | -------------------------------------------------------------------------------- /schema/0007.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE feeds 2 | ADD COLUMN `websub_hub` varchar(512) DEFAULT NULL AFTER `tier`, 3 | ADD COLUMN `websub_topic` varchar(512) DEFAULT NULL AFTER `websub_hub`, 4 | ADD COLUMN `websub_expiration` datetime DEFAULT NULL AFTER `websub_topic`, 5 | ADD COLUMN `websub_active` tinyint(4) NOT NULL DEFAULT 0 AFTER `websub_expiration`, 6 | ADD COLUMN `websub_last_ping_at` datetime DEFAULT NULL AFTER `websub_active`, 7 | ADD COLUMN `websub_subscribed_at` datetime DEFAULT NULL AFTER `websub_last_ping_at` 8 | ; 9 | -------------------------------------------------------------------------------- /schema/0008.sql: -------------------------------------------------------------------------------- 1 | UPDATE feeds SET tier = 360 WHERE tier > 360; 2 | -------------------------------------------------------------------------------- /schema/0009.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE feeds 2 | ADD COLUMN `pending` tinyint(4) NOT NULL DEFAULT '0' AFTER `next_check_at`; 3 | -------------------------------------------------------------------------------- /schema/schema.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE `users` ( 2 | `id` int(11) unsigned NOT NULL AUTO_INCREMENT, 3 | `url` varchar(2048) DEFAULT NULL, 4 | `token` varchar(255) DEFAULT NULL, 5 | `created_at` datetime DEFAULT NULL, 6 | PRIMARY KEY (`id`) 7 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 8 | 9 | CREATE TABLE `feeds` ( 10 | `id` bigint(11) unsigned NOT NULL AUTO_INCREMENT, 11 | `url` varchar(2048) DEFAULT NULL, 12 | `domain` varchar(255) DEFAULT NULL, 13 | `content_hash` varchar(255) DEFAULT NULL, 14 | `content_type` varchar(255) DEFAULT NULL, 15 | `content_length` int(11) DEFAULT NULL, 16 | `tier` int(11) DEFAULT NULL, 17 | `websub_hub` varchar(512) DEFAULT NULL, 18 | `websub_topic` varchar(512) DEFAULT NULL, 19 | `websub_expiration` datetime DEFAULT NULL, 20 | `websub_active` tinyint(4) NOT NULL DEFAULT '0', 21 | `websub_last_ping_at` datetime DEFAULT NULL, 22 | `websub_subscribed_at` datetime DEFAULT NULL, 23 | `checks_since_last_change` int(11) NOT NULL DEFAULT '0', 24 | `http_last_modified` varchar(100) NOT NULL DEFAULT '', 25 | `http_etag` varchar(255) NOT NULL DEFAULT '', 26 | `last_checked_at` datetime DEFAULT NULL, 27 | `next_check_at` datetime DEFAULT NULL, 28 | `pending` tinyint(4) NOT NULL DEFAULT '0', 29 | `created_at` datetime DEFAULT NULL, 30 | `updated_at` datetime DEFAULT NULL, 31 | PRIMARY KEY (`id`), 32 | KEY `domain` (`domain`), 33 | KEY `tier` (`tier`) 34 | ) ENGINE=InnoDB AUTO_INCREMENT=13 DEFAULT CHARSET=utf8mb4; 35 | 36 | CREATE TABLE `subscribers` ( 37 | `id` bigint(11) unsigned NOT NULL AUTO_INCREMENT, 38 | `user_id` bigint(20) DEFAULT NULL, 39 | `feed_id` bigint(20) DEFAULT NULL, 40 | `created_at` datetime DEFAULT NULL, 41 | `callback_url` varchar(2048) DEFAULT NULL, 42 | `last_http_status` int(11) DEFAULT NULL, 43 | `error_count` int(11) NOT NULL DEFAULT '0', 44 | `last_notified_at` datetime DEFAULT NULL, 45 | PRIMARY KEY (`id`) 46 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 47 | 48 | CREATE TABLE `stats` ( 49 | `key` varchar(30) NOT NULL, 50 | `value` int(11) DEFAULT NULL, 51 | PRIMARY KEY (`key`) 52 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; 53 | 54 | INSERT INTO `stats` (`key`, `value`) VALUES ("fetches", 0); 55 | -------------------------------------------------------------------------------- /scripts/cleanup.php: -------------------------------------------------------------------------------- 1 | select_many_expr('feeds.*') 12 | ->join('subscribers', ['feeds.id','=','subscribers.feed_id']) 13 | ->where_lt('next_check_at', date('Y-m-d H:i:s', time()-86400)) 14 | ->where('pending', 1) 15 | ->find_many(); 16 | foreach($feeds as $feed) { 17 | echo "Resetting $feed->id $feed->url\n"; 18 | $feed->pending = 0; 19 | $feed->save(); 20 | } 21 | -------------------------------------------------------------------------------- /scripts/cron.php: -------------------------------------------------------------------------------- 1 | select_many_expr('feeds.*') 8 | ->join('subscribers', ['feeds.id','=','subscribers.feed_id']) 9 | ->where_lt('next_check_at', date('Y-m-d H:i:s')) 10 | ->where('pending', 0) // don't double queue poll tasks if one is already pending 11 | ->find_many(); 12 | foreach($feeds as $feed) { 13 | echo "Queuing $feed->id $feed->url\n"; 14 | $feed->pending = 1; 15 | $feed->save(); 16 | q()->queue('Jobs\\CheckFeed', 'poll', $feed->id); 17 | } 18 | -------------------------------------------------------------------------------- /scripts/debug.php: -------------------------------------------------------------------------------- 1 | run_foreground(); 6 | -------------------------------------------------------------------------------- /scripts/logs/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | -------------------------------------------------------------------------------- /scripts/stats.php: -------------------------------------------------------------------------------- 1 | print_stats(); 6 | 7 | -------------------------------------------------------------------------------- /scripts/watchtower.php: -------------------------------------------------------------------------------- 1 | run_workers(array_key_exists(1, $argv) ? $argv[1] : 4); 6 | -------------------------------------------------------------------------------- /views/index.php: -------------------------------------------------------------------------------- 1 |

Watchtower

2 | --------------------------------------------------------------------------------