├── .gitignore ├── lib └── db.js ├── package-lock.json └── scraper.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .vscode 3 | scraper.code-workspace 4 | .env 5 | partners.json -------------------------------------------------------------------------------- /lib/db.js: -------------------------------------------------------------------------------- 1 | const firebase = require('firebase/app'); 2 | require('firebase/firestore'); 3 | 4 | module.exports = function loadFirebase() { 5 | try { 6 | // Initialize Firebase 7 | const config = { 8 | apiKey: process.env.FIREBASE_API_KEY, 9 | authDomain: process.env.FIREBASE_AUTH_DOMAIN, 10 | databaseURL: process.env.FIREBASE_DATA_URL, 11 | projectId: process.env.FIREBASE_PROJECT_ID, 12 | storageBucket: process.env.FIREBASE_STORAGE_BUCKET, 13 | messagingSenderId: process.env.FIREBASE_MESSAGING_SENDER_ID 14 | } 15 | firebase.initializeApp(config) 16 | firebase.firestore().settings({timestampsInSnapshots: true}) 17 | } catch (error) { 18 | if (!/already exists/.test(error.message)) { 19 | console.log(`Firebase didn't initialize correctly: ${error.message}`) 20 | } 21 | } 22 | return firebase 23 | } -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "requires": true, 3 | "lockfileVersion": 1, 4 | "dependencies": { 5 | "@firebase/app": { 6 | "version": "0.3.4", 7 | "resolved": "https://registry.npmjs.org/@firebase/app/-/app-0.3.4.tgz", 8 | "integrity": "sha512-Q6sNpWZ3x+FeuBkLCCRrsOraGJOKVLUCc9Amj8zu2vAC1v2uWifRR6kZ60TrpaIxtY4N6pcPTaG0YIUT5lgeSA==", 9 | "requires": { 10 | "@firebase/app-types": "0.3.2", 11 | "@firebase/util": "0.2.2", 12 | "dom-storage": "2.1.0", 13 | "tslib": "1.9.0", 14 | "xmlhttprequest": "1.8.0" 15 | } 16 | }, 17 | "@firebase/app-types": { 18 | "version": "0.3.2", 19 | "resolved": "https://registry.npmjs.org/@firebase/app-types/-/app-types-0.3.2.tgz", 20 | "integrity": "sha512-ZD8lTgW07NGgo75bTyBJA8Lt9+NweNzot7lrsBtIvfciwUzaFJLsv2EShqjBeuhF7RpG6YFucJ6m67w5buCtzw==" 21 | }, 22 | "@firebase/auth": { 23 | "version": "0.7.8", 24 | "resolved": "https://registry.npmjs.org/@firebase/auth/-/auth-0.7.8.tgz", 25 | "integrity": "sha512-49WyZekDuqoekW5Yl9JG+CNJb65UcWKgeI2fA9hDUdxQ2d0xFHcEVJGkEiHrgkv2a4/v44UP9xggQrvmbUg23w==", 26 | "requires": { 27 | "@firebase/auth-types": "0.3.4" 28 | } 29 | }, 30 | "@firebase/auth-types": { 31 | "version": "0.3.4", 32 | "resolved": "https://registry.npmjs.org/@firebase/auth-types/-/auth-types-0.3.4.tgz", 33 | "integrity": "sha512-0r3gSQk9jw5orFHCTUIgao0zan6dHt2J0BO3t/uEzbod+uwqvUn/gh+yg+kK6HX92Fg8E7y030KX4Bw/aXt0Ew==" 34 | }, 35 | "@firebase/database": { 36 | "version": "0.3.6", 37 | "resolved": "https://registry.npmjs.org/@firebase/database/-/database-0.3.6.tgz", 38 | "integrity": "sha512-r02JOqTLcd2/qn7QkkJvIAxMiMxmeyd5B76kl9hHAs+3cil5mUzHnI3svtb4h0VIJYDHFKJMlVl/bE3GfcTR3A==", 39 | "requires": { 40 | "@firebase/database-types": "0.3.2", 41 | "@firebase/logger": "0.1.1", 42 | "@firebase/util": "0.2.2", 43 | "faye-websocket": "0.11.1", 44 | "tslib": "1.9.0" 45 | } 46 | }, 47 | "@firebase/database-types": { 48 | "version": "0.3.2", 49 | "resolved": "https://registry.npmjs.org/@firebase/database-types/-/database-types-0.3.2.tgz", 50 | "integrity": "sha512-9ZYdvYQ6r3aaHJarhUM5Hf6lQWu3ZJme+RR0o8qfBb9L04TL3uNjt+AJFku1ysVPntTn+9GqJjiIB2/OC3JtwA==" 51 | }, 52 | "@firebase/firestore": { 53 | "version": "0.8.6", 54 | "resolved": "https://registry.npmjs.org/@firebase/firestore/-/firestore-0.8.6.tgz", 55 | "integrity": "sha512-/mEnGgGJxXJfAW6hvMOiPGv3cPS5z0azCMzcKlAQ2/lhXyQotubqWDnJmNcdyi1LGZ3QGBesB3K8U4x6tZXTIQ==", 56 | "requires": { 57 | "@firebase/firestore-types": "0.7.0", 58 | "@firebase/logger": "0.1.1", 59 | "@firebase/webchannel-wrapper": "0.2.11", 60 | "grpc": "1.13.1", 61 | "tslib": "1.9.0" 62 | } 63 | }, 64 | "@firebase/firestore-types": { 65 | "version": "0.7.0", 66 | "resolved": "https://registry.npmjs.org/@firebase/firestore-types/-/firestore-types-0.7.0.tgz", 67 | "integrity": "sha512-jyKRcKnSh3CSEPL4xGOZNoOXEiv7YmFK/JEcdd/4cAH17/Xo+Pk67gk1E648LRKh6QPghgNvzNTY5R10mKbQNw==" 68 | }, 69 | "@firebase/functions": { 70 | "version": "0.3.1", 71 | "resolved": "https://registry.npmjs.org/@firebase/functions/-/functions-0.3.1.tgz", 72 | "integrity": "sha512-6A38x7zPU6zCTyOgGx2v+z4ugcsWnciL6YkcZXLNlCLveUmFdL0DmaW5MEBSpSOOe8kOJMl7f3aaD6lWUHNOBg==", 73 | "requires": { 74 | "@firebase/functions-types": "0.2.0", 75 | "@firebase/messaging-types": "0.2.3", 76 | "isomorphic-fetch": "2.2.1", 77 | "tslib": "1.9.0" 78 | } 79 | }, 80 | "@firebase/functions-types": { 81 | "version": "0.2.0", 82 | "resolved": "https://registry.npmjs.org/@firebase/functions-types/-/functions-types-0.2.0.tgz", 83 | "integrity": "sha512-q1FB3YKEAnWd+FpIL5Xn0B1BXO2IowrAdrSViXkFxNZVpp9iCzQ8Ytcbr3V1xUr3dnmoW/V7zkZJZGuwBgiVhw==" 84 | }, 85 | "@firebase/logger": { 86 | "version": "0.1.1", 87 | "resolved": "https://registry.npmjs.org/@firebase/logger/-/logger-0.1.1.tgz", 88 | "integrity": "sha512-5jn3HHbEfdOwychyIEIkP1cik+MW/vvoOavTOzwDkH+fv6Bx+HBUOzh09M7sCYzXFtKzjbUax9+g39mJNBLklQ==" 89 | }, 90 | "@firebase/messaging": { 91 | "version": "0.3.6", 92 | "resolved": "https://registry.npmjs.org/@firebase/messaging/-/messaging-0.3.6.tgz", 93 | "integrity": "sha512-Sz/fWOXMa3HxDZxE64Fm335kwP9um1rmun5PIka7od7I4hZ8US+SjYVyUe6jWTh1V/YjcqDi6Xkhoj2nF8yu9g==", 94 | "requires": { 95 | "@firebase/messaging-types": "0.2.3", 96 | "@firebase/util": "0.2.2", 97 | "tslib": "1.9.0" 98 | } 99 | }, 100 | "@firebase/messaging-types": { 101 | "version": "0.2.3", 102 | "resolved": "https://registry.npmjs.org/@firebase/messaging-types/-/messaging-types-0.2.3.tgz", 103 | "integrity": "sha512-avwCgZzcx2uxIW/wT3p3G/EyHftIrvMyiTS7AA7dxDlzfx+8dpAeTsb1+jsHJT4F6foSh5HG17Nw8sDzYuxH1Q==" 104 | }, 105 | "@firebase/polyfill": { 106 | "version": "0.3.3", 107 | "resolved": "https://registry.npmjs.org/@firebase/polyfill/-/polyfill-0.3.3.tgz", 108 | "integrity": "sha512-xs8IZf1WEbufYXyfV8YjmiFZOaujRRq0T03NteihYfuGVTTym7z5SmvLvEHLEUjf2fgeobPEzZ2JgrCQHS+QHw==", 109 | "requires": { 110 | "core-js": "2.5.5", 111 | "promise-polyfill": "7.1.2", 112 | "whatwg-fetch": "2.0.4" 113 | }, 114 | "dependencies": { 115 | "whatwg-fetch": { 116 | "version": "2.0.4", 117 | "resolved": "http://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-2.0.4.tgz", 118 | "integrity": "sha512-dcQ1GWpOD/eEQ97k66aiEVpNnapVj90/+R+SXTPYGHpYBBypfKJEQjLrvMZ7YXbKm21gXd4NcuxUTjiv1YtLng==" 119 | } 120 | } 121 | }, 122 | "@firebase/storage": { 123 | "version": "0.2.3", 124 | "resolved": "https://registry.npmjs.org/@firebase/storage/-/storage-0.2.3.tgz", 125 | "integrity": "sha512-2sq5jckWszW53gfQMkPNc7EumJ92oErRhzGJANbVzBumwR8qwKZU8/I+/uV9SPK1tVmSUc3S21jdoW5oOJVEuA==", 126 | "requires": { 127 | "@firebase/storage-types": "0.2.3", 128 | "tslib": "1.9.0" 129 | } 130 | }, 131 | "@firebase/storage-types": { 132 | "version": "0.2.3", 133 | "resolved": "https://registry.npmjs.org/@firebase/storage-types/-/storage-types-0.2.3.tgz", 134 | "integrity": "sha512-RaZeam2LgsB7xwAtOQr4G0Geoyf7D5TnLF3a12By6Rh0Z9PqBSlWn0SVYGW3SkmxIdqvWZMZvCyamUlqQvQzWw==" 135 | }, 136 | "@firebase/util": { 137 | "version": "0.2.2", 138 | "resolved": "https://registry.npmjs.org/@firebase/util/-/util-0.2.2.tgz", 139 | "integrity": "sha512-vfRjmCWuxtJx3txHocaNlDwCDwwv6KLL5YtlSNi73wBdvF3UfnpLGrth7G3X6gn5rDhOKamRg2+9L8cfsjSS1A==", 140 | "requires": { 141 | "tslib": "1.9.0" 142 | } 143 | }, 144 | "@firebase/webchannel-wrapper": { 145 | "version": "0.2.11", 146 | "resolved": "https://registry.npmjs.org/@firebase/webchannel-wrapper/-/webchannel-wrapper-0.2.11.tgz", 147 | "integrity": "sha512-WyMXDxk/WZ+f2lOCeEvDWUce2f5Kk2sNfvArK8f+PlUnzFdy/MBzLXrmbMgyZXP7GP4ooUxYV8Sdmoh1hGk1Uw==" 148 | }, 149 | "agent-base": { 150 | "version": "4.2.1", 151 | "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz", 152 | "integrity": "sha512-JVwXMr9nHYTUXsBFKUqhJwvlcYU/blreOEUkhNR2eXZIvwd+c+o5V4MgDPKWnMS/56awN3TRzIP+KoPn+roQtg==", 153 | "requires": { 154 | "es6-promisify": "^5.0.0" 155 | } 156 | }, 157 | "ansi-regex": { 158 | "version": "2.1.1", 159 | "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", 160 | "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" 161 | }, 162 | "ascli": { 163 | "version": "1.0.1", 164 | "resolved": "https://registry.npmjs.org/ascli/-/ascli-1.0.1.tgz", 165 | "integrity": "sha1-vPpZdKYvGOgcq660lzKrSoj5Brw=", 166 | "requires": { 167 | "colour": "~0.7.1", 168 | "optjs": "~3.2.2" 169 | } 170 | }, 171 | "async-limiter": { 172 | "version": "1.0.0", 173 | "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", 174 | "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" 175 | }, 176 | "balanced-match": { 177 | "version": "1.0.0", 178 | "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", 179 | "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" 180 | }, 181 | "brace-expansion": { 182 | "version": "1.1.11", 183 | "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", 184 | "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", 185 | "requires": { 186 | "balanced-match": "^1.0.0", 187 | "concat-map": "0.0.1" 188 | } 189 | }, 190 | "buffer-from": { 191 | "version": "1.1.1", 192 | "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", 193 | "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" 194 | }, 195 | "bytebuffer": { 196 | "version": "5.0.1", 197 | "resolved": "https://registry.npmjs.org/bytebuffer/-/bytebuffer-5.0.1.tgz", 198 | "integrity": "sha1-WC7qSxqHO20CCkjVjfhfC7ps/d0=", 199 | "requires": { 200 | "long": "~3" 201 | } 202 | }, 203 | "camelcase": { 204 | "version": "2.1.1", 205 | "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz", 206 | "integrity": "sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8=" 207 | }, 208 | "cliui": { 209 | "version": "3.2.0", 210 | "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz", 211 | "integrity": "sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0=", 212 | "requires": { 213 | "string-width": "^1.0.1", 214 | "strip-ansi": "^3.0.1", 215 | "wrap-ansi": "^2.0.0" 216 | } 217 | }, 218 | "code-point-at": { 219 | "version": "1.1.0", 220 | "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", 221 | "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" 222 | }, 223 | "colour": { 224 | "version": "0.7.1", 225 | "resolved": "https://registry.npmjs.org/colour/-/colour-0.7.1.tgz", 226 | "integrity": "sha1-nLFpkX7F0SwHNtPoaFdG3xyt93g=" 227 | }, 228 | "concat-map": { 229 | "version": "0.0.1", 230 | "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", 231 | "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" 232 | }, 233 | "concat-stream": { 234 | "version": "1.6.2", 235 | "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", 236 | "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", 237 | "requires": { 238 | "buffer-from": "^1.0.0", 239 | "inherits": "^2.0.3", 240 | "readable-stream": "^2.2.2", 241 | "typedarray": "^0.0.6" 242 | } 243 | }, 244 | "core-js": { 245 | "version": "2.5.5", 246 | "resolved": "http://registry.npmjs.org/core-js/-/core-js-2.5.5.tgz", 247 | "integrity": "sha1-sU3ek2xkDAV5prUMq8wTLdYSfjs=" 248 | }, 249 | "core-util-is": { 250 | "version": "1.0.2", 251 | "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", 252 | "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" 253 | }, 254 | "debug": { 255 | "version": "3.2.5", 256 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.5.tgz", 257 | "integrity": "sha512-D61LaDQPQkxJ5AUM2mbSJRbPkNs/TmdmOeLAi1hgDkpDfIfetSrjmWhccwtuResSwMbACjx/xXQofvM9CE/aeg==", 258 | "requires": { 259 | "ms": "^2.1.1" 260 | } 261 | }, 262 | "decamelize": { 263 | "version": "1.2.0", 264 | "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", 265 | "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" 266 | }, 267 | "dom-storage": { 268 | "version": "2.1.0", 269 | "resolved": "https://registry.npmjs.org/dom-storage/-/dom-storage-2.1.0.tgz", 270 | "integrity": "sha512-g6RpyWXzl0RR6OTElHKBl7nwnK87GUyZMYC7JWsB/IA73vpqK2K6LT39x4VepLxlSsWBFrPVLnsSR5Jyty0+2Q==" 271 | }, 272 | "encoding": { 273 | "version": "0.1.12", 274 | "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz", 275 | "integrity": "sha1-U4tm8+5izRq1HsMjgp0flIDHS+s=", 276 | "requires": { 277 | "iconv-lite": "~0.4.13" 278 | } 279 | }, 280 | "es6-promise": { 281 | "version": "4.2.5", 282 | "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.5.tgz", 283 | "integrity": "sha512-n6wvpdE43VFtJq+lUDYDBFUwV8TZbuGXLV4D6wKafg13ldznKsyEvatubnmUe31zcvelSzOHF+XbaT+Bl9ObDg==" 284 | }, 285 | "es6-promisify": { 286 | "version": "5.0.0", 287 | "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", 288 | "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", 289 | "requires": { 290 | "es6-promise": "^4.0.3" 291 | } 292 | }, 293 | "extract-zip": { 294 | "version": "1.6.7", 295 | "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz", 296 | "integrity": "sha1-qEC0uK9kAyZMjbV/Txp0Mz74H+k=", 297 | "requires": { 298 | "concat-stream": "1.6.2", 299 | "debug": "2.6.9", 300 | "mkdirp": "0.5.1", 301 | "yauzl": "2.4.1" 302 | }, 303 | "dependencies": { 304 | "debug": { 305 | "version": "2.6.9", 306 | "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", 307 | "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", 308 | "requires": { 309 | "ms": "2.0.0" 310 | } 311 | }, 312 | "ms": { 313 | "version": "2.0.0", 314 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", 315 | "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" 316 | } 317 | } 318 | }, 319 | "faye-websocket": { 320 | "version": "0.11.1", 321 | "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.1.tgz", 322 | "integrity": "sha1-8O/hjE9W5PQK/H4Gxxn9XuYYjzg=", 323 | "requires": { 324 | "websocket-driver": ">=0.5.1" 325 | } 326 | }, 327 | "fd-slicer": { 328 | "version": "1.0.1", 329 | "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz", 330 | "integrity": "sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU=", 331 | "requires": { 332 | "pend": "~1.2.0" 333 | } 334 | }, 335 | "firebase": { 336 | "version": "5.5.6", 337 | "resolved": "https://registry.npmjs.org/firebase/-/firebase-5.5.6.tgz", 338 | "integrity": "sha512-fHqI7iOqUSyyU90xGeHejm/niVyA5uNs7C52IBezYRlDjXimz/FgaGoEgXrdhPOJdDnmzMgvtms+Wf5oWfh9OQ==", 339 | "requires": { 340 | "@firebase/app": "0.3.4", 341 | "@firebase/auth": "0.7.8", 342 | "@firebase/database": "0.3.6", 343 | "@firebase/firestore": "0.8.6", 344 | "@firebase/functions": "0.3.1", 345 | "@firebase/messaging": "0.3.6", 346 | "@firebase/polyfill": "0.3.3", 347 | "@firebase/storage": "0.2.3" 348 | } 349 | }, 350 | "fs.realpath": { 351 | "version": "1.0.0", 352 | "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", 353 | "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" 354 | }, 355 | "glob": { 356 | "version": "7.1.3", 357 | "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", 358 | "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", 359 | "requires": { 360 | "fs.realpath": "^1.0.0", 361 | "inflight": "^1.0.4", 362 | "inherits": "2", 363 | "minimatch": "^3.0.4", 364 | "once": "^1.3.0", 365 | "path-is-absolute": "^1.0.0" 366 | } 367 | }, 368 | "grpc": { 369 | "version": "1.13.1", 370 | "resolved": "https://registry.npmjs.org/grpc/-/grpc-1.13.1.tgz", 371 | "integrity": "sha512-yl0xChnlUISTefOPU2NQ1cYPh5m/DTatEUV6jdRyQPE9NCrtPq7Gn6J2alMTglN7ufYbJapOd00dvhGurHH6HQ==", 372 | "requires": { 373 | "lodash": "^4.17.5", 374 | "nan": "^2.0.0", 375 | "node-pre-gyp": "^0.10.0", 376 | "protobufjs": "^5.0.3" 377 | }, 378 | "dependencies": { 379 | "abbrev": { 380 | "version": "1.1.1", 381 | "bundled": true 382 | }, 383 | "ansi-regex": { 384 | "version": "2.1.1", 385 | "bundled": true 386 | }, 387 | "aproba": { 388 | "version": "1.2.0", 389 | "bundled": true 390 | }, 391 | "are-we-there-yet": { 392 | "version": "1.1.5", 393 | "bundled": true, 394 | "requires": { 395 | "delegates": "^1.0.0", 396 | "readable-stream": "^2.0.6" 397 | } 398 | }, 399 | "balanced-match": { 400 | "version": "1.0.0", 401 | "bundled": true 402 | }, 403 | "brace-expansion": { 404 | "version": "1.1.11", 405 | "bundled": true, 406 | "requires": { 407 | "balanced-match": "^1.0.0", 408 | "concat-map": "0.0.1" 409 | } 410 | }, 411 | "chownr": { 412 | "version": "1.0.1", 413 | "bundled": true 414 | }, 415 | "code-point-at": { 416 | "version": "1.1.0", 417 | "bundled": true 418 | }, 419 | "concat-map": { 420 | "version": "0.0.1", 421 | "bundled": true 422 | }, 423 | "console-control-strings": { 424 | "version": "1.1.0", 425 | "bundled": true 426 | }, 427 | "core-util-is": { 428 | "version": "1.0.2", 429 | "bundled": true 430 | }, 431 | "debug": { 432 | "version": "2.6.9", 433 | "bundled": true, 434 | "requires": { 435 | "ms": "2.0.0" 436 | } 437 | }, 438 | "deep-extend": { 439 | "version": "0.6.0", 440 | "bundled": true 441 | }, 442 | "delegates": { 443 | "version": "1.0.0", 444 | "bundled": true 445 | }, 446 | "detect-libc": { 447 | "version": "1.0.3", 448 | "bundled": true 449 | }, 450 | "fs-minipass": { 451 | "version": "1.2.5", 452 | "bundled": true, 453 | "requires": { 454 | "minipass": "^2.2.1" 455 | } 456 | }, 457 | "fs.realpath": { 458 | "version": "1.0.0", 459 | "bundled": true 460 | }, 461 | "gauge": { 462 | "version": "2.7.4", 463 | "bundled": true, 464 | "requires": { 465 | "aproba": "^1.0.3", 466 | "console-control-strings": "^1.0.0", 467 | "has-unicode": "^2.0.0", 468 | "object-assign": "^4.1.0", 469 | "signal-exit": "^3.0.0", 470 | "string-width": "^1.0.1", 471 | "strip-ansi": "^3.0.1", 472 | "wide-align": "^1.1.0" 473 | } 474 | }, 475 | "glob": { 476 | "version": "7.1.2", 477 | "bundled": true, 478 | "requires": { 479 | "fs.realpath": "^1.0.0", 480 | "inflight": "^1.0.4", 481 | "inherits": "2", 482 | "minimatch": "^3.0.4", 483 | "once": "^1.3.0", 484 | "path-is-absolute": "^1.0.0" 485 | } 486 | }, 487 | "has-unicode": { 488 | "version": "2.0.1", 489 | "bundled": true 490 | }, 491 | "iconv-lite": { 492 | "version": "0.4.23", 493 | "bundled": true, 494 | "requires": { 495 | "safer-buffer": ">= 2.1.2 < 3" 496 | } 497 | }, 498 | "ignore-walk": { 499 | "version": "3.0.1", 500 | "bundled": true, 501 | "requires": { 502 | "minimatch": "^3.0.4" 503 | } 504 | }, 505 | "inflight": { 506 | "version": "1.0.6", 507 | "bundled": true, 508 | "requires": { 509 | "once": "^1.3.0", 510 | "wrappy": "1" 511 | } 512 | }, 513 | "inherits": { 514 | "version": "2.0.3", 515 | "bundled": true 516 | }, 517 | "ini": { 518 | "version": "1.3.5", 519 | "bundled": true 520 | }, 521 | "is-fullwidth-code-point": { 522 | "version": "1.0.0", 523 | "bundled": true, 524 | "requires": { 525 | "number-is-nan": "^1.0.0" 526 | } 527 | }, 528 | "isarray": { 529 | "version": "1.0.0", 530 | "bundled": true 531 | }, 532 | "minimatch": { 533 | "version": "3.0.4", 534 | "bundled": true, 535 | "requires": { 536 | "brace-expansion": "^1.1.7" 537 | } 538 | }, 539 | "minimist": { 540 | "version": "1.2.0", 541 | "bundled": true 542 | }, 543 | "minipass": { 544 | "version": "2.3.3", 545 | "bundled": true, 546 | "requires": { 547 | "safe-buffer": "^5.1.2", 548 | "yallist": "^3.0.0" 549 | } 550 | }, 551 | "minizlib": { 552 | "version": "1.1.0", 553 | "bundled": true, 554 | "requires": { 555 | "minipass": "^2.2.1" 556 | } 557 | }, 558 | "mkdirp": { 559 | "version": "0.5.1", 560 | "bundled": true, 561 | "requires": { 562 | "minimist": "0.0.8" 563 | }, 564 | "dependencies": { 565 | "minimist": { 566 | "version": "0.0.8", 567 | "bundled": true 568 | } 569 | } 570 | }, 571 | "ms": { 572 | "version": "2.0.0", 573 | "bundled": true 574 | }, 575 | "needle": { 576 | "version": "2.2.1", 577 | "bundled": true, 578 | "requires": { 579 | "debug": "^2.1.2", 580 | "iconv-lite": "^0.4.4", 581 | "sax": "^1.2.4" 582 | } 583 | }, 584 | "node-pre-gyp": { 585 | "version": "0.10.3", 586 | "bundled": true, 587 | "requires": { 588 | "detect-libc": "^1.0.2", 589 | "mkdirp": "^0.5.1", 590 | "needle": "^2.2.1", 591 | "nopt": "^4.0.1", 592 | "npm-packlist": "^1.1.6", 593 | "npmlog": "^4.0.2", 594 | "rc": "^1.2.7", 595 | "rimraf": "^2.6.1", 596 | "semver": "^5.3.0", 597 | "tar": "^4" 598 | } 599 | }, 600 | "nopt": { 601 | "version": "4.0.1", 602 | "bundled": true, 603 | "requires": { 604 | "abbrev": "1", 605 | "osenv": "^0.1.4" 606 | } 607 | }, 608 | "npm-bundled": { 609 | "version": "1.0.3", 610 | "bundled": true 611 | }, 612 | "npm-packlist": { 613 | "version": "1.1.11", 614 | "bundled": true, 615 | "requires": { 616 | "ignore-walk": "^3.0.1", 617 | "npm-bundled": "^1.0.1" 618 | } 619 | }, 620 | "npmlog": { 621 | "version": "4.1.2", 622 | "bundled": true, 623 | "requires": { 624 | "are-we-there-yet": "~1.1.2", 625 | "console-control-strings": "~1.1.0", 626 | "gauge": "~2.7.3", 627 | "set-blocking": "~2.0.0" 628 | } 629 | }, 630 | "number-is-nan": { 631 | "version": "1.0.1", 632 | "bundled": true 633 | }, 634 | "object-assign": { 635 | "version": "4.1.1", 636 | "bundled": true 637 | }, 638 | "once": { 639 | "version": "1.4.0", 640 | "bundled": true, 641 | "requires": { 642 | "wrappy": "1" 643 | } 644 | }, 645 | "os-homedir": { 646 | "version": "1.0.2", 647 | "bundled": true 648 | }, 649 | "os-tmpdir": { 650 | "version": "1.0.2", 651 | "bundled": true 652 | }, 653 | "osenv": { 654 | "version": "0.1.5", 655 | "bundled": true, 656 | "requires": { 657 | "os-homedir": "^1.0.0", 658 | "os-tmpdir": "^1.0.0" 659 | } 660 | }, 661 | "path-is-absolute": { 662 | "version": "1.0.1", 663 | "bundled": true 664 | }, 665 | "process-nextick-args": { 666 | "version": "2.0.0", 667 | "bundled": true 668 | }, 669 | "rc": { 670 | "version": "1.2.8", 671 | "bundled": true, 672 | "requires": { 673 | "deep-extend": "^0.6.0", 674 | "ini": "~1.3.0", 675 | "minimist": "^1.2.0", 676 | "strip-json-comments": "~2.0.1" 677 | } 678 | }, 679 | "readable-stream": { 680 | "version": "2.3.6", 681 | "bundled": true, 682 | "requires": { 683 | "core-util-is": "~1.0.0", 684 | "inherits": "~2.0.3", 685 | "isarray": "~1.0.0", 686 | "process-nextick-args": "~2.0.0", 687 | "safe-buffer": "~5.1.1", 688 | "string_decoder": "~1.1.1", 689 | "util-deprecate": "~1.0.1" 690 | } 691 | }, 692 | "rimraf": { 693 | "version": "2.6.2", 694 | "bundled": true, 695 | "requires": { 696 | "glob": "^7.0.5" 697 | } 698 | }, 699 | "safe-buffer": { 700 | "version": "5.1.2", 701 | "bundled": true 702 | }, 703 | "safer-buffer": { 704 | "version": "2.1.2", 705 | "bundled": true 706 | }, 707 | "sax": { 708 | "version": "1.2.4", 709 | "bundled": true 710 | }, 711 | "semver": { 712 | "version": "5.5.0", 713 | "bundled": true 714 | }, 715 | "set-blocking": { 716 | "version": "2.0.0", 717 | "bundled": true 718 | }, 719 | "signal-exit": { 720 | "version": "3.0.2", 721 | "bundled": true 722 | }, 723 | "string-width": { 724 | "version": "1.0.2", 725 | "bundled": true, 726 | "requires": { 727 | "code-point-at": "^1.0.0", 728 | "is-fullwidth-code-point": "^1.0.0", 729 | "strip-ansi": "^3.0.0" 730 | } 731 | }, 732 | "string_decoder": { 733 | "version": "1.1.1", 734 | "bundled": true, 735 | "requires": { 736 | "safe-buffer": "~5.1.0" 737 | } 738 | }, 739 | "strip-ansi": { 740 | "version": "3.0.1", 741 | "bundled": true, 742 | "requires": { 743 | "ansi-regex": "^2.0.0" 744 | } 745 | }, 746 | "strip-json-comments": { 747 | "version": "2.0.1", 748 | "bundled": true 749 | }, 750 | "tar": { 751 | "version": "4.4.4", 752 | "bundled": true, 753 | "requires": { 754 | "chownr": "^1.0.1", 755 | "fs-minipass": "^1.2.5", 756 | "minipass": "^2.3.3", 757 | "minizlib": "^1.1.0", 758 | "mkdirp": "^0.5.0", 759 | "safe-buffer": "^5.1.2", 760 | "yallist": "^3.0.2" 761 | } 762 | }, 763 | "util-deprecate": { 764 | "version": "1.0.2", 765 | "bundled": true 766 | }, 767 | "wide-align": { 768 | "version": "1.1.3", 769 | "bundled": true, 770 | "requires": { 771 | "string-width": "^1.0.2 || 2" 772 | } 773 | }, 774 | "wrappy": { 775 | "version": "1.0.2", 776 | "bundled": true 777 | }, 778 | "yallist": { 779 | "version": "3.0.2", 780 | "bundled": true 781 | } 782 | } 783 | }, 784 | "http-parser-js": { 785 | "version": "0.5.0", 786 | "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.0.tgz", 787 | "integrity": "sha512-cZdEF7r4gfRIq7ezX9J0T+kQmJNOub71dWbgAXVHDct80TKP4MCETtZQ31xyv38UwgzkWPYF/Xc0ge55dW9Z9w==" 788 | }, 789 | "https-proxy-agent": { 790 | "version": "2.2.1", 791 | "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", 792 | "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==", 793 | "requires": { 794 | "agent-base": "^4.1.0", 795 | "debug": "^3.1.0" 796 | } 797 | }, 798 | "iconv-lite": { 799 | "version": "0.4.24", 800 | "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", 801 | "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", 802 | "requires": { 803 | "safer-buffer": ">= 2.1.2 < 3" 804 | } 805 | }, 806 | "inflight": { 807 | "version": "1.0.6", 808 | "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", 809 | "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", 810 | "requires": { 811 | "once": "^1.3.0", 812 | "wrappy": "1" 813 | } 814 | }, 815 | "inherits": { 816 | "version": "2.0.3", 817 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", 818 | "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" 819 | }, 820 | "invert-kv": { 821 | "version": "1.0.0", 822 | "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", 823 | "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=" 824 | }, 825 | "is-fullwidth-code-point": { 826 | "version": "1.0.0", 827 | "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", 828 | "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", 829 | "requires": { 830 | "number-is-nan": "^1.0.0" 831 | } 832 | }, 833 | "is-stream": { 834 | "version": "1.1.0", 835 | "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", 836 | "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" 837 | }, 838 | "isarray": { 839 | "version": "1.0.0", 840 | "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", 841 | "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" 842 | }, 843 | "isomorphic-fetch": { 844 | "version": "2.2.1", 845 | "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-2.2.1.tgz", 846 | "integrity": "sha1-YRrhrPFPXoH3KVB0coGf6XM1WKk=", 847 | "requires": { 848 | "node-fetch": "^1.0.1", 849 | "whatwg-fetch": ">=0.10.0" 850 | } 851 | }, 852 | "lcid": { 853 | "version": "1.0.0", 854 | "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", 855 | "integrity": "sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU=", 856 | "requires": { 857 | "invert-kv": "^1.0.0" 858 | } 859 | }, 860 | "lodash": { 861 | "version": "4.17.11", 862 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", 863 | "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" 864 | }, 865 | "long": { 866 | "version": "3.2.0", 867 | "resolved": "https://registry.npmjs.org/long/-/long-3.2.0.tgz", 868 | "integrity": "sha1-2CG3E4yhy1gcFymQ7xTbIAtcR0s=" 869 | }, 870 | "mime": { 871 | "version": "2.3.1", 872 | "resolved": "https://registry.npmjs.org/mime/-/mime-2.3.1.tgz", 873 | "integrity": "sha512-OEUllcVoydBHGN1z84yfQDimn58pZNNNXgZlHXSboxMlFvgI6MXSWpWKpFRra7H1HxpVhHTkrghfRW49k6yjeg==" 874 | }, 875 | "minimatch": { 876 | "version": "3.0.4", 877 | "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", 878 | "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", 879 | "requires": { 880 | "brace-expansion": "^1.1.7" 881 | } 882 | }, 883 | "minimist": { 884 | "version": "0.0.8", 885 | "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", 886 | "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" 887 | }, 888 | "mkdirp": { 889 | "version": "0.5.1", 890 | "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", 891 | "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", 892 | "requires": { 893 | "minimist": "0.0.8" 894 | } 895 | }, 896 | "ms": { 897 | "version": "2.1.1", 898 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", 899 | "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" 900 | }, 901 | "nan": { 902 | "version": "2.11.1", 903 | "resolved": "https://registry.npmjs.org/nan/-/nan-2.11.1.tgz", 904 | "integrity": "sha512-iji6k87OSXa0CcrLl9z+ZiYSuR2o+c0bGuNmXdrhTQTakxytAFsC56SArGYoiHlJlFoHSnvmhpceZJaXkVuOtA==" 905 | }, 906 | "node-fetch": { 907 | "version": "1.7.3", 908 | "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz", 909 | "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==", 910 | "requires": { 911 | "encoding": "^0.1.11", 912 | "is-stream": "^1.0.1" 913 | } 914 | }, 915 | "number-is-nan": { 916 | "version": "1.0.1", 917 | "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", 918 | "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" 919 | }, 920 | "once": { 921 | "version": "1.4.0", 922 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", 923 | "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", 924 | "requires": { 925 | "wrappy": "1" 926 | } 927 | }, 928 | "optjs": { 929 | "version": "3.2.2", 930 | "resolved": "https://registry.npmjs.org/optjs/-/optjs-3.2.2.tgz", 931 | "integrity": "sha1-aabOicRCpEQDFBrS+bNwvVu29O4=" 932 | }, 933 | "os-locale": { 934 | "version": "1.4.0", 935 | "resolved": "http://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", 936 | "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=", 937 | "requires": { 938 | "lcid": "^1.0.0" 939 | } 940 | }, 941 | "path-is-absolute": { 942 | "version": "1.0.1", 943 | "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", 944 | "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" 945 | }, 946 | "pend": { 947 | "version": "1.2.0", 948 | "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", 949 | "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" 950 | }, 951 | "process-nextick-args": { 952 | "version": "2.0.0", 953 | "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", 954 | "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" 955 | }, 956 | "progress": { 957 | "version": "2.0.0", 958 | "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.0.tgz", 959 | "integrity": "sha1-ihvjZr+Pwj2yvSPxDG/pILQ4nR8=" 960 | }, 961 | "promise-polyfill": { 962 | "version": "7.1.2", 963 | "resolved": "https://registry.npmjs.org/promise-polyfill/-/promise-polyfill-7.1.2.tgz", 964 | "integrity": "sha512-FuEc12/eKqqoRYIGBrUptCBRhobL19PS2U31vMNTfyck1FxPyMfgsXyW4Mav85y/ZN1hop3hOwRlUDok23oYfQ==" 965 | }, 966 | "protobufjs": { 967 | "version": "5.0.3", 968 | "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-5.0.3.tgz", 969 | "integrity": "sha512-55Kcx1MhPZX0zTbVosMQEO5R6/rikNXd9b6RQK4KSPcrSIIwoXTtebIczUrXlwaSrbz4x8XUVThGPob1n8I4QA==", 970 | "requires": { 971 | "ascli": "~1", 972 | "bytebuffer": "~5", 973 | "glob": "^7.0.5", 974 | "yargs": "^3.10.0" 975 | } 976 | }, 977 | "proxy-from-env": { 978 | "version": "1.0.0", 979 | "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz", 980 | "integrity": "sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4=" 981 | }, 982 | "puppeteer": { 983 | "version": "1.9.0", 984 | "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.9.0.tgz", 985 | "integrity": "sha512-GH4PmhJf9wBRAPvtJkEJLAvdNNOofZortmBZSj8cGWYni98GUFqsf66blOEfJbo5B8l0KG5HR2d/W2MejnUrzg==", 986 | "requires": { 987 | "debug": "^3.1.0", 988 | "extract-zip": "^1.6.6", 989 | "https-proxy-agent": "^2.2.1", 990 | "mime": "^2.0.3", 991 | "progress": "^2.0.0", 992 | "proxy-from-env": "^1.0.0", 993 | "rimraf": "^2.6.1", 994 | "ws": "^5.1.1" 995 | } 996 | }, 997 | "readable-stream": { 998 | "version": "2.3.6", 999 | "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", 1000 | "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", 1001 | "requires": { 1002 | "core-util-is": "~1.0.0", 1003 | "inherits": "~2.0.3", 1004 | "isarray": "~1.0.0", 1005 | "process-nextick-args": "~2.0.0", 1006 | "safe-buffer": "~5.1.1", 1007 | "string_decoder": "~1.1.1", 1008 | "util-deprecate": "~1.0.1" 1009 | } 1010 | }, 1011 | "rimraf": { 1012 | "version": "2.6.2", 1013 | "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz", 1014 | "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==", 1015 | "requires": { 1016 | "glob": "^7.0.5" 1017 | } 1018 | }, 1019 | "safe-buffer": { 1020 | "version": "5.1.2", 1021 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", 1022 | "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" 1023 | }, 1024 | "safer-buffer": { 1025 | "version": "2.1.2", 1026 | "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", 1027 | "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" 1028 | }, 1029 | "string-width": { 1030 | "version": "1.0.2", 1031 | "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", 1032 | "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", 1033 | "requires": { 1034 | "code-point-at": "^1.0.0", 1035 | "is-fullwidth-code-point": "^1.0.0", 1036 | "strip-ansi": "^3.0.0" 1037 | } 1038 | }, 1039 | "string_decoder": { 1040 | "version": "1.1.1", 1041 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", 1042 | "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", 1043 | "requires": { 1044 | "safe-buffer": "~5.1.0" 1045 | } 1046 | }, 1047 | "strip-ansi": { 1048 | "version": "3.0.1", 1049 | "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", 1050 | "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", 1051 | "requires": { 1052 | "ansi-regex": "^2.0.0" 1053 | } 1054 | }, 1055 | "tslib": { 1056 | "version": "1.9.0", 1057 | "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.0.tgz", 1058 | "integrity": "sha512-f/qGG2tUkrISBlQZEjEqoZ3B2+npJjIf04H1wuAv9iA8i04Icp+61KRXxFdha22670NJopsZCIjhC3SnjPRKrQ==" 1059 | }, 1060 | "typedarray": { 1061 | "version": "0.0.6", 1062 | "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", 1063 | "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" 1064 | }, 1065 | "util-deprecate": { 1066 | "version": "1.0.2", 1067 | "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", 1068 | "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" 1069 | }, 1070 | "websocket-driver": { 1071 | "version": "0.7.0", 1072 | "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.0.tgz", 1073 | "integrity": "sha1-DK+dLXVdk67gSdS90NP+LMoqJOs=", 1074 | "requires": { 1075 | "http-parser-js": ">=0.4.0", 1076 | "websocket-extensions": ">=0.1.1" 1077 | } 1078 | }, 1079 | "websocket-extensions": { 1080 | "version": "0.1.3", 1081 | "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.3.tgz", 1082 | "integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==" 1083 | }, 1084 | "whatwg-fetch": { 1085 | "version": "3.0.0", 1086 | "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.0.0.tgz", 1087 | "integrity": "sha512-9GSJUgz1D4MfyKU7KRqwOjXCXTqWdFNvEr7eUBYchQiVc744mqK/MzXPNR2WsPkmkOa4ywfg8C2n8h+13Bey1Q==" 1088 | }, 1089 | "window-size": { 1090 | "version": "0.1.4", 1091 | "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.1.4.tgz", 1092 | "integrity": "sha1-+OGqHuWlPsW/FR/6CXQqatdpeHY=" 1093 | }, 1094 | "wrap-ansi": { 1095 | "version": "2.1.0", 1096 | "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", 1097 | "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", 1098 | "requires": { 1099 | "string-width": "^1.0.1", 1100 | "strip-ansi": "^3.0.1" 1101 | } 1102 | }, 1103 | "wrappy": { 1104 | "version": "1.0.2", 1105 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", 1106 | "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" 1107 | }, 1108 | "ws": { 1109 | "version": "5.2.2", 1110 | "resolved": "https://registry.npmjs.org/ws/-/ws-5.2.2.tgz", 1111 | "integrity": "sha512-jaHFD6PFv6UgoIVda6qZllptQsMlDEJkTQcybzzXDYM1XO9Y8em691FGMPmM46WGyLU4z9KMgQN+qrux/nhlHA==", 1112 | "requires": { 1113 | "async-limiter": "~1.0.0" 1114 | } 1115 | }, 1116 | "xmlhttprequest": { 1117 | "version": "1.8.0", 1118 | "resolved": "https://registry.npmjs.org/xmlhttprequest/-/xmlhttprequest-1.8.0.tgz", 1119 | "integrity": "sha1-Z/4HXFwk/vOfnWX197f+dRcZaPw=" 1120 | }, 1121 | "y18n": { 1122 | "version": "3.2.1", 1123 | "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz", 1124 | "integrity": "sha1-bRX7qITAhnnA136I53WegR4H+kE=" 1125 | }, 1126 | "yargs": { 1127 | "version": "3.32.0", 1128 | "resolved": "http://registry.npmjs.org/yargs/-/yargs-3.32.0.tgz", 1129 | "integrity": "sha1-AwiOnr+edWtpdRYR0qXvWRSCyZU=", 1130 | "requires": { 1131 | "camelcase": "^2.0.1", 1132 | "cliui": "^3.0.3", 1133 | "decamelize": "^1.1.1", 1134 | "os-locale": "^1.4.0", 1135 | "string-width": "^1.0.1", 1136 | "window-size": "^0.1.4", 1137 | "y18n": "^3.2.0" 1138 | } 1139 | }, 1140 | "yauzl": { 1141 | "version": "2.4.1", 1142 | "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz", 1143 | "integrity": "sha1-lSj0QtqxsihOWLQ3m7GU4i4MQAU=", 1144 | "requires": { 1145 | "fd-slicer": "~1.0.1" 1146 | } 1147 | } 1148 | } 1149 | } 1150 | -------------------------------------------------------------------------------- /scraper.js: -------------------------------------------------------------------------------- 1 | const puppeteer = require("puppeteer"); 2 | const fs = require('fs'); 3 | const loadFirebase = require('./lib/db.js'); 4 | 5 | (async () => { 6 | 7 | // Extract partners on the page, recursively check the next page in the URL pattern 8 | const extractPartners = async url => { 9 | // Scrape the data we want 10 | const page = await browser.newPage(); 11 | await page.goto(url); 12 | const partnersOnPage = await page.evaluate(() => 13 | Array.from(document.querySelectorAll("div.compact")).map(compact => ({ 14 | title: compact.querySelector("h3.title").innerText.trim(), 15 | logo: compact.querySelector(".logo img").src 16 | })) 17 | ); 18 | await page.close(); 19 | 20 | // Recursively scrape the next page 21 | if (partnersOnPage.length < 1) { 22 | // Terminate if no partners exist 23 | return partnersOnPage 24 | } else { 25 | // Go fetch the next page ?page=X+1 26 | const nextPageNumber = parseInt(url.match(/page=(\d+)$/)[1], 10) + 1; 27 | const nextUrl = `https://marketingplatform.google.com/about/partners/find-a-partner?page=${nextPageNumber}`; 28 | 29 | return partnersOnPage.concat(await extractPartners(nextUrl)) 30 | } 31 | }; 32 | 33 | let partners; 34 | // If there's a local JSON, don't fetch anything 35 | const rawdata = fs.readFileSync('partners.json'); 36 | const browser = await puppeteer.launch(); 37 | 38 | if (rawdata) { 39 | partners = JSON.parse(rawdata) 40 | } else { 41 | const firstUrl = 42 | "https://marketingplatform.google.com/about/partners/find-a-partner?page=1"; 43 | partners = await extractPartners(firstUrl); 44 | } 45 | 46 | 47 | 48 | 49 | // Todo: Update database with partners 50 | console.log(partners); 51 | 52 | // loadFirebase().firestore().collection('agencies') 53 | // .limit(10) 54 | // .get() 55 | // .then(snapshot => { 56 | // let data = [] 57 | // snapshot.forEach((doc) => { 58 | // data.push({ 59 | // id: doc.id, 60 | // ...doc.data() 61 | // }) 62 | // }) 63 | // return { agencies: data } 64 | // }) 65 | 66 | 67 | // // Save to JSON file 68 | // const data = JSON.stringify(partners); 69 | // fs.writeFileSync('partners.json', data); 70 | 71 | 72 | await browser.close(); 73 | })(); 74 | --------------------------------------------------------------------------------