├── .gitignore
├── README.md
├── constants.js
├── helpers.js
├── package-lock.json
├── package.json
├── scraper.js
└── server.js


/.gitignore:
--------------------------------------------------------------------------------
 1 | #data directory
 2 | /data
 3 | 
 4 | #csv files
 5 | *.csv
 6 | 
 7 | #jshint config
 8 | .jshintrc
 9 | 
10 | # Logs
11 | logs
12 | *.log
13 | npm-debug.log*
14 | yarn-debug.log*
15 | yarn-error.log*
16 | 
17 | # Runtime data
18 | pids
19 | *.pid
20 | *.seed
21 | *.pid.lock
22 | 
23 | # Directory for instrumented libs generated by jscoverage/JSCover
24 | lib-cov
25 | 
26 | # Coverage directory used by tools like istanbul
27 | coverage
28 | 
29 | # nyc test coverage
30 | .nyc_output
31 | 
32 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
33 | .grunt
34 | 
35 | # Bower dependency directory (https://bower.io/)
36 | bower_components
37 | 
38 | # node-waf configuration
39 | .lock-wscript
40 | 
41 | # Compiled binary addons (http://nodejs.org/api/addons.html)
42 | build/Release
43 | 
44 | # Dependency directories
45 | node_modules/
46 | jspm_packages/
47 | 
48 | # Typescript v1 declaration files
49 | typings/
50 | 
51 | # Optional npm cache directory
52 | .npm
53 | 
54 | # Optional eslint cache
55 | .eslintcache
56 | 
57 | # Optional REPL history
58 | .node_repl_history
59 | 
60 | # Output of 'npm pack'
61 | *.tgz
62 | 
63 | # Yarn Integrity file
64 | .yarn-integrity
65 | 
66 | # dotenv environment variables file
67 | .env
68 | 
69 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Node-Twitter-Scraper
 2 | Scraping twitter using Puppeteer and Node
 3 | 
 4 | This is a work in progress.
 5 | 
 6 | Make sure you enter the email address and password for your email in the helpers.js file 
 7 | 
 8 | # Running the package
 9 | ```bash
10 | npm install
11 | ```
12 | to install all dependencies. Then connect to your mongo server and `npm start` to run everything.
13 | 


--------------------------------------------------------------------------------
/constants.js:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/dansalerno712/Node-Twitter-Scraper/e3002ac639aa95b138d6661c8abd0854e363c372/constants.js


--------------------------------------------------------------------------------
/helpers.js:
--------------------------------------------------------------------------------
  1 | const chunk = require('chunk-date-range');
  2 | const dateformat = require('dateformat');
  3 | const createCsvWriter = require('csv-writer').createObjectCsvWriter;
  4 | const nodemailer = require('nodemailer');
  5 | 
  6 | module.exports = {
  7 |     /**
  8 |      * Sends an email containing the download link for the tweet csv
  9 |      * @param  {string} toEmail Email to send to
 10 |      * @param  {string} link    Link to download the csv
 11 |      */
 12 |     "sendEmail": function(toEmail, link) {
 13 |         var transporter = nodemailer.createTransport({
 14 |             service: 'gmail',
 15 |             auth: {
 16 |                 user: "<email>",
 17 |                 pass: "<password>"
 18 |             }
 19 |         });
 20 | 
 21 |         var mailOptions = {
 22 |             from: "<email>",
 23 |             to: toEmail + "",
 24 |             subject: "Your Tweet Download Link",
 25 |             text: link + ""
 26 |         };
 27 | 
 28 |         transporter.sendMail(mailOptions, function(error, info) {
 29 |             if (error) {
 30 |                 throw error
 31 |             } else {
 32 |                 console.log("Email sent: " + info.response);
 33 |             }
 34 |         });
 35 |     },
 36 |     /*
 37 |      * Function to write an array of tweets to a csv
 38 |      * @input: tweets: An array of tweet objects
 39 |      * @input: path: A string of the path to save the csv at
 40 |      * @return: Nothing, but a csv is created
 41 |      */
 42 |     "toCSV": function(tweets, path) {
 43 |         // create header schema
 44 |         const csvWriter = createCsvWriter({
 45 |             path: path,
 46 |             header: [{
 47 |                 id: "text",
 48 |                 title: "Text"
 49 |             }, {
 50 |                 id: "timestamp",
 51 |                 title: "Timestamp"
 52 |             }, {
 53 |                 id: "id",
 54 |                 title: "ID"
 55 |             }, {
 56 |                 id: "retweets",
 57 |                 title: "Retweets"
 58 |             }, {
 59 |                 id: "likes",
 60 |                 title: "Likes"
 61 |             }]
 62 |         });
 63 | 
 64 |         // output to csv
 65 |         csvWriter.writeRecords(tweets)
 66 |             .then(() => {
 67 |                 console.log("Done writing to csv");
 68 |             });
 69 |     },
 70 |     /*
 71 |      * Function to split the Start/End Date into either chunks or by Date/Week/Month/Year
 72 |      * @input startDate: A string in the format YYYY/MM/DD
 73 |      * @input endDate: A string in the format YYYY/MM/DD
 74 |      * @input chunks: Either a number that specifies how many equal chunks the user wants to 
 75 |      * split the date range into or a String day|week|month|year that splits the date range that way
 76 |      *
 77 |      * @return: An array of {startDate, endDate} objects where start and end date are in the format
 78 |      *  of a YYYY/MM/DD string
 79 |      */
 80 |     "splitDateRange": function(startDate, endDate, chunks) {
 81 |         let start = new Date(startDate);
 82 |         let end = new Date(endDate);
 83 |         let ret = chunk(start, end, chunks);
 84 |         return ret.map(function(dateRange) {
 85 |             return {
 86 |                 'start': dateformat(dateRange.start, "yyyy-mm-dd"),
 87 |                 'end': dateformat(dateRange.end, "yyyy-mm-dd")
 88 |             };
 89 |         });
 90 |     },
 91 |     /*
 92 |      * Function to scroll on a page until all lazy loading has been done
 93 |      * @input page: the page you want to scroll on
 94 |      */
 95 |     "autoScroll": function(page) {
 96 |         // evaluate some javascript
 97 |         return page.evaluate(function() {
 98 |             return new Promise(function(resolve, reject) {
 99 |                 let totalHeight = 0;
100 | 
101 |                 //distance per scroll
102 |                 let distance = 1000;
103 |                 let timer = setInterval(function() {
104 |                     //get current height
105 |                     let scrollHeight = document.body.scrollHeight;
106 | 
107 |                     //scroll and increment
108 |                     window.scrollBy(0, distance);
109 |                     totalHeight += distance;
110 | 
111 |                     //if we didnt scroll, lazy loading must be done, so return
112 |                     if (totalHeight >= scrollHeight) {
113 |                         clearInterval(timer);
114 |                         resolve();
115 |                     }
116 |                     //how long to wait between scrolls
117 |                 }, 1000);
118 |             });
119 |         });
120 |     }
121 | };


--------------------------------------------------------------------------------
/package-lock.json:
--------------------------------------------------------------------------------
   1 | {
   2 |   "name": "node-twitter-scraper",
   3 |   "version": "1.0.0",
   4 |   "lockfileVersion": 1,
   5 |   "requires": true,
   6 |   "dependencies": {
   7 |     "@types/node": {
   8 |       "version": "12.6.0",
   9 |       "resolved": "https://registry.npmjs.org/@types/node/-/node-12.6.0.tgz",
  10 |       "integrity": "sha512-dVeOVH/lhZ2Cki5Emh0aKeXUcWG1+EDTkqyzdgPe0ZjzgvBhzSFlogc6rm8uUd0I+XGK5fcp9DsMv5Wofe0/3w=="
  11 |     },
  12 |     "accepts": {
  13 |       "version": "1.3.5",
  14 |       "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz",
  15 |       "integrity": "sha1-63d99gEXI6OxTopywIBcjoZ0a9I=",
  16 |       "requires": {
  17 |         "mime-types": "~2.1.18",
  18 |         "negotiator": "0.6.1"
  19 |       }
  20 |     },
  21 |     "agent-base": {
  22 |       "version": "4.3.0",
  23 |       "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz",
  24 |       "integrity": "sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg==",
  25 |       "requires": {
  26 |         "es6-promisify": "^5.0.0"
  27 |       }
  28 |     },
  29 |     "ansicolors": {
  30 |       "version": "0.2.1",
  31 |       "resolved": "https://registry.npmjs.org/ansicolors/-/ansicolors-0.2.1.tgz",
  32 |       "integrity": "sha1-vgiVmQl7dKXJxKhKDNvNtivYeu8="
  33 |     },
  34 |     "any-promise": {
  35 |       "version": "1.3.0",
  36 |       "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz",
  37 |       "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8="
  38 |     },
  39 |     "array-flatten": {
  40 |       "version": "1.1.1",
  41 |       "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
  42 |       "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
  43 |     },
  44 |     "async-limiter": {
  45 |       "version": "1.0.0",
  46 |       "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz",
  47 |       "integrity": "sha1-ePrtjD0HSrgfIrTphdeehzj3IPg="
  48 |     },
  49 |     "balanced-match": {
  50 |       "version": "1.0.0",
  51 |       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
  52 |       "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
  53 |     },
  54 |     "bluebird": {
  55 |       "version": "3.5.5",
  56 |       "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.5.tgz",
  57 |       "integrity": "sha512-5am6HnnfN+urzt4yfg7IgTbotDjIT/u8AJpEt0sIU9FtXfVeezXAPKswrG+xKUCOYAINpSdgZVDU6QFh+cuH3w=="
  58 |     },
  59 |     "body-parser": {
  60 |       "version": "1.18.2",
  61 |       "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz",
  62 |       "integrity": "sha1-h2eKGdhLR9hZuDGZvVm84iKxBFQ=",
  63 |       "requires": {
  64 |         "bytes": "3.0.0",
  65 |         "content-type": "~1.0.4",
  66 |         "debug": "2.6.9",
  67 |         "depd": "~1.1.1",
  68 |         "http-errors": "~1.6.2",
  69 |         "iconv-lite": "0.4.19",
  70 |         "on-finished": "~2.3.0",
  71 |         "qs": "6.5.1",
  72 |         "raw-body": "2.3.2",
  73 |         "type-is": "~1.6.15"
  74 |       }
  75 |     },
  76 |     "brace-expansion": {
  77 |       "version": "1.1.11",
  78 |       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
  79 |       "integrity": "sha1-PH/L9SnYcibz0vUrlm/1Jx60Qd0=",
  80 |       "requires": {
  81 |         "balanced-match": "^1.0.0",
  82 |         "concat-map": "0.0.1"
  83 |       }
  84 |     },
  85 |     "bson": {
  86 |       "version": "1.1.5",
  87 |       "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.5.tgz",
  88 |       "integrity": "sha512-kDuEzldR21lHciPQAIulLs1LZlCXdLziXI6Mb/TDkwXhb//UORJNPXgcRs2CuO4H0DcMkpfT3/ySsP3unoZjBg=="
  89 |     },
  90 |     "buffer-from": {
  91 |       "version": "1.1.1",
  92 |       "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
  93 |       "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
  94 |     },
  95 |     "bytes": {
  96 |       "version": "3.0.0",
  97 |       "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
  98 |       "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg="
  99 |     },
 100 |     "cardinal": {
 101 |       "version": "1.0.0",
 102 |       "resolved": "https://registry.npmjs.org/cardinal/-/cardinal-1.0.0.tgz",
 103 |       "integrity": "sha1-UOIcGwqjdyn5N33vGWtanOyTLuk=",
 104 |       "requires": {
 105 |         "ansicolors": "~0.2.1",
 106 |         "redeyed": "~1.0.0"
 107 |       }
 108 |     },
 109 |     "chunk-date-range": {
 110 |       "version": "0.1.0",
 111 |       "resolved": "https://registry.npmjs.org/chunk-date-range/-/chunk-date-range-0.1.0.tgz",
 112 |       "integrity": "sha1-xtJweyWIUa6WaJtVBcPVWJzUCIc=",
 113 |       "requires": {
 114 |         "date-math": "0.0.1"
 115 |       }
 116 |     },
 117 |     "concat-map": {
 118 |       "version": "0.0.1",
 119 |       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
 120 |       "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
 121 |     },
 122 |     "concat-stream": {
 123 |       "version": "1.6.2",
 124 |       "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
 125 |       "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
 126 |       "requires": {
 127 |         "buffer-from": "^1.0.0",
 128 |         "inherits": "^2.0.3",
 129 |         "readable-stream": "^2.2.2",
 130 |         "typedarray": "^0.0.6"
 131 |       }
 132 |     },
 133 |     "content-disposition": {
 134 |       "version": "0.5.2",
 135 |       "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz",
 136 |       "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ="
 137 |     },
 138 |     "content-type": {
 139 |       "version": "1.0.4",
 140 |       "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz",
 141 |       "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA=="
 142 |     },
 143 |     "cookie": {
 144 |       "version": "0.3.1",
 145 |       "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz",
 146 |       "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s="
 147 |     },
 148 |     "cookie-signature": {
 149 |       "version": "1.0.6",
 150 |       "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
 151 |       "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw="
 152 |     },
 153 |     "core-util-is": {
 154 |       "version": "1.0.2",
 155 |       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
 156 |       "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
 157 |     },
 158 |     "csv-writer": {
 159 |       "version": "1.0.0",
 160 |       "resolved": "https://registry.npmjs.org/csv-writer/-/csv-writer-1.0.0.tgz",
 161 |       "integrity": "sha512-BhOFA4OIlC1cngdDLDmKJrVCDqoye0kKtZhRs93X0BDlWgHoVIY8zWFZXMgeSWuOCLTwdm0aBV4MMRpR8Y83/g=="
 162 |     },
 163 |     "date-math": {
 164 |       "version": "0.0.1",
 165 |       "resolved": "https://registry.npmjs.org/date-math/-/date-math-0.0.1.tgz",
 166 |       "integrity": "sha1-ih1S0CT9lKeVEXNeynIDYVbV77Y="
 167 |     },
 168 |     "dateformat": {
 169 |       "version": "3.0.3",
 170 |       "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz",
 171 |       "integrity": "sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q=="
 172 |     },
 173 |     "debug": {
 174 |       "version": "2.6.9",
 175 |       "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
 176 |       "integrity": "sha1-XRKFFd8TT/Mn6QpMk/Tgd6U2NB8=",
 177 |       "requires": {
 178 |         "ms": "2.0.0"
 179 |       }
 180 |     },
 181 |     "denque": {
 182 |       "version": "1.2.3",
 183 |       "resolved": "https://registry.npmjs.org/denque/-/denque-1.2.3.tgz",
 184 |       "integrity": "sha512-BOjyD1zPf7gqgXlXBCnCsz84cbRNfqpQNvWOUiw3Onu9s7a2afW2LyHzctoie/2KELfUoZkNHTnW02C3hCU20w=="
 185 |     },
 186 |     "depd": {
 187 |       "version": "1.1.2",
 188 |       "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
 189 |       "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak="
 190 |     },
 191 |     "destroy": {
 192 |       "version": "1.0.4",
 193 |       "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
 194 |       "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
 195 |     },
 196 |     "dottie": {
 197 |       "version": "2.0.1",
 198 |       "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.1.tgz",
 199 |       "integrity": "sha512-ch5OQgvGDK2u8pSZeSYAQaV/lczImd7pMJ7BcEPXmnFVjy4yJIzP6CsODJUTH8mg1tyH1Z2abOiuJO3DjZ/GBw=="
 200 |     },
 201 |     "ee-first": {
 202 |       "version": "1.1.1",
 203 |       "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
 204 |       "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
 205 |     },
 206 |     "encodeurl": {
 207 |       "version": "1.0.2",
 208 |       "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
 209 |       "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k="
 210 |     },
 211 |     "es6-promise": {
 212 |       "version": "4.2.8",
 213 |       "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz",
 214 |       "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w=="
 215 |     },
 216 |     "es6-promisify": {
 217 |       "version": "5.0.0",
 218 |       "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz",
 219 |       "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=",
 220 |       "requires": {
 221 |         "es6-promise": "^4.0.3"
 222 |       }
 223 |     },
 224 |     "escape-html": {
 225 |       "version": "1.0.3",
 226 |       "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
 227 |       "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg="
 228 |     },
 229 |     "esprima": {
 230 |       "version": "3.0.0",
 231 |       "resolved": "https://registry.npmjs.org/esprima/-/esprima-3.0.0.tgz",
 232 |       "integrity": "sha1-U88kes2ncxPlUcOqLnM0LT+099k="
 233 |     },
 234 |     "etag": {
 235 |       "version": "1.8.1",
 236 |       "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
 237 |       "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc="
 238 |     },
 239 |     "express": {
 240 |       "version": "4.16.3",
 241 |       "resolved": "https://registry.npmjs.org/express/-/express-4.16.3.tgz",
 242 |       "integrity": "sha1-avilAjUNsyRuzEvs9rWjTSL37VM=",
 243 |       "requires": {
 244 |         "accepts": "~1.3.5",
 245 |         "array-flatten": "1.1.1",
 246 |         "body-parser": "1.18.2",
 247 |         "content-disposition": "0.5.2",
 248 |         "content-type": "~1.0.4",
 249 |         "cookie": "0.3.1",
 250 |         "cookie-signature": "1.0.6",
 251 |         "debug": "2.6.9",
 252 |         "depd": "~1.1.2",
 253 |         "encodeurl": "~1.0.2",
 254 |         "escape-html": "~1.0.3",
 255 |         "etag": "~1.8.1",
 256 |         "finalhandler": "1.1.1",
 257 |         "fresh": "0.5.2",
 258 |         "merge-descriptors": "1.0.1",
 259 |         "methods": "~1.1.2",
 260 |         "on-finished": "~2.3.0",
 261 |         "parseurl": "~1.3.2",
 262 |         "path-to-regexp": "0.1.7",
 263 |         "proxy-addr": "~2.0.3",
 264 |         "qs": "6.5.1",
 265 |         "range-parser": "~1.2.0",
 266 |         "safe-buffer": "5.1.1",
 267 |         "send": "0.16.2",
 268 |         "serve-static": "1.13.2",
 269 |         "setprototypeof": "1.1.0",
 270 |         "statuses": "~1.4.0",
 271 |         "type-is": "~1.6.16",
 272 |         "utils-merge": "1.0.1",
 273 |         "vary": "~1.1.2"
 274 |       }
 275 |     },
 276 |     "extract-zip": {
 277 |       "version": "1.6.7",
 278 |       "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz",
 279 |       "integrity": "sha1-qEC0uK9kAyZMjbV/Txp0Mz74H+k=",
 280 |       "requires": {
 281 |         "concat-stream": "1.6.2",
 282 |         "debug": "2.6.9",
 283 |         "mkdirp": "0.5.1",
 284 |         "yauzl": "2.4.1"
 285 |       }
 286 |     },
 287 |     "fd-slicer": {
 288 |       "version": "1.0.1",
 289 |       "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz",
 290 |       "integrity": "sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU=",
 291 |       "requires": {
 292 |         "pend": "~1.2.0"
 293 |       }
 294 |     },
 295 |     "finalhandler": {
 296 |       "version": "1.1.1",
 297 |       "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz",
 298 |       "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==",
 299 |       "requires": {
 300 |         "debug": "2.6.9",
 301 |         "encodeurl": "~1.0.2",
 302 |         "escape-html": "~1.0.3",
 303 |         "on-finished": "~2.3.0",
 304 |         "parseurl": "~1.3.2",
 305 |         "statuses": "~1.4.0",
 306 |         "unpipe": "~1.0.0"
 307 |       }
 308 |     },
 309 |     "forwarded": {
 310 |       "version": "0.1.2",
 311 |       "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
 312 |       "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
 313 |     },
 314 |     "fresh": {
 315 |       "version": "0.5.2",
 316 |       "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
 317 |       "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac="
 318 |     },
 319 |     "fs.realpath": {
 320 |       "version": "1.0.0",
 321 |       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
 322 |       "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
 323 |     },
 324 |     "generate-function": {
 325 |       "version": "2.0.0",
 326 |       "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz",
 327 |       "integrity": "sha1-aFj+fAlpt9TpCTM3ZHrHn2DfvnQ="
 328 |     },
 329 |     "glob": {
 330 |       "version": "7.1.4",
 331 |       "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
 332 |       "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==",
 333 |       "requires": {
 334 |         "fs.realpath": "^1.0.0",
 335 |         "inflight": "^1.0.4",
 336 |         "inherits": "2",
 337 |         "minimatch": "^3.0.4",
 338 |         "once": "^1.3.0",
 339 |         "path-is-absolute": "^1.0.0"
 340 |       }
 341 |     },
 342 |     "http-errors": {
 343 |       "version": "1.6.3",
 344 |       "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz",
 345 |       "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=",
 346 |       "requires": {
 347 |         "depd": "~1.1.2",
 348 |         "inherits": "2.0.3",
 349 |         "setprototypeof": "1.1.0",
 350 |         "statuses": ">= 1.4.0 < 2"
 351 |       }
 352 |     },
 353 |     "https-proxy-agent": {
 354 |       "version": "2.2.4",
 355 |       "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.4.tgz",
 356 |       "integrity": "sha512-OmvfoQ53WLjtA9HeYP9RNrWMJzzAz1JGaSFr1nijg0PVR1JaD/xbJq1mdEIIlxGpXp9eSe/O2LgU9DJmTPd0Eg==",
 357 |       "requires": {
 358 |         "agent-base": "^4.3.0",
 359 |         "debug": "^3.1.0"
 360 |       },
 361 |       "dependencies": {
 362 |         "debug": {
 363 |           "version": "3.2.6",
 364 |           "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
 365 |           "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
 366 |           "requires": {
 367 |             "ms": "^2.1.1"
 368 |           }
 369 |         },
 370 |         "ms": {
 371 |           "version": "2.1.2",
 372 |           "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
 373 |           "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
 374 |         }
 375 |       }
 376 |     },
 377 |     "iconv-lite": {
 378 |       "version": "0.4.19",
 379 |       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz",
 380 |       "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ=="
 381 |     },
 382 |     "inflection": {
 383 |       "version": "1.12.0",
 384 |       "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.12.0.tgz",
 385 |       "integrity": "sha1-ogCTVlbW9fa8TcdQLhrstwMihBY="
 386 |     },
 387 |     "inflight": {
 388 |       "version": "1.0.6",
 389 |       "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
 390 |       "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
 391 |       "requires": {
 392 |         "once": "^1.3.0",
 393 |         "wrappy": "1"
 394 |       }
 395 |     },
 396 |     "inherits": {
 397 |       "version": "2.0.3",
 398 |       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
 399 |       "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
 400 |     },
 401 |     "ipaddr.js": {
 402 |       "version": "1.6.0",
 403 |       "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.6.0.tgz",
 404 |       "integrity": "sha1-4/o1e3c9phnybpXwSdBVxyeW+Gs="
 405 |     },
 406 |     "isarray": {
 407 |       "version": "1.0.0",
 408 |       "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
 409 |       "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
 410 |     },
 411 |     "long": {
 412 |       "version": "4.0.0",
 413 |       "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz",
 414 |       "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA=="
 415 |     },
 416 |     "media-typer": {
 417 |       "version": "0.3.0",
 418 |       "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
 419 |       "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="
 420 |     },
 421 |     "memory-pager": {
 422 |       "version": "1.5.0",
 423 |       "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz",
 424 |       "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
 425 |       "optional": true
 426 |     },
 427 |     "merge-descriptors": {
 428 |       "version": "1.0.1",
 429 |       "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
 430 |       "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E="
 431 |     },
 432 |     "methods": {
 433 |       "version": "1.1.2",
 434 |       "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
 435 |       "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4="
 436 |     },
 437 |     "mime": {
 438 |       "version": "2.4.4",
 439 |       "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz",
 440 |       "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA=="
 441 |     },
 442 |     "mime-db": {
 443 |       "version": "1.33.0",
 444 |       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz",
 445 |       "integrity": "sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ=="
 446 |     },
 447 |     "mime-types": {
 448 |       "version": "2.1.18",
 449 |       "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz",
 450 |       "integrity": "sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==",
 451 |       "requires": {
 452 |         "mime-db": "~1.33.0"
 453 |       }
 454 |     },
 455 |     "minimatch": {
 456 |       "version": "3.0.4",
 457 |       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
 458 |       "integrity": "sha1-UWbihkV/AzBgZL5Ul+jbsMPTIIM=",
 459 |       "requires": {
 460 |         "brace-expansion": "^1.1.7"
 461 |       }
 462 |     },
 463 |     "minimist": {
 464 |       "version": "0.0.8",
 465 |       "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
 466 |       "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
 467 |     },
 468 |     "mkdirp": {
 469 |       "version": "0.5.1",
 470 |       "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
 471 |       "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
 472 |       "requires": {
 473 |         "minimist": "0.0.8"
 474 |       }
 475 |     },
 476 |     "moment": {
 477 |       "version": "2.24.0",
 478 |       "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz",
 479 |       "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg=="
 480 |     },
 481 |     "moment-timezone": {
 482 |       "version": "0.5.26",
 483 |       "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.26.tgz",
 484 |       "integrity": "sha512-sFP4cgEKTCymBBKgoxZjYzlSovC20Y6J7y3nanDc5RoBIXKlZhoYwBoZGe3flwU6A372AcRwScH8KiwV6zjy1g==",
 485 |       "requires": {
 486 |         "moment": ">= 2.9.0"
 487 |       }
 488 |     },
 489 |     "mongodb": {
 490 |       "version": "3.1.13",
 491 |       "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-3.1.13.tgz",
 492 |       "integrity": "sha512-sz2dhvBZQWf3LRNDhbd30KHVzdjZx9IKC0L+kSZ/gzYquCF5zPOgGqRz6sSCqYZtKP2ekB4nfLxhGtzGHnIKxA==",
 493 |       "requires": {
 494 |         "mongodb-core": "3.1.11",
 495 |         "safe-buffer": "^5.1.2"
 496 |       },
 497 |       "dependencies": {
 498 |         "safe-buffer": {
 499 |           "version": "5.2.1",
 500 |           "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
 501 |           "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
 502 |         }
 503 |       }
 504 |     },
 505 |     "mongodb-core": {
 506 |       "version": "3.1.11",
 507 |       "resolved": "https://registry.npmjs.org/mongodb-core/-/mongodb-core-3.1.11.tgz",
 508 |       "integrity": "sha512-rD2US2s5qk/ckbiiGFHeu+yKYDXdJ1G87F6CG3YdaZpzdOm5zpoAZd/EKbPmFO6cQZ+XVXBXBJ660sSI0gc6qg==",
 509 |       "requires": {
 510 |         "bson": "^1.1.0",
 511 |         "require_optional": "^1.0.1",
 512 |         "safe-buffer": "^5.1.2",
 513 |         "saslprep": "^1.0.0"
 514 |       },
 515 |       "dependencies": {
 516 |         "safe-buffer": {
 517 |           "version": "5.2.1",
 518 |           "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
 519 |           "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
 520 |         }
 521 |       }
 522 |     },
 523 |     "ms": {
 524 |       "version": "2.0.0",
 525 |       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
 526 |       "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
 527 |     },
 528 |     "mysql2": {
 529 |       "version": "1.5.3",
 530 |       "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-1.5.3.tgz",
 531 |       "integrity": "sha512-Oov36YQSeciNP9SeqE7je4eWgeGADOorXLmsqhtxOJmPGUOJSNJT0s6/eq1Byy4nhXTRQUvlMHsI4Q/eMEs88Q==",
 532 |       "requires": {
 533 |         "cardinal": "1.0.0",
 534 |         "denque": "1.2.3",
 535 |         "generate-function": "^2.0.0",
 536 |         "iconv-lite": "^0.4.18",
 537 |         "long": "^4.0.0",
 538 |         "lru-cache": "4.1.1",
 539 |         "named-placeholders": "1.1.1",
 540 |         "object-assign": "^4.1.1",
 541 |         "readable-stream": "2.3.5",
 542 |         "safe-buffer": "^5.0.1",
 543 |         "seq-queue": "0.0.5",
 544 |         "sqlstring": "2.3.1"
 545 |       },
 546 |       "dependencies": {
 547 |         "lru-cache": {
 548 |           "version": "4.1.1",
 549 |           "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.1.tgz",
 550 |           "integrity": "sha512-q4spe4KTfsAS1SUHLO0wz8Qiyf1+vMIAgpRYioFYDMNqKfHQbg+AVDH3i4fvpl71/P1L0dBl+fQi+P37UYf0ew==",
 551 |           "requires": {
 552 |             "pseudomap": "^1.0.2",
 553 |             "yallist": "^2.1.2"
 554 |           }
 555 |         },
 556 |         "readable-stream": {
 557 |           "version": "2.3.5",
 558 |           "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.5.tgz",
 559 |           "integrity": "sha512-tK0yDhrkygt/knjowCUiWP9YdV7c5R+8cR0r/kt9ZhBU906Fs6RpQJCEilamRJj1Nx2rWI6LkW9gKqjTkshhEw==",
 560 |           "requires": {
 561 |             "core-util-is": "~1.0.0",
 562 |             "inherits": "~2.0.3",
 563 |             "isarray": "~1.0.0",
 564 |             "process-nextick-args": "~2.0.0",
 565 |             "safe-buffer": "~5.1.1",
 566 |             "string_decoder": "~1.0.3",
 567 |             "util-deprecate": "~1.0.1"
 568 |           }
 569 |         },
 570 |         "string_decoder": {
 571 |           "version": "1.0.3",
 572 |           "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz",
 573 |           "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==",
 574 |           "requires": {
 575 |             "safe-buffer": "~5.1.0"
 576 |           }
 577 |         }
 578 |       }
 579 |     },
 580 |     "named-placeholders": {
 581 |       "version": "1.1.1",
 582 |       "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.1.tgz",
 583 |       "integrity": "sha1-O3oNJiA910s6nfTJz7gnsvuQfmQ=",
 584 |       "requires": {
 585 |         "lru-cache": "2.5.0"
 586 |       },
 587 |       "dependencies": {
 588 |         "lru-cache": {
 589 |           "version": "2.5.0",
 590 |           "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.5.0.tgz",
 591 |           "integrity": "sha1-2COIrpyWC+y+oMc7uet5tsbOmus="
 592 |         }
 593 |       }
 594 |     },
 595 |     "negotiator": {
 596 |       "version": "0.6.1",
 597 |       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz",
 598 |       "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk="
 599 |     },
 600 |     "nodemailer": {
 601 |       "version": "6.4.16",
 602 |       "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.4.16.tgz",
 603 |       "integrity": "sha512-68K0LgZ6hmZ7PVmwL78gzNdjpj5viqBdFqKrTtr9bZbJYj6BRj5W6WGkxXrEnUl3Co3CBXi3CZBUlpV/foGnOQ=="
 604 |     },
 605 |     "object-assign": {
 606 |       "version": "4.1.1",
 607 |       "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
 608 |       "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
 609 |     },
 610 |     "on-finished": {
 611 |       "version": "2.3.0",
 612 |       "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
 613 |       "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=",
 614 |       "requires": {
 615 |         "ee-first": "1.1.1"
 616 |       }
 617 |     },
 618 |     "once": {
 619 |       "version": "1.4.0",
 620 |       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
 621 |       "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
 622 |       "requires": {
 623 |         "wrappy": "1"
 624 |       }
 625 |     },
 626 |     "parseurl": {
 627 |       "version": "1.3.2",
 628 |       "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz",
 629 |       "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M="
 630 |     },
 631 |     "path-is-absolute": {
 632 |       "version": "1.0.1",
 633 |       "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
 634 |       "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
 635 |     },
 636 |     "path-to-regexp": {
 637 |       "version": "0.1.7",
 638 |       "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
 639 |       "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
 640 |     },
 641 |     "pend": {
 642 |       "version": "1.2.0",
 643 |       "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
 644 |       "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA="
 645 |     },
 646 |     "process-nextick-args": {
 647 |       "version": "2.0.0",
 648 |       "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
 649 |       "integrity": "sha1-o31zL0JxtKsa0HDTVQjoKQeI/6o="
 650 |     },
 651 |     "progress": {
 652 |       "version": "2.0.3",
 653 |       "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
 654 |       "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA=="
 655 |     },
 656 |     "proxy-addr": {
 657 |       "version": "2.0.3",
 658 |       "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.3.tgz",
 659 |       "integrity": "sha512-jQTChiCJteusULxjBp8+jftSQE5Obdl3k4cnmLA6WXtK6XFuWRnvVL7aCiBqaLPM8c4ph0S4tKna8XvmIwEnXQ==",
 660 |       "requires": {
 661 |         "forwarded": "~0.1.2",
 662 |         "ipaddr.js": "1.6.0"
 663 |       }
 664 |     },
 665 |     "proxy-from-env": {
 666 |       "version": "1.0.0",
 667 |       "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz",
 668 |       "integrity": "sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4="
 669 |     },
 670 |     "pseudomap": {
 671 |       "version": "1.0.2",
 672 |       "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
 673 |       "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM="
 674 |     },
 675 |     "puppeteer": {
 676 |       "version": "1.18.1",
 677 |       "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.18.1.tgz",
 678 |       "integrity": "sha512-luUy0HPSuWPsPZ1wAp6NinE0zgetWtudf5zwZ6dHjMWfYpTQcmKveFRox7VBNhQ98OjNA9PQ9PzQyX8k/KrxTg==",
 679 |       "requires": {
 680 |         "debug": "^4.1.0",
 681 |         "extract-zip": "^1.6.6",
 682 |         "https-proxy-agent": "^2.2.1",
 683 |         "mime": "^2.0.3",
 684 |         "progress": "^2.0.1",
 685 |         "proxy-from-env": "^1.0.0",
 686 |         "rimraf": "^2.6.1",
 687 |         "ws": "^6.1.0"
 688 |       },
 689 |       "dependencies": {
 690 |         "debug": {
 691 |           "version": "4.1.1",
 692 |           "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
 693 |           "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
 694 |           "requires": {
 695 |             "ms": "^2.1.1"
 696 |           }
 697 |         },
 698 |         "ms": {
 699 |           "version": "2.1.2",
 700 |           "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
 701 |           "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
 702 |         }
 703 |       }
 704 |     },
 705 |     "qs": {
 706 |       "version": "6.5.1",
 707 |       "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz",
 708 |       "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A=="
 709 |     },
 710 |     "range-parser": {
 711 |       "version": "1.2.0",
 712 |       "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz",
 713 |       "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4="
 714 |     },
 715 |     "raw-body": {
 716 |       "version": "2.3.2",
 717 |       "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.2.tgz",
 718 |       "integrity": "sha1-vNYMd9Prk83gBQKVw/N5OJvIj4k=",
 719 |       "requires": {
 720 |         "bytes": "3.0.0",
 721 |         "http-errors": "1.6.2",
 722 |         "iconv-lite": "0.4.19",
 723 |         "unpipe": "1.0.0"
 724 |       },
 725 |       "dependencies": {
 726 |         "depd": {
 727 |           "version": "1.1.1",
 728 |           "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz",
 729 |           "integrity": "sha1-V4O04cRZ8G+lyif5kfPQbnoxA1k="
 730 |         },
 731 |         "http-errors": {
 732 |           "version": "1.6.2",
 733 |           "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz",
 734 |           "integrity": "sha1-CgAsyFcHGSp+eUbO7cERVfYOxzY=",
 735 |           "requires": {
 736 |             "depd": "1.1.1",
 737 |             "inherits": "2.0.3",
 738 |             "setprototypeof": "1.0.3",
 739 |             "statuses": ">= 1.3.1 < 2"
 740 |           }
 741 |         },
 742 |         "setprototypeof": {
 743 |           "version": "1.0.3",
 744 |           "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz",
 745 |           "integrity": "sha1-ZlZ+NwQ+608E2RvWWMDL77VbjgQ="
 746 |         }
 747 |       }
 748 |     },
 749 |     "readable-stream": {
 750 |       "version": "2.3.6",
 751 |       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
 752 |       "integrity": "sha1-sRwn2IuP8fvgcGQ8+UsMea4bCq8=",
 753 |       "requires": {
 754 |         "core-util-is": "~1.0.0",
 755 |         "inherits": "~2.0.3",
 756 |         "isarray": "~1.0.0",
 757 |         "process-nextick-args": "~2.0.0",
 758 |         "safe-buffer": "~5.1.1",
 759 |         "string_decoder": "~1.1.1",
 760 |         "util-deprecate": "~1.0.1"
 761 |       }
 762 |     },
 763 |     "redeyed": {
 764 |       "version": "1.0.1",
 765 |       "resolved": "https://registry.npmjs.org/redeyed/-/redeyed-1.0.1.tgz",
 766 |       "integrity": "sha1-6WwZO0DAgWsArshCaY5hGF5VSYo=",
 767 |       "requires": {
 768 |         "esprima": "~3.0.0"
 769 |       }
 770 |     },
 771 |     "require_optional": {
 772 |       "version": "1.0.1",
 773 |       "resolved": "https://registry.npmjs.org/require_optional/-/require_optional-1.0.1.tgz",
 774 |       "integrity": "sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g==",
 775 |       "requires": {
 776 |         "resolve-from": "^2.0.0",
 777 |         "semver": "^5.1.0"
 778 |       }
 779 |     },
 780 |     "resolve-from": {
 781 |       "version": "2.0.0",
 782 |       "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-2.0.0.tgz",
 783 |       "integrity": "sha1-lICrIOlP+h2egKgEx+oUdhGWa1c="
 784 |     },
 785 |     "retry-as-promised": {
 786 |       "version": "3.2.0",
 787 |       "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-3.2.0.tgz",
 788 |       "integrity": "sha512-CybGs60B7oYU/qSQ6kuaFmRd9sTZ6oXSc0toqePvV74Ac6/IFZSI1ReFQmtCN+uvW1Mtqdwpvt/LGOiCBAY2Mg==",
 789 |       "requires": {
 790 |         "any-promise": "^1.3.0"
 791 |       }
 792 |     },
 793 |     "rimraf": {
 794 |       "version": "2.6.3",
 795 |       "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz",
 796 |       "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==",
 797 |       "requires": {
 798 |         "glob": "^7.1.3"
 799 |       }
 800 |     },
 801 |     "safe-buffer": {
 802 |       "version": "5.1.1",
 803 |       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
 804 |       "integrity": "sha1-iTMSr2myEj3vcfV4iQAWce6yyFM="
 805 |     },
 806 |     "saslprep": {
 807 |       "version": "1.0.3",
 808 |       "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz",
 809 |       "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==",
 810 |       "optional": true,
 811 |       "requires": {
 812 |         "sparse-bitfield": "^3.0.3"
 813 |       }
 814 |     },
 815 |     "semver": {
 816 |       "version": "5.7.1",
 817 |       "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
 818 |       "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ=="
 819 |     },
 820 |     "send": {
 821 |       "version": "0.16.2",
 822 |       "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz",
 823 |       "integrity": "sha512-E64YFPUssFHEFBvpbbjr44NCLtI1AohxQ8ZSiJjQLskAdKuriYEP6VyGEsRDH8ScozGpkaX1BGvhanqCwkcEZw==",
 824 |       "requires": {
 825 |         "debug": "2.6.9",
 826 |         "depd": "~1.1.2",
 827 |         "destroy": "~1.0.4",
 828 |         "encodeurl": "~1.0.2",
 829 |         "escape-html": "~1.0.3",
 830 |         "etag": "~1.8.1",
 831 |         "fresh": "0.5.2",
 832 |         "http-errors": "~1.6.2",
 833 |         "mime": "1.4.1",
 834 |         "ms": "2.0.0",
 835 |         "on-finished": "~2.3.0",
 836 |         "range-parser": "~1.2.0",
 837 |         "statuses": "~1.4.0"
 838 |       },
 839 |       "dependencies": {
 840 |         "mime": {
 841 |           "version": "1.4.1",
 842 |           "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz",
 843 |           "integrity": "sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ=="
 844 |         }
 845 |       }
 846 |     },
 847 |     "seq-queue": {
 848 |       "version": "0.0.5",
 849 |       "resolved": "https://registry.npmjs.org/seq-queue/-/seq-queue-0.0.5.tgz",
 850 |       "integrity": "sha1-1WgS4cAXpuTnw+Ojeh2m143TyT4="
 851 |     },
 852 |     "sequelize": {
 853 |       "version": "6.0.0",
 854 |       "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.0.0.tgz",
 855 |       "integrity": "sha512-MNgSS7aSy49KLtTMduUzFuwY2SQDtgSY1t7l+5+se4HTGxh+/RwaJHPI3CsGZMgEc8wQZ+r4xBCUkekoSgqJmg==",
 856 |       "requires": {
 857 |         "bluebird": "^3.5.0",
 858 |         "debug": "^4.1.1",
 859 |         "dottie": "^2.0.0",
 860 |         "inflection": "1.12.0",
 861 |         "lodash": "^4.17.11",
 862 |         "moment": "^2.24.0",
 863 |         "moment-timezone": "^0.5.21",
 864 |         "retry-as-promised": "^3.1.0",
 865 |         "semver": "^5.6.0",
 866 |         "sequelize-pool": "^2.1.0",
 867 |         "toposort-class": "^1.0.1",
 868 |         "uuid": "^3.2.1",
 869 |         "validator": "^10.11.0",
 870 |         "wkx": "^0.4.6"
 871 |       },
 872 |       "dependencies": {
 873 |         "debug": {
 874 |           "version": "4.1.1",
 875 |           "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
 876 |           "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
 877 |           "requires": {
 878 |             "ms": "^2.1.1"
 879 |           }
 880 |         },
 881 |         "lodash": {
 882 |           "version": "4.17.11",
 883 |           "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
 884 |           "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
 885 |         },
 886 |         "ms": {
 887 |           "version": "2.1.2",
 888 |           "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
 889 |           "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
 890 |         },
 891 |         "semver": {
 892 |           "version": "5.7.0",
 893 |           "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.0.tgz",
 894 |           "integrity": "sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA=="
 895 |         }
 896 |       }
 897 |     },
 898 |     "sequelize-pool": {
 899 |       "version": "2.3.0",
 900 |       "resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-2.3.0.tgz",
 901 |       "integrity": "sha512-Ibz08vnXvkZ8LJTiUOxRcj1Ckdn7qafNZ2t59jYHMX1VIebTAOYefWdRYFt6z6+hy52WGthAHAoLc9hvk3onqA=="
 902 |     },
 903 |     "serve-static": {
 904 |       "version": "1.13.2",
 905 |       "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.2.tgz",
 906 |       "integrity": "sha512-p/tdJrO4U387R9oMjb1oj7qSMaMfmOyd4j9hOFoxZe2baQszgHcSWjuya/CiT5kgZZKRudHNOA0pYXOl8rQ5nw==",
 907 |       "requires": {
 908 |         "encodeurl": "~1.0.2",
 909 |         "escape-html": "~1.0.3",
 910 |         "parseurl": "~1.3.2",
 911 |         "send": "0.16.2"
 912 |       }
 913 |     },
 914 |     "setprototypeof": {
 915 |       "version": "1.1.0",
 916 |       "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz",
 917 |       "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ=="
 918 |     },
 919 |     "sparse-bitfield": {
 920 |       "version": "3.0.3",
 921 |       "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",
 922 |       "integrity": "sha1-/0rm5oZWBWuks+eSqzM004JzyhE=",
 923 |       "optional": true,
 924 |       "requires": {
 925 |         "memory-pager": "^1.0.2"
 926 |       }
 927 |     },
 928 |     "sqlstring": {
 929 |       "version": "2.3.1",
 930 |       "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz",
 931 |       "integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A="
 932 |     },
 933 |     "statuses": {
 934 |       "version": "1.4.0",
 935 |       "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz",
 936 |       "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew=="
 937 |     },
 938 |     "string_decoder": {
 939 |       "version": "1.1.1",
 940 |       "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
 941 |       "integrity": "sha1-nPFhG6YmhdcDCunkujQUnDrwP8g=",
 942 |       "requires": {
 943 |         "safe-buffer": "~5.1.0"
 944 |       }
 945 |     },
 946 |     "toposort-class": {
 947 |       "version": "1.0.1",
 948 |       "resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz",
 949 |       "integrity": "sha1-f/0feMi+KMO6Rc1OGj9e4ZO9mYg="
 950 |     },
 951 |     "type-is": {
 952 |       "version": "1.6.16",
 953 |       "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz",
 954 |       "integrity": "sha512-HRkVv/5qY2G6I8iab9cI7v1bOIdhm94dVjQCPFElW9W+3GeDOSHmy2EBYe4VTApuzolPcmgFTN3ftVJRKR2J9Q==",
 955 |       "requires": {
 956 |         "media-typer": "0.3.0",
 957 |         "mime-types": "~2.1.18"
 958 |       }
 959 |     },
 960 |     "typedarray": {
 961 |       "version": "0.0.6",
 962 |       "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
 963 |       "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
 964 |     },
 965 |     "unpipe": {
 966 |       "version": "1.0.0",
 967 |       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
 968 |       "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
 969 |     },
 970 |     "util-deprecate": {
 971 |       "version": "1.0.2",
 972 |       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
 973 |       "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
 974 |     },
 975 |     "utils-merge": {
 976 |       "version": "1.0.1",
 977 |       "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
 978 |       "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM="
 979 |     },
 980 |     "uuid": {
 981 |       "version": "3.3.2",
 982 |       "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
 983 |       "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA=="
 984 |     },
 985 |     "validator": {
 986 |       "version": "10.11.0",
 987 |       "resolved": "https://registry.npmjs.org/validator/-/validator-10.11.0.tgz",
 988 |       "integrity": "sha512-X/p3UZerAIsbBfN/IwahhYaBbY68EN/UQBWHtsbXGT5bfrH/p4NQzUCG1kF/rtKaNpnJ7jAu6NGTdSNtyNIXMw=="
 989 |     },
 990 |     "vary": {
 991 |       "version": "1.1.2",
 992 |       "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
 993 |       "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw="
 994 |     },
 995 |     "wkx": {
 996 |       "version": "0.4.7",
 997 |       "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.7.tgz",
 998 |       "integrity": "sha512-pHf546L96TK8RradLt1cWaIffstgv/zXZ14CGz5KnBs1AxBX0wm+IDphjJw0qrEqRv8P9W9CdTt8Z1unMRZ19A==",
 999 |       "requires": {
1000 |         "@types/node": "*"
1001 |       }
1002 |     },
1003 |     "wrappy": {
1004 |       "version": "1.0.2",
1005 |       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
1006 |       "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
1007 |     },
1008 |     "ws": {
1009 |       "version": "6.2.2",
1010 |       "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.2.tgz",
1011 |       "integrity": "sha512-zmhltoSR8u1cnDsD43TX59mzoMZsLKqUweyYBAIvTngR3shc0W6aOZylZmq/7hqyVxPdi+5Ud2QInblgyE72fw==",
1012 |       "requires": {
1013 |         "async-limiter": "~1.0.0"
1014 |       }
1015 |     },
1016 |     "yallist": {
1017 |       "version": "2.1.2",
1018 |       "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
1019 |       "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI="
1020 |     },
1021 |     "yauzl": {
1022 |       "version": "2.4.1",
1023 |       "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz",
1024 |       "integrity": "sha1-lSj0QtqxsihOWLQ3m7GU4i4MQAU=",
1025 |       "requires": {
1026 |         "fd-slicer": "~1.0.1"
1027 |       }
1028 |     }
1029 |   }
1030 | }
1031 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "node-twitter-scraper",
 3 |   "version": "1.0.0",
 4 |   "description": "Scraping twitter with chrome headless and puppeteer",
 5 |   "main": "server.js",
 6 |   "scripts": {
 7 |     "test": "echo \"Error: no test specified\" && exit 1",
 8 |     "start": "node server.js"
 9 |   },
10 |   "repository": {
11 |     "type": "git",
12 |     "url": "git+https://github.com/dansalerno712/Node-Twitter-Scraper.git"
13 |   },
14 |   "author": "Dan Salerno",
15 |   "license": "ISC",
16 |   "bugs": {
17 |     "url": "https://github.com/dansalerno712/Node-Twitter-Scraper/issues"
18 |   },
19 |   "homepage": "https://github.com/dansalerno712/Node-Twitter-Scraper#readme",
20 |   "dependencies": {
21 |     "chunk-date-range": "^0.1.0",
22 |     "csv-writer": "^1.0.0",
23 |     "dateformat": "^3.0.3",
24 |     "express": "^4.16.3",
25 |     "mongodb": "^3.1.13",
26 |     "mysql2": "^1.5.3",
27 |     "nodemailer": "^6.4.16",
28 |     "puppeteer": "^1.18.1",
29 |     "sequelize": "^6.0.0"
30 |   }
31 | }
32 | 


--------------------------------------------------------------------------------
/scraper.js:
--------------------------------------------------------------------------------
  1 | const puppeteer = require('puppeteer');
  2 | const helpers = require('./helpers');
  3 | const toCSV = helpers.toCSV;
  4 | const splitDateRange = helpers.splitDateRange;
  5 | const autoScroll = helpers.autoScroll;
  6 | 
  7 | /*
  8 |  * Function that scrapes all the tweets from a single twitter advanced search and returns them
  9 |  * @input query: The search query
 10 |  * @input startDate: Starting date in the format "YYYY-MM-DD"
 11 |  * @input endDate: Ending date in the format "YYYY-MM-DD"
 12 |  *
 13 |  * @return: An array of Tweet objects that contain tweet text, id, timestamp, date, likes, retweets
 14 |  */
 15 | async function run(query, startDate, endDate, chunks) {
 16 |     // hold results to output to csv
 17 |     let ret = [];
 18 | 
 19 |     // make sure we encode the query correctly for URLs
 20 |     let encodedQuery = encodeURI(query);
 21 | 
 22 |     //chunk the dates
 23 |     let dateChunks = splitDateRange(startDate, endDate, chunks);
 24 | 
 25 |     //hold the urls to parse
 26 |     let urls = [];
 27 |     for (var i = 0; i < dateChunks.length; i += 1) {
 28 |         //put the search parameters into the search url
 29 |         urls.push(`https://twitter.com/search?l=&q=${encodedQuery}%20since%3A${dateChunks[i].start}%20until%3A${dateChunks[i].end}&src=typd&lang=en`);
 30 |     }
 31 | 
 32 |     //make and launch a new page
 33 |     const browser = await puppeteer.launch({
 34 |         headless: true
 35 |     });
 36 | 
 37 |     for (i = 0; i < urls.length; i += 1) {
 38 |         let page = await browser.newPage();
 39 | 
 40 |         console.log("Starting scraping on " + urls[i]);
 41 |         //goto the twitter search page
 42 |         await page.goto(urls[i]);
 43 | 
 44 |         //set viewport for the autoscroll function
 45 |         await page.setViewport({
 46 |             width: 1200,
 47 |             height: 800
 48 |         });
 49 | 
 50 |         //scroll until twitter is done lazy loading
 51 |         await autoScroll(page);
 52 | 
 53 |         //scrape the tweets
 54 |         const tweets = await page.evaluate(function() {
 55 |             //constant selector for the actual tweets on the screen
 56 |             const TWEET_SELECTOR = '.js-stream-tweet';
 57 | 
 58 |             //grab the DOM elements for the tweets
 59 |             let elements = Array.from(document.querySelectorAll(TWEET_SELECTOR));
 60 | 
 61 |             //create an array to return
 62 |             let ret = [];
 63 | 
 64 |             //get the info from within the tweet DOM elements
 65 |             for (var i = 0; i < elements.length; i += 1) {
 66 |                 //object to store data
 67 |                 let tweet = {};
 68 | 
 69 |                 //get text of tweet
 70 |                 const TWEET_TEXT_SELECTOR = ".tweet-text";
 71 |                 tweet.text = elements[i].querySelector(TWEET_TEXT_SELECTOR).textContent;
 72 | 
 73 |                 //get timestamp
 74 |                 const TWEET_TIMESTAMP_SELECTOR = '.tweet-timestamp';
 75 |                 tweet.timestamp = elements[i].querySelector(TWEET_TIMESTAMP_SELECTOR).getAttribute('title');
 76 | 
 77 |                 //get tweet id
 78 |                 const TWEET_ID_SELECTOR = 'data-tweet-id';
 79 |                 tweet.id = elements[i].getAttribute(TWEET_ID_SELECTOR);
 80 | 
 81 |                 //get likes/retweets
 82 |                 const ACTIONS_SELECTOR = ".ProfileTweet-actionCountForPresentation";
 83 |                 let actions = elements[i].querySelectorAll(ACTIONS_SELECTOR);
 84 | 
 85 |                 //loop through the DOM elements for the actions
 86 |                 for (var j = 0; j < actions.length; j += 1) {
 87 |                     //for some reason, retweets are the 2nd action and likes are the 4th
 88 |                     tweet.retweets = actions[1].innerHTML ? actions[1].innerHTML : 0;
 89 |                     tweet.likes = actions[3].innerHTML ? actions[3].innerHTML : 0;
 90 |                 }
 91 | 
 92 |                 //add tweet data to return array
 93 |                 ret.push(tweet);
 94 |             }
 95 |             return ret;
 96 |         });
 97 | 
 98 |         //add to csv
 99 |         ret.push(tweets);
100 | 
101 |         //close the page
102 |         await page.close();
103 |     }
104 | 
105 |     //exit the browser
106 |     await browser.close();
107 | 
108 |     // collapse into one array and return
109 |     return [].concat.apply([], ret);
110 | }
111 | 
112 | module.exports.run = run;


--------------------------------------------------------------------------------
/server.js:
--------------------------------------------------------------------------------
 1 | const express = require('express');
 2 | const app = express();
 3 | const bodyParser = require("body-parser");
 4 | const scraper = require("./scraper");
 5 | const toCSV = require("./helpers").toCSV;
 6 | const MongoClient = require("mongodb").MongoClient;
 7 | const assert = require("assert");
 8 | const sendEmail = require("./helpers").sendEmail;
 9 | 
10 | app.use(bodyParser.urlencoded({
11 |     extended: true
12 | }));
13 | app.use(bodyParser.json());
14 | 
15 | var mongoURL = 'mongodb://localhost:27017';
16 | 
17 | app.get('/', (req, res) => res.send("Henlo World"));
18 | 
19 | app.post('/scrape', async (req, res) => {
20 |     let term = req.body.term;
21 |     let startDate = req.body.startDate;
22 |     let endDate = req.body.endDate;
23 |     let chunk = req.body.chunk;
24 |     let email = req.body.email;
25 | 
26 |     res.status(200);
27 |     res.send("Starting Scraping");
28 | 
29 |     let ret = await scraper.run(term, startDate, endDate, chunk);
30 | 
31 |     MongoClient.connect(mongoURL, (err, client) => {
32 |         assert.equal(null, err);
33 | 
34 |         var db = client.db('tweetFiles')
35 | 
36 |         file = {
37 |             term: term,
38 |             startDate: startDate,
39 |             endDate: endDate,
40 |         };
41 |         db.collection('files').insertOne(file, (err, response) => {
42 |             if (err) {
43 |                 throw err;
44 |             } else {
45 |                 path = "./files/" + response.ops[0]._id + ".csv";
46 |                 toCSV(ret, path);
47 |                 let link = "http://localhost:3000/download?id=" + response.ops[0]._id
48 |                 sendEmail(email, link);
49 |             }
50 |         });
51 | 
52 |         client.close();
53 |     });
54 | });
55 | 
56 | app.get("/download", (req, res) => {
57 |     let id = req.query.id
58 |     let path = __dirname + "/files/" + id + ".csv";
59 |     res.download(path);
60 | });
61 | 
62 | app.listen(3000, () => console.log("Listening on 3000"));


--------------------------------------------------------------------------------