├── .editorconfig ├── .gitignore ├── README.md ├── UNUSED ├── CouchDB.js ├── crawler.js ├── demo-downloaded.js ├── hackernews.js └── rig-timing.js ├── app ├── args │ ├── getAction.js │ ├── getDomain.js │ ├── getMongoConnectionString.js │ ├── getMysqlSettings.js │ ├── getThreads.js │ └── index.js ├── crawler │ ├── README.md │ └── index.js ├── hud │ ├── README.md │ ├── error.js │ ├── index.js │ ├── initOnce.js │ ├── intro.js │ ├── keyboard.js │ ├── message.js │ ├── numLines.js │ ├── progress.js │ ├── title.js │ ├── urlState.js │ └── writeLine.js ├── mongo │ ├── README.md │ ├── getDb.js │ ├── index.js │ ├── init.js │ ├── setup │ │ └── index.js │ └── webgraph │ │ ├── getProgress.js │ │ ├── getUncrawledUrl.js │ │ ├── updateUrl.js │ │ └── upsertNewUrls.js └── mysql │ ├── README.md │ ├── index.js │ ├── init.js │ ├── query.js │ ├── setup │ ├── scraper.mwb │ └── scraper.sql │ └── webgraph │ ├── getProgress.js │ ├── getUncrawledUrl.js │ ├── updateUrl.js │ └── upsertNewUrls.js ├── domains ├── checkatrade.com │ └── index.js ├── oodavid.com │ └── index.js └── template │ └── template.js ├── index.js ├── package-lock.json └── package.json /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | indent_style = space 8 | indent_size = 2 9 | end_of_line = lf 10 | insert_final_newline = true 11 | trim_trailing_whitespace = true 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Private Credentials 2 | creds.json 3 | 4 | # Errors 5 | error.log 6 | 7 | # Backup files 8 | *.bak 9 | 10 | # Images 11 | images/* 12 | 13 | 14 | 15 | # Created by https://www.gitignore.io/api/osx,node,linux,windows,visualstudiocode 16 | 17 | ### Linux ### 18 | *~ 19 | 20 | # temporary files which can be created if a process still has a handle open of a deleted file 21 | .fuse_hidden* 22 | 23 | # KDE directory preferences 24 | .directory 25 | 26 | # Linux trash folder which might appear on any partition or disk 27 | .Trash-* 28 | 29 | # .nfs files are created when an open file is removed but is still being accessed 30 | .nfs* 31 | 32 | ### Node ### 33 | # Logs 34 | logs 35 | *.log 36 | npm-debug.log* 37 | yarn-debug.log* 38 | yarn-error.log* 39 | 40 | # Runtime data 41 | pids 42 | *.pid 43 | *.seed 44 | *.pid.lock 45 | 46 | # Directory for instrumented libs generated by jscoverage/JSCover 47 | lib-cov 48 | 49 | # Coverage directory used by tools like istanbul 50 | coverage 51 | 52 | # nyc test coverage 53 | .nyc_output 54 | 55 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 56 | .grunt 57 | 58 | # Bower dependency directory (https://bower.io/) 59 | bower_components 60 | 61 | # node-waf configuration 62 | .lock-wscript 63 | 64 | # Compiled binary addons (http://nodejs.org/api/addons.html) 65 | build/Release 66 | 67 | # Dependency directories 68 | node_modules/ 69 | jspm_packages/ 70 | 71 | # Typescript v1 declaration files 72 | typings/ 73 | 74 | # Optional npm cache directory 75 | .npm 76 | 77 | # Optional eslint cache 78 | .eslintcache 79 | 80 | # Optional REPL history 81 | .node_repl_history 82 | 83 | # Output of 'npm pack' 84 | *.tgz 85 | 86 | # Yarn Integrity file 87 | .yarn-integrity 88 | 89 | # dotenv environment variables file 90 | .env 91 | 92 | 93 | ### OSX ### 94 | *.DS_Store 95 | .AppleDouble 96 | .LSOverride 97 | 98 | # Icon must end with two \r 99 | Icon 100 | 101 | # Thumbnails 102 | ._* 103 | 104 | # Files that might appear in the root of a volume 105 | .DocumentRevisions-V100 106 | .fseventsd 107 | .Spotlight-V100 108 | .TemporaryItems 109 | .Trashes 110 | .VolumeIcon.icns 111 | .com.apple.timemachine.donotpresent 112 | 113 | # Directories potentially created on remote AFP share 114 | .AppleDB 115 | .AppleDesktop 116 | Network Trash Folder 117 | Temporary Items 118 | .apdisk 119 | 120 | 121 | ### Windows ### 122 | # Windows thumbnail cache files 123 | Thumbs.db 124 | ehthumbs.db 125 | ehthumbs_vista.db 126 | 127 | # Folder config file 128 | Desktop.ini 129 | 130 | # Recycle Bin used on file shares 131 | $RECYCLE.BIN/ 132 | 133 | # Windows Installer files 134 | *.cab 135 | *.msi 136 | *.msm 137 | *.msp 138 | 139 | # Windows shortcuts 140 | *.lnk 141 | 142 | 143 | ### VisualStudioCode ### 144 | .vscode/* 145 | # !.vscode/settings.json 146 | # !.vscode/tasks.json 147 | # !.vscode/launch.json 148 | # !.vscode/extensions.json 149 | # .history 150 | 151 | 152 | # End of https://www.gitignore.io/api/osx,node,linux,windows,visualstudiocode 153 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pup 2 | 3 |

4 | Demonstrating the Puppeteer Scraper 5 |

6 | 7 | Rough Todo 8 | 9 | When doing "from memory" it really should be "existing promise" for multithreading 10 | 11 | Always use `module.exports`, not `exports` 12 | 13 | 14 | # Northstar - Scrape for Paul! 15 | 16 | What remains (15th Nov) 17 | 18 | * [x] `updateUrl.js` - 1hr 19 | * [x] `browserPool.js` review - 2hr 20 | * ...conclusion, I don't need my own browserPool, each puppeteer instance has `pages` (ala tabs) 21 | * [x] MD5 Logic 22 | * [x] DB - Rename `host` to `domain` 23 | * [x] Progress Bar 24 | * [x] Update periodically 25 | * [x] Add time estimate 26 | * [x] Ignore #hashes in URLs 27 | * [x] Switch back to MongoDB, MySQL is irking me 28 | * [x] Args + Connect 29 | * [x] Setup Script 30 | * [x] Reimplement webgraph logic 31 | * [x] Thread Management 32 | * [x] Obey `--threads` argument 33 | * [x] Pause / Unpause 34 | * [x] Increase / Decrease Threads 35 | * [x] `hud.progress()` 36 | * [x] Progress - Show start time 37 | * [x] Pretty this up a little 38 | * [x] `hud.message()` 39 | * [x] Write to file `messages.log` 40 | * [x] `hud.writeLine()` 41 | * [x] Coerce messages to a single line 42 | * [x] Truncate 43 | * [x] Per-domain parsing 44 | * [x] URL weighting 45 | * [x] During Write 46 | * [x] During Read 47 | * [ ] Add proxy support 48 | * https://proxymesh.com/ 49 | * [ ] Saved to `creds.json` via args 50 | * [ ] User Agent Randmiser 51 | * [ ] Apply adblock to strip ads, analytics and social tools 52 | * https://github.com/bbondy/abp-filter-parser 53 | * https://easylist.to/ 54 | * [ ] Screenshots 55 | * [ ] Sits behind a flag (domain settings) 56 | * [ ] When disabled, option to block image requests (speed) 57 | * [ ] Upload Screenshots to remote 58 | * https://www.npmjs.com/package/aws-sdk 59 | * [ ] Save to DB 60 | * [ ] Throw an error if we get a duplicate hash 61 | * [ ] Draft a MapReduce function to show how to combine data objects from multiple sources 62 | * See: https://docs.mongodb.com/manual/aggregation/ 63 | * Just for demonstration (`README.md`), this doesn't really need coding 64 | * Example: `db.urls.find({ 'data.id': 'JamesPond' }, { url: 1, domain: 1, data: 1 }).pretty()` 65 | * [ ] Update the README 66 | * [ ] Punchy title and intro 67 | * [ ] Merge all these todo lists! 68 | * [ ] Up-to-date GIF 69 | * [ ] Set the github topics 70 | * [ ] Link to how-to tutorial 71 | * [ ] Youtube Video? 72 | 73 | # Parsing tasks 74 | 75 | * [ ] Parse companies house / duedil for business data 76 | * [ ] Get a list of competitors, with locations 77 | * [ ] Compute complementary businesses (by location) 78 | * [ ] Build a drip campaign 79 | 80 | -- 81 | 82 | * [ ] Upon boot 83 | * [ ] Extra prompt - reset database? 84 | * [ ] Extra prompt - reset wip? 85 | * [ ] Review `domain settings` files 86 | 87 | -- 88 | 89 | * [x] `app/mysql/index.js` 90 | * [x] Pool, Connect, Query 91 | * [x] Return some mysql interface (3rd party) 92 | * [ ] BONUS - Dump useful error if we can't connect 93 | * [x] `app/crawler/index.js` 94 | * [x] `init(browser)` 95 | * [ ] Send notification (SMS) when complete 96 | * [ ] Write about URL structure, use this as a reference: 97 | * [ ] https://developer.mozilla.org/en-US/docs/Web/API/Location 98 | * [x] Create a HUD 99 | * [x] Use https://github.com/cronvel/terminal-kit 100 | * [x] initialize - reserve space for the hud 101 | * [x] title - set title in terminal and HUD 102 | * [x] keyboard - handle `n` keyboard callbacks, render them on HUD 103 | * [x] progress - show the overall progress 104 | * [x] urlState - show the running threads (might rename) 105 | * [x] message - render short message on HUD 106 | * [x] error - for handling errors - write to file, render short message on HUD 107 | * [x] Hash MD5 logic 108 | * [ ] BONUS - Consider adding this to the domain settings (it may want to exclude non-content html) 109 | * [ ] Bonus - Create an executable - https://github.com/zeit/pkg 110 | * [ ] Consider using `scrapy style pipelines` - https://doc.scrapy.org/en/latest/topics/item-pipeline.html 111 | 112 | 113 | 114 | # Reading List 115 | 116 | Read these for reference and notes 117 | 118 | 119 | * https://www.quora.com/What-are-examples-of-how-real-businesses-use-web-scraping-Are-there-any-types-of-businesses-which-use-this-more-than-others 120 | * Turbocharge the data with: 121 | * https://www.integrationsjs.com/ 122 | * https://open.blockspring.com/browse 123 | * https://www.blockspring.com/ 124 | * https://hunter.io/ 125 | * Test sites: 126 | * http://webscraper.io/test-sites 127 | * http://airbnb.com 128 | * https://www.google.com/intl/bn/insidesearch/howsearchworks/crawling-indexing.html 129 | * https://hexfox.com/p/scrapy-vs-beautifulsoup/ 130 | * https://hexfox.com/p/how-to-filter-out-duplicate-urls-from-scrapys-start-urls/ 131 | * https://stackoverflow.com/a/9736414/1122851 - database pools (and pools in general) 132 | 133 | # A Better Scraper, with Puppeteer 134 | 135 | Firstly, go forth and read [Getting started with Puppeteer and Chrome Headless for Web Scraping](https://medium.com/@e_mad_ehsan/getting-started-with-puppeteer-and-chrome-headless-for-web-scrapping-6bf5979dee3e). 136 | 137 | Now we will scale this idea up by: 138 | 139 | * Adding multi-threading, proxies and throttling 140 | * Allowing workers to be distributed 141 | * Catering for multiple domains 142 | * Allowing our data to be analysed after-the-fact 143 | 144 | # Requirements 145 | 146 | * [ ] Data is decentralised 147 | * [ ] Use FireBase, it's vogue and I'm keen 148 | * [ ] Workers are distributed 149 | * [ ] Prevent working at cross purposes 150 | * [ ] Track name and IP 151 | * [ ] Optional authentication 152 | * [ ] Users can select which domain to scrape 153 | * [x] Users can control how many threads they run 154 | * [x] Keyboard control 155 | * [ ] Can use proxies 156 | * [ ] Has IP throttling 157 | 158 | # Components 159 | 160 | Having a clear separation of concerns will keep the system simple and workable. 161 | 162 | The scraper should be broken down into the following parts. 163 | 164 | ## 1 - Create The Webgraph 165 | 166 | This can be thought of as a virtual surfer. She will surf the web, clicking all links, submitting all forms and documenting her findings. 167 | 168 | * Deals with authentication 169 | * Navigates between pages 170 | * Evaluates javascript 171 | * Builds lists of links 172 | * Follows links / submits forms 173 | * Takes screenshots (they're useful) 174 | 175 | The effect of this would be to create a Webgraph, stored in a database, of this structure: 176 | 177 | * The nodes 178 | * Represent URLs 179 | * Contains a key denoting the "state" of the URL (none > wip > scraped > analysed) 180 | * Contains the URL and HTML (after javascript has run) 181 | * Contains the date the URL was scraped 182 | * Unsolved 183 | * How to represent pages which have the same URL but different "context"? (session, or POST data) 184 | * The edges 185 | * Represents links between nodes 186 | 187 | You could use this data to create a "PageRank", if you so wished. 188 | 189 | ## 2 - Page Analysis 190 | 191 | Loads an URL that is `scraped` (ie: `unanlysed`) 192 | Load the analysis rules for the URL (see section #3 below) 193 | Parse the HTML into useful data (users, organizations, whatever) 194 | 195 | ## 3 - Management Interface 196 | 197 | At first this will not be necessary, config. can be filebased and analysis direct on the database. 198 | 199 | * Visualisation 200 | * Active Scrapers 201 | * Results Overview 202 | * Results Drilldown 203 | * Management 204 | * Add domains 205 | * Add domain rules 206 | * DOM selectors 207 | * URL weights (0-1) 208 | * A weight of `0` would mean **don't visit this URL** 209 | * A weight of `1` would mean **visit this URL ASAP** 210 | * Trigger Re-Analysis 211 | 212 | # Tools 213 | 214 | ## Headless Browser 215 | 216 | **Puppeteer** is now the obvious choice. 217 | 218 | ## Database 219 | 220 | **CouchDB** gives versioning out of the box 221 | **CouchDB** + PouchDB means our visualisations can be realtime 222 | 223 | **Firebase Firestore** does the same, with zero "setup" 224 | 225 | #### References 226 | 227 | * https://github.com/GoogleChrome/puppeteer 228 | * https://github.com/emadehsan/thal 229 | * https://github.com/Quramy/angular-puppeteer-demo 230 | -------------------------------------------------------------------------------- /UNUSED/CouchDB.js: -------------------------------------------------------------------------------- 1 | const Promise = require('bluebird'); 2 | const CREDS = require('../creds.js').CouchDb; 3 | const nano = require('nano')(`${CREDS.protocol}://${CREDS.username}:${CREDS.userpass}@${CREDS.host}:${CREDS.port}`); 4 | 5 | 6 | var webgraphDb = nano.db.use('webgraph'); 7 | Promise.promisifyAll(webgraphDb); 8 | 9 | 10 | module.exports.getUnscrapedUrls = getUnscrapedUrls; 11 | module.exports.setScrapeData = setScrapeData; 12 | 13 | 14 | // Promise to return `num` URLs 15 | // Flag said URLs as `queued` 16 | async function getUnscrapedUrls(num){ 17 | const docs = await webgraphDb.viewAsync('pages', 'unscraped', { limit: num }); 18 | // Update the docs 19 | let updatedDocs = docs.rows.map((doc) => { 20 | doc.key.status = 'queued'; 21 | return doc.key; 22 | }); 23 | const bulkResults = await webgraphDb.bulkAsync({ docs: updatedDocs }); 24 | // Review the update 25 | for(var b=bulkResults.length-1; b>=0; b--){ 26 | const bulkResult = bulkResults[b]; 27 | if(bulkResult.error){ 28 | updatedDocs.splice(b, 1); // Remove any docs that couldn't be flagged as `queued` (likely due to conflicts) 29 | } else { 30 | updatedDocs[b]._rev = bulkResult.rev; // Update the _rev of those that could 31 | } 32 | } 33 | // Make sure the url is a property 34 | updatedDocs.forEach(function(doc){ 35 | doc.url = doc._id; 36 | }); 37 | // Return 38 | return updatedDocs; 39 | } 40 | 41 | 42 | // Promise to update the URL status, html and screenshot 43 | // add links add new urls to database 44 | async function setScrapeData(doc, html, status_code, links){ 45 | // Update the doc 46 | delete doc.url; 47 | doc.status = 'scraped'; 48 | doc.html = html; 49 | doc.status_code = status_code; 50 | doc.links = {}; 51 | links.forEach(function(o){ 52 | doc.links[o] = true; 53 | }); 54 | // Update 55 | await webgraphDb.insertAsync(doc); 56 | // Create the link URLs 57 | let docs = links.map(function(link){ 58 | return { _id: link }; 59 | }); 60 | await webgraphDb.bulkAsync({ docs: docs }); 61 | // Make sure the url is a property 62 | doc.url = doc._id; 63 | // Return 64 | return doc; 65 | } 66 | 67 | 68 | // Test out the commands 69 | // 70 | (async () => { 71 | // Read a batch of URLs 72 | console.log(`Getting Unscraped URLs`); 73 | const docs = await getUnscrapedUrls(1); 74 | console.log(docs); 75 | 76 | // Scrape an URL 77 | const doc = docs[0]; 78 | console.log(`Scraping ${doc.url} (demo)`); 79 | const html = '

Hello World

'; 80 | const status_code = 200; 81 | const links = [ 82 | 'https://oodavid.com/testing/', 83 | 'https://oodavid.com/testing-123/', 84 | ]; 85 | const updatedDoc = await setScrapeData(doc, html, status_code, links); 86 | console.log(updatedDoc); 87 | })(); 88 | -------------------------------------------------------------------------------- /UNUSED/crawler.js: -------------------------------------------------------------------------------- 1 | const browserPool = require('./browserPool/'); 2 | const sanitize = require("sanitize-filename"); 3 | require('./keyboard.js'); 4 | 5 | let unscraped = [ 6 | 'https://oodavid.com', 7 | 'https://oodavid.com/todo/', 8 | 'https://oodavid.com/about-me/', 9 | 'https://oodavid.com/article/angularjs-meta-tags-management/', 10 | 'https://oodavid.com/article/jekyll-is-dead-long-live-hugo/', 11 | 'https://oodavid.com/article/angularjs-meta-tags-management/', 12 | 'https://oodavid.com/article/style-rendering-test/', 13 | ]; 14 | 15 | browserPool.setPoolSize(2); 16 | spawnBrowsers(); 17 | 18 | function spawnBrowsers(){ 19 | let numSpawned = 0; 20 | for(var b=0; b<=browserPool.getIdleCount(); b++){ 21 | const url = getUnscrapedUrl(); 22 | if(url){ 23 | spawnBrowser(url); 24 | numSpawned++; 25 | } 26 | } 27 | if(!numSpawned){ 28 | console.log('nothing to do (start exponential backoff)'); 29 | } 30 | }; 31 | 32 | async function spawnBrowser(url){ 33 | console.log('parsing', url); 34 | let browser; 35 | try { 36 | browser = await browserPool.getBrowser(); 37 | const page = await browser.newPage(); 38 | await page.goto(url); 39 | // Analysis 40 | const filename = sanitize(url); 41 | await page.screenshot({ path: `${__dirname}/../../images/${filename}.png` }); 42 | const html = await page.content(); 43 | } catch(e) { 44 | console.error(e); 45 | } 46 | // Always release the browser 47 | await browserPool.releaseBrowser(browser); 48 | spawnBrowsers(); 49 | } 50 | 51 | function getUnscrapedUrl(){ 52 | return unscraped.shift(); 53 | } 54 | -------------------------------------------------------------------------------- /UNUSED/demo-downloaded.js: -------------------------------------------------------------------------------- 1 | const puppeteer = require('puppeteer'); 2 | const CREDS = require('./creds'); 3 | const mongoose = require('mongoose'); 4 | const User = require('./models/user'); 5 | 6 | async function run() { 7 | const browser = await puppeteer.launch(); 8 | 9 | const page = await browser.newPage(); 10 | 11 | // await page.goto('https://github.com'); 12 | // await page.screenshot path: 'screenshots/github.png' }); 13 | 14 | await page.goto('https://github.com/login'); 15 | 16 | // dom element selectors 17 | const USERNAME_SELECTOR = '#login_field'; 18 | const PASSWORD_SELECTOR = '#password'; 19 | const BUTTON_SELECTOR = '#login > form > div.auth-form-body.mt-3 > input.btn.btn-primary.btn-block'; 20 | 21 | await page.click(USERNAME_SELECTOR); 22 | await page.type(CREDS.username); 23 | 24 | await page.click(PASSWORD_SELECTOR); 25 | await page.type(CREDS.password); 26 | 27 | await page.click(BUTTON_SELECTOR); 28 | await page.waitForNavigation(); 29 | 30 | const userToSearch = 'oodavid'; 31 | const searchUrl = `https://github.com/search?q=${userToSearch}&type=Users&utf8=%E2%9C%93`; 32 | // let searchUrl = 'https://github.com/search?utf8=%E2%9C%93&q=bashua&type=Users'; 33 | 34 | await page.goto(searchUrl); 35 | await page.waitFor(2 * 1000); 36 | 37 | // const LIST_USERNAME_SELECTOR = '#user_search_results > div.user-list > div:nth-child(1) > div.d-flex > div > a'; 38 | const LIST_USERNAME_SELECTOR = '#user_search_results > div.user-list > div:nth-child(INDEX) > div.d-flex > div > a'; 39 | // const LIST_EMAIL_SELECTOR = '#user_search_results > div.user-list > div:nth-child(1) > div.d-flex > div > ul > li:nth-child(2) > a'; 40 | const LIST_EMAIL_SELECTOR = '#user_search_results > div.user-list > div:nth-child(INDEX) > div.d-flex > div > ul > li:nth-child(2) > a'; 41 | const LENGTH_SELECTOR_CLASS = 'user-list-item'; 42 | // const numPages = await getNumPages(page); 43 | const numPages = 2; 44 | 45 | console.log('Numpages: ', numPages); 46 | 47 | for (let h = 1; h <= numPages; h++) { 48 | let pageUrl = searchUrl + '&p=' + h; 49 | await page.goto(pageUrl); 50 | 51 | let listLength = await page.evaluate((sel) => { 52 | return document.getElementsByClassName(sel).length; 53 | }, LENGTH_SELECTOR_CLASS); 54 | 55 | for (let i = 1; i <= listLength; i++) { 56 | // change the index to the next child 57 | let usernameSelector = LIST_USERNAME_SELECTOR.replace("INDEX", i); 58 | let emailSelector = LIST_EMAIL_SELECTOR.replace("INDEX", i); 59 | 60 | let username = await page.evaluate((sel) => { 61 | return document.querySelector(sel).getAttribute('href').replace('/', ''); 62 | }, usernameSelector); 63 | 64 | let email = await page.evaluate((sel) => { 65 | let element = document.querySelector(sel); 66 | return element ? element.innerHTML : null; 67 | }, emailSelector); 68 | 69 | // not all users have emails visible 70 | if (!email) 71 | continue; 72 | 73 | console.log(username, ' -> ', email); 74 | 75 | upsertUser({ 76 | username: username, 77 | email: email, 78 | dateCrawled: new Date() 79 | }); 80 | } 81 | } 82 | 83 | browser.close(); 84 | } 85 | 86 | async function getNumPages(page) { 87 | const NUM_USER_SELECTOR = '#js-pjax-container > div.container > div > div.column.three-fourths.codesearch-results.pr-6 > div.d-flex.flex-justify-between.border-bottom.pb-3 > h3'; 88 | 89 | let inner = await page.evaluate((sel) => { 90 | let html = document.querySelector(sel).innerHTML; 91 | 92 | // format is: "69,803 users" 93 | return html.replace(',', '').replace('users', '').trim(); 94 | }, NUM_USER_SELECTOR); 95 | 96 | const numUsers = parseInt(inner); 97 | 98 | console.log('numUsers: ', numUsers); 99 | 100 | /** 101 | * GitHub shows 10 resuls per page, so 102 | */ 103 | return Math.ceil(numUsers / 10); 104 | } 105 | 106 | function upsertUser(userObj) { 107 | const DB_URL = 'mongodb://localhost/thal'; 108 | console.log('Upserting user', userObj); 109 | 110 | if (mongoose.connection.readyState == 0) { 111 | mongoose.connect(DB_URL); 112 | } 113 | 114 | // if this email exists, update the entry, don't insert 115 | const conditions = { email: userObj.email }; 116 | const options = { upsert: true, new: true, setDefaultsOnInsert: true }; 117 | 118 | User.findOneAndUpdate(conditions, userObj, options, (err, result) => { 119 | if (err) { 120 | throw err; 121 | } 122 | }); 123 | } 124 | 125 | run(); -------------------------------------------------------------------------------- /UNUSED/hackernews.js: -------------------------------------------------------------------------------- 1 | const puppeteer = require('puppeteer'); 2 | const cheerio = require('cheerio'); 3 | 4 | async function run() { 5 | const browser = await puppeteer.launch(); 6 | const page = await browser.newPage(); 7 | await page.goto('https://news.ycombinator.com'); 8 | 9 | let content = await page.content(); 10 | var $ = cheerio.load(content); 11 | $('span.comhead').each(function(i, element){ 12 | var a = $(this).prev(); 13 | var rank = a.parent().parent().text(); 14 | var title = a.text(); 15 | var url = a.attr('href'); 16 | var subtext = a.parent().parent().next().children('.subtext').children(); 17 | var points = $(subtext).eq(0).text(); 18 | var username = $(subtext).eq(1).text(); 19 | var comments = $(subtext).eq(2).text(); 20 | 21 | var metadata = { 22 | rank: parseInt(rank), 23 | title: title, 24 | url: url, 25 | points: parseInt(points), 26 | username: username, 27 | comments: parseInt(comments) 28 | }; 29 | console.log(metadata); 30 | }); 31 | 32 | browser.close(); 33 | } 34 | 35 | run(); -------------------------------------------------------------------------------- /UNUSED/rig-timing.js: -------------------------------------------------------------------------------- 1 | const puppeteer = require('puppeteer'); 2 | 3 | (async () => { 4 | 5 | console.time('puppeteer.launch'); 6 | const browser = await puppeteer.launch(); 7 | console.timeEnd('puppeteer.launch'); 8 | 9 | console.time('browser.newPage'); 10 | const page = await browser.newPage(); 11 | console.timeEnd('browser.newPage'); 12 | 13 | console.time('page.goto'); 14 | await page.goto('https://oodavid.com/article/angularjs-memento-factory/'); 15 | console.timeEnd('page.goto'); 16 | 17 | console.time('page.screenshot'); 18 | await page.screenshot({ path: 'screenshots/oodavid.com.png' }); 19 | console.timeEnd('page.screenshot'); 20 | 21 | console.time('page.content'); 22 | const html = await page.content(); 23 | console.timeEnd('page.content'); 24 | 25 | await browser.close(); 26 | 27 | })(); 28 | -------------------------------------------------------------------------------- /app/args/getAction.js: -------------------------------------------------------------------------------- 1 | module.exports = getAction; 2 | 3 | 4 | const argv = require('yargs').argv; 5 | const inquirer = require('inquirer'); 6 | 7 | 8 | const actions = { 9 | 'Crawl and Parse': 'crawl', 10 | 'Just Parse': 'parse', 11 | }; 12 | let action; 13 | 14 | 15 | async function getAction(){ 16 | // From memory 17 | if(action){ 18 | return action; 19 | } 20 | // From the command-line 21 | if(argv.action){ 22 | action = argv.action; 23 | return action; 24 | } 25 | // Prompt 26 | return inquirer.prompt([ 27 | { 28 | type: 'list', 29 | name: 'action', 30 | message: 'What would you like to do?', 31 | choices: Object.keys(actions), 32 | } 33 | ]) 34 | .then(function(answers){ 35 | action = actions[answers.action]; 36 | return action; 37 | }) 38 | .catch(function(err){ 39 | console.error(err); 40 | }); 41 | } 42 | -------------------------------------------------------------------------------- /app/args/getDomain.js: -------------------------------------------------------------------------------- 1 | module.exports = getDomain; 2 | 3 | 4 | const argv = require('yargs').argv; 5 | const inquirer = require('inquirer'); 6 | const autocomplete = require('inquirer-autocomplete-prompt'); 7 | const fuzzy = require('fuzzy'); 8 | const glob = require('glob'); 9 | inquirer.registerPrompt('autocomplete', autocomplete); 10 | 11 | 12 | let domain; 13 | 14 | 15 | async function getDomain(){ 16 | // From memory 17 | if(domain){ 18 | return domain; 19 | } 20 | // From the command-line 21 | if(argv.domain){ 22 | domain = argv.domain; 23 | return domain; 24 | } 25 | // Prompt 26 | return inquirer.prompt([ 27 | { 28 | type: 'autocomplete', 29 | name: 'domain', 30 | message: 'Which domain?', 31 | source: autocompleteDomains, 32 | }, 33 | ]) 34 | .then(function(answers){ 35 | domain = answers.domain; 36 | return domain; 37 | }) 38 | .catch(function(err){ 39 | console.error(err); 40 | }); 41 | } 42 | 43 | 44 | var domainsCache = false; 45 | function getDomains(){ 46 | if(!domainsCache){ 47 | domainsCache = glob 48 | .sync('./domains/*/') 49 | .map(function(path){ 50 | return path.split('/').splice(-2, 1).pop(); 51 | }) 52 | .filter(function(path){ 53 | return path !== 'template'; 54 | }); 55 | } 56 | return domainsCache; 57 | } 58 | 59 | 60 | function autocompleteDomains(answersSoFar, input){ 61 | return new Promise(function(resolve, reject){ 62 | resolve( 63 | fuzzy 64 | .filter((input || ''), getDomains()) 65 | .map(el => el.original) 66 | ); 67 | }); 68 | } 69 | -------------------------------------------------------------------------------- /app/args/getMongoConnectionString.js: -------------------------------------------------------------------------------- 1 | module.exports = getMongoConnectionString; 2 | 3 | 4 | const inquirer = require('inquirer'); 5 | const fs = require('fs'); 6 | const util = require('util'); 7 | const writeFile = util.promisify(fs.writeFile); 8 | const term = require('terminal-kit').terminal; 9 | 10 | 11 | const credsFilePath = `${__dirname}/../../creds.json`; 12 | 13 | 14 | let creds; 15 | 16 | 17 | async function getMongoConnectionString(){ 18 | // From memory 19 | if(creds && creds.mongoConnectionString){ 20 | return creds.mongoConnectionString; 21 | } 22 | // From file... 23 | try { 24 | creds = require(credsFilePath); 25 | } catch(e){ 26 | creds = {}; 27 | } 28 | if(creds.mongoConnectionString){ 29 | return creds.mongoConnectionString; 30 | } 31 | // Prompt 32 | var answers = await inquirer.prompt([{ 33 | type: 'input', 34 | name: 'mongoConnectionString', 35 | message: 'Please Enter your MongoDB Connection String', 36 | }]); 37 | // Update 38 | creds.mongoConnectionString = answers.mongoConnectionString; 39 | // Write the file 40 | await writeFile(credsFilePath, JSON.stringify(creds, null, 4)); 41 | term(`^r❯^: ^+MongoDB settings saved to^: ^ccreds.json^:\n`); 42 | // Return 43 | return creds.mongoConnectionString; 44 | } 45 | -------------------------------------------------------------------------------- /app/args/getMysqlSettings.js: -------------------------------------------------------------------------------- 1 | module.exports = getMysqlSettings; 2 | 3 | 4 | const inquirer = require('inquirer'); 5 | const fs = require('fs'); 6 | const util = require('util'); 7 | const writeFile = util.promisify(fs.writeFile); 8 | const term = require('terminal-kit').terminal; 9 | 10 | 11 | const credsFilePath = `${__dirname}/../../creds.json`; 12 | const allPrompts = [ 13 | { 14 | type: 'input', 15 | name: 'host', 16 | message: 'Enter host', 17 | default: 'localhost', 18 | }, 19 | { 20 | type: 'input', 21 | name: 'user', 22 | message: 'Enter user', 23 | default: 'charlotte', 24 | }, 25 | { 26 | type: 'password', 27 | name: 'password', 28 | message: 'Enter password', 29 | default: 'Wilbur', 30 | }, 31 | { 32 | type: 'input', 33 | name: 'database', 34 | message: 'Enter database', 35 | default: 'scraper', 36 | }, 37 | { 38 | type: 'input', 39 | name: 'connectionLimit', 40 | message: 'Connection Limit', 41 | default: 10, 42 | filter: function(value){ 43 | return parseInt(value, 10); 44 | }, 45 | validate: function(value){ 46 | var isValid = (value > 0 && value === parseInt(value, 10)); 47 | return isValid || "Please enter an integer"; 48 | }, 49 | }, 50 | ]; 51 | 52 | 53 | let creds; 54 | 55 | 56 | async function getMysqlSettings(){ 57 | // From memory 58 | if(creds && creds.mysql){ 59 | return creds.mysql; 60 | } 61 | // From file... 62 | try { 63 | creds = require(credsFilePath); 64 | } catch(e){ 65 | creds = {}; 66 | } 67 | creds.mysql = creds.mysql || {}; 68 | // ...does the object contain all the required properties? 69 | var prompts = allPrompts.filter(function(prompt){ 70 | return !creds.mysql.hasOwnProperty(prompt.name); 71 | }); 72 | if(!prompts.length){ 73 | return creds.mysql; 74 | } 75 | // Prompt 76 | term(`^r❯^: ^+Please Configure MySQL^: ^-(you only need to do this once)^:\n`); 77 | var answers = await inquirer.prompt(prompts) 78 | .then(function(answers){ 79 | return answers; 80 | }) 81 | .catch(function(err){ 82 | console.error(err); 83 | }); 84 | Object.assign(creds.mysql, answers); 85 | // Write the file 86 | await writeFile(credsFilePath, JSON.stringify(creds, null, 4)); 87 | term(`^r❯^: ^+MySQL settings saved to^: ^ccreds.json^:\n`); 88 | // Return 89 | return creds.mysql; 90 | } 91 | -------------------------------------------------------------------------------- /app/args/getThreads.js: -------------------------------------------------------------------------------- 1 | module.exports = getThreads; 2 | 3 | 4 | const argv = require('yargs').argv; 5 | const inquirer = require('inquirer'); 6 | 7 | 8 | let threads; 9 | 10 | 11 | async function getThreads(){ 12 | // From memory 13 | if(threads){ 14 | return threads; 15 | } 16 | // From the command-line 17 | if(argv.threads){ 18 | threads = argv.threads; 19 | return threads; 20 | } 21 | // Prompt 22 | return inquirer.prompt([ 23 | { 24 | type: 'input', 25 | name: 'threads', 26 | message: 'How many threads to run?', 27 | default: 10, 28 | filter: function(value){ 29 | return parseInt(value, 10); 30 | }, 31 | validate: function(value){ 32 | var isValid = (value > 0 && value === parseInt(value, 10)); 33 | return isValid || "Please enter an integer"; 34 | } 35 | } 36 | ]) 37 | .then(function(answers){ 38 | threads = answers.threads; 39 | return threads; 40 | }) 41 | .catch(function(err){ 42 | console.error(err); 43 | }); 44 | } 45 | -------------------------------------------------------------------------------- /app/args/index.js: -------------------------------------------------------------------------------- 1 | module.exports.getAction = require('./getAction.js'); 2 | module.exports.getDomain = require('./getDomain.js'); 3 | module.exports.getMysqlSettings = require('./getMysqlSettings.js'); 4 | module.exports.getMongoConnectionString = require('./getMongoConnectionString.js'); 5 | module.exports.getThreads = require('./getThreads.js'); 6 | -------------------------------------------------------------------------------- /app/crawler/README.md: -------------------------------------------------------------------------------- 1 | # Browser Pool 2 | 3 | 4 | 5 | WikiPedia :: Pool (computer science) 6 | 7 | In computer science, a pool is a set of resources that are kept ready to use, 8 | rather than acquired on use and released afterwards. In this context, resources 9 | can refer to system resources such as file handles, which are external to a 10 | process, or internal resources such as objects. A pool client requests a 11 | resource from the pool and performs desired operations on the returned resource. 12 | When the client finishes its use of the resource, it is returned to the pool 13 | rather than released and lost. 14 | 15 | The pooling of resources can offer a significant performance boost in situations 16 | that have high cost associated with resource acquiring, high rate of the 17 | requests for resources, and a low overall count of simultaneously used resources. 18 | Pooling is also useful when the latency is a concern, because a pool offers 19 | predictable times required to obtain resources since they have already been 20 | acquired. These benefits are mostly true for system resources that require a 21 | system call, or remote resources that require a network communication, such as 22 | database connections, socket connections, threads, and memory allocation. 23 | Pooling is also useful for expensive-to-compute data, notably large graphic 24 | objects like fonts or bitmaps, acting essentially as a data cache or a 25 | memoization technique. 26 | 27 | Special cases of pools are connection pools, thread pools, and memory pools. 28 | -------------------------------------------------------------------------------- /app/crawler/index.js: -------------------------------------------------------------------------------- 1 | module.exports.start = start; 2 | 3 | 4 | const hud = require('../hud/'); 5 | const datastore = require('../mongo/'); 6 | const sanitize = require("sanitize-filename"); 7 | const puppeteer = require('puppeteer'); 8 | const md5 = require('md5'); 9 | 10 | 11 | let domainConfig; 12 | let browser; 13 | let runningThreads = 0; 14 | let maxThreads = 10; 15 | 16 | 17 | async function start(myDomainConfig, myNumThreads){ 18 | domainConfig = myDomainConfig; 19 | maxThreads = myNumThreads; 20 | // Title 21 | hud.title(`Crawling ^c${domainConfig.domain}^:`); 22 | // Start keyboard control 23 | hud.keyboard.assign('UP', increaseThreads, 'Increase Threads'); 24 | hud.keyboard.assign('DOWN', decreaseThreads, 'Decrease Threads'); 25 | hud.keyboard.assign('p', pause, 'Pause'); 26 | hud.keyboard.start(); 27 | // Update the progress bar 28 | updateProgress(); 29 | // Upsert the seed URLs 30 | hud.message(`upserting ${domainConfig.seedUrls.length} seed URLS`); 31 | await datastore.upsertNewUrls(domainConfig, domainConfig.seedUrls); 32 | // Initialise our browser and start crawling 33 | browser = await puppeteer.launch(); 34 | crawlUrls(); 35 | } 36 | 37 | 38 | 39 | 40 | 41 | async function updateProgress(){ 42 | try { 43 | let progress = await datastore.getProgress(domainConfig.domain); 44 | hud.progress(progress.crawled, progress.total); 45 | } catch (e) { 46 | hud.error(e); 47 | } finally { 48 | setTimeout(updateProgress, 1000); 49 | } 50 | } 51 | 52 | 53 | 54 | 55 | 56 | async function getPage(){ 57 | if(runningThreads < maxThreads){ 58 | runningThreads ++; 59 | return await browser.newPage(); 60 | } 61 | } 62 | 63 | async function releasePage(page){ 64 | if(page){ 65 | await page.close(); 66 | runningThreads --; 67 | } 68 | } 69 | 70 | async function crawlUrls(){ 71 | const page = await getPage(); 72 | if(page){ 73 | const urlObject = await datastore.getUncrawledUrl(domainConfig.domain); 74 | if(urlObject){ 75 | crawlUrl(page, urlObject); 76 | // Recur 77 | setTimeout(crawlUrls, 100); 78 | } else { 79 | releasePage(page); 80 | } 81 | } 82 | } 83 | 84 | async function crawlUrl(page, urlObject){ 85 | hud.urlState(urlObject.url, 'Parsing'); 86 | try { 87 | /* Blocking images 88 | await page.setRequestInterception(true); 89 | page.on('request', request => { 90 | if (request.resourceType === 'image') 91 | request.abort(); 92 | else 93 | request.continue(); 94 | }); 95 | */ 96 | const response = await page.goto(urlObject.url, { waitUntil: 'networkidle' }); 97 | const html = await page.content(); 98 | // Webgraph Basics 99 | urlObject.status = response.status; 100 | urlObject.hash = md5(html); 101 | urlObject.links = await page.evaluate(function(){ // ...this runs in the context of the browser 102 | let links = [... document.querySelectorAll('a')]; 103 | return links.map(function(link){ 104 | return link.href; 105 | }); 106 | }); 107 | // Screenshot 108 | // const filename = sanitize(urlObject.url); 109 | // await page.screenshot({ path: `${__dirname}/../../images/${filename}.png` }); 110 | // Parsing (domain specific) 111 | if(typeof domainConfig.parse === 'function'){ 112 | await injectJQuery(page); 113 | urlObject.data = await domainConfig.parse(page, urlObject); 114 | } 115 | // Save 116 | await datastore.updateUrl(domainConfig, urlObject); 117 | } catch(e) { 118 | // Log errors 119 | hud.error(e); 120 | } finally { 121 | // Always tidy up 122 | hud.urlState(urlObject.url, false); 123 | await releasePage(page); 124 | // Spawn more crawlers 125 | crawlUrls(); 126 | } 127 | } 128 | 129 | 130 | 131 | 132 | async function injectJQuery(page){ 133 | await page.evaluate(() => { 134 | var jq = document.createElement("script") 135 | jq.setAttribute('type','text/javascript'); 136 | jq.src = "https://code.jquery.com/jquery-3.2.1.min.js" 137 | return new Promise((resolve) => { 138 | jq.addEventListener("load", ()=> { 139 | resolve(); 140 | }); 141 | document.getElementsByTagName("head")[0].appendChild(jq); 142 | }); 143 | }) 144 | const watchDog = page.waitForFunction('window.jQuery !== undefined'); 145 | await watchDog; 146 | } 147 | 148 | 149 | 150 | 151 | 152 | let pausedThreads; 153 | function increaseThreads(){ 154 | maxThreads ++; 155 | crawlUrls(); 156 | hud.message(`Max: ${maxThreads} threads`); 157 | } 158 | function decreaseThreads(){ 159 | maxThreads --; 160 | hud.message(`Max: ${maxThreads} threads`); 161 | } 162 | function pause(){ 163 | pausedThreads = maxThreads; 164 | maxThreads = 0; 165 | hud.keyboard.assign('p', unpause, 'Unpause'); 166 | hud.message('Paused. Wait for threads to complete'); 167 | } 168 | function unpause(){ 169 | maxThreads = pausedThreads; 170 | crawlUrls(); 171 | hud.keyboard.assign('p', pause, 'Pause'); 172 | hud.message(`Unpaused. Max: ${maxThreads} threads`); 173 | } 174 | -------------------------------------------------------------------------------- /app/hud/README.md: -------------------------------------------------------------------------------- 1 | # Heads-up Display (HUD) 2 | 3 | Renders a Heads-up Display with title, keyboard control, progress meter, urlStates, messages and errors. 4 | 5 | Uses [terminal-kit](https://github.com/cronvel/terminal-kit/) for the heavy lifting. 6 | 7 | Renders colors and styles with [terminal-kit Style Markup](https://github.com/cronvel/terminal-kit/blob/master/doc/low-level.md#style-markup). 8 | 9 | ## Example output 10 | 11 | This is without styling, I've added line numbers for reference. 12 | 13 | 
14 |  0  Crawling oodavid.com
15 |  1  [CTRL+C] to exit, [UP] Increase Threads, [DOWN] Increase Threads, [p] Unpause
16 |  2
17 |  3  Progress [==================---------------------------------------------------] 26% [15022 / 61280]   1,068 / 47,326
18 |  4  Started: 21 Nov 20:38 Completed: 296 Rate: 5,530 / hour TTC: 8 hours
19 |  5
20 |  6  Loading     http://domain.com/1
21 |  7  Loading     http://domain.com/2
22 |  8  Loading     http://domain.com/3
23 |  9  Processing  http://domain.com/4
24 | 10  -------     http://----.--
25 | 11  Loading     http://domain.com/6
26 | 12  Processing  http://domain.com/7
27 | 13  Loading     http://domain.com/8
28 | 14  -------     http://----.--
29 | 15  Loading     http://domain.com/10
30 | 16  12 threads (4 threads are hidden)
31 | 17
32 | 18  Paused. Wait for threads to complete
33 | 19  ./error.log - 20 new errors - latest: 2017/11/14 11:40:58
34 |
35 | 36 | ## Usage 37 | 38 | ```js 39 | // line 0 - hud.title 40 | hud.title('Crawling ^goodavid.com^:'); 41 | 42 | // line 1 - hud.keyboard 43 | hud.keyboard.assign('UP', increaseProcesses, 'Increase Processes'); 44 | hud.keyboard.assign('DOWN', decreaseProcesses, 'Decrease Processes'); 45 | hud.keyboard.assign('p', pause, 'Pause'); 46 | hud.keyboard.start(); 47 | // ...can be reassigned 48 | hud.keyboard.assign('p', unpause, 'Unpause'); 49 | 50 | // line 3-4 - hud.progress 51 | hud.progress(15022, 61280); 52 | 53 | // lines 6-16 - hud.urlState 54 | for(var n=1;n<=20;n++){ 55 | hud.urlState(`http://domain.com/${n}`, 'Loading'); 56 | } 57 | hud.urlState(`http://domain.com/4`, 'Processing'); 58 | hud.urlState(`http://domain.com/5`, false); 59 | hud.urlState(`http://domain.com/7`, 'Processing'); 60 | hud.urlState(`http://domain.com/9`, false); 61 | 62 | // line 18 - hud.message 63 | hud.message('Paused - processes will spin down'); 64 | 65 | // line 19 - hud.error 66 | hud.error(new Error('Oh dear!')); 67 | ``` 68 | -------------------------------------------------------------------------------- /app/hud/error.js: -------------------------------------------------------------------------------- 1 | module.exports = error; 2 | 3 | 4 | const writeLine = require('./writeLine.js'); 5 | const fs = require('fs'); 6 | 7 | 8 | const errorLog = fs.createWriteStream(`${__dirname}/../../error.log`, { flags: 'a' }); 9 | let errors = 0; 10 | function error(e){ 11 | // Write to file 12 | const now = new Date(); 13 | errorLog.write(`${now}\n${e.stack}\n\n`); 14 | // Update the HUD 15 | errors ++; 16 | writeLine(19, `^-./error.log - ^:^r${errors} new errors^- - latest: ${now}`); 17 | } 18 | -------------------------------------------------------------------------------- /app/hud/index.js: -------------------------------------------------------------------------------- 1 | module.exports.intro = require('./intro.js'); 2 | module.exports.initOnce = require('./initOnce.js'); 3 | module.exports.title = require('./title.js'); 4 | module.exports.keyboard = require('./keyboard.js'); 5 | module.exports.progress = require('./progress.js'); 6 | module.exports.urlState = require('./urlState.js'); 7 | module.exports.message = require('./message.js'); 8 | module.exports.error = require('./error.js'); 9 | -------------------------------------------------------------------------------- /app/hud/initOnce.js: -------------------------------------------------------------------------------- 1 | module.exports = initOnce; 2 | 3 | 4 | const numLines = require('./numLines.js'); 5 | const term = require('terminal-kit').terminal; 6 | 7 | 8 | let initRun = false; 9 | function initOnce(){ 10 | if(initRun){ return; } 11 | initRun = true; 12 | console.log((new Array(numLines+1)).join('\n')); 13 | term.saveCursor(); 14 | } 15 | -------------------------------------------------------------------------------- /app/hud/intro.js: -------------------------------------------------------------------------------- 1 | module.exports = printIntro; 2 | 3 | 4 | const term = require('terminal-kit').terminal; 5 | 6 | 7 | function printIntro(){ 8 | term(addSplashOfColor(` 9 | ∇ ____ ∇ 10 | | : : | ·---------------------------- - - 11 | {| ♥ ♥ |} | Web Scraper; Crawler and Parser 12 | |___==___| | oodavid 2017 13 | / \\ ·---------------------------- - - 14 | 15 | `)); 16 | } 17 | 18 | 19 | function addSplashOfColor(message){ 20 | return message.replace(/([∇♥])/g, `^r$1^:`); 21 | } 22 | -------------------------------------------------------------------------------- /app/hud/keyboard.js: -------------------------------------------------------------------------------- 1 | module.exports.assign = assign; 2 | module.exports.start = start; 3 | 4 | 5 | const term = require('terminal-kit').terminal; 6 | const writeLine = require('./writeLine.js'); 7 | 8 | 9 | let keys = { 10 | // keyName: { command, description } 11 | }; 12 | let isActive = false; 13 | 14 | 15 | function assign(keyName, command, description){ 16 | keys[keyName] = { command, description }; // ES6 syntactic sugar 17 | renderKeys(); 18 | } 19 | 20 | 21 | function start(){ 22 | // Only start when inactive 23 | if(isActive){ return; } 24 | isActive = true; 25 | // Listen for input 26 | term.grabInput(true); 27 | term.on('key', function(keyName, matches, data){ 28 | // Has that key been assigned? 29 | if(keys.hasOwnProperty(keyName)){ 30 | keys[keyName].command(); 31 | } 32 | // Manually handle the exit command 33 | if(keyName === 'CTRL_C'){ 34 | terminate(); 35 | return; 36 | } 37 | }); 38 | // Render the keys to screen 39 | renderKeys(); 40 | } 41 | 42 | 43 | function terminate(){ 44 | term.grabInput(false); 45 | setTimeout(function(){ 46 | process.exit(); 47 | }, 100); 48 | } 49 | 50 | 51 | function renderKeys(){ 52 | if(!isActive){ return; } 53 | let commands = ['[CTRL+C] to exit']; 54 | Object.entries(keys).forEach(function(value){ 55 | commands.push(`[${value[0]}] ${value[1].description}`); 56 | }); 57 | writeLine(1, `^-${commands.join(', ')}^:`); 58 | } 59 | -------------------------------------------------------------------------------- /app/hud/message.js: -------------------------------------------------------------------------------- 1 | module.exports = message; 2 | 3 | 4 | const writeLine = require('./writeLine.js'); 5 | const fs = require('fs'); 6 | 7 | 8 | const messageLog = fs.createWriteStream(`${__dirname}/../../message.log`, { flags: 'a' }); 9 | let timeoutId; 10 | 11 | 12 | function message(message, duration=5000){ 13 | // Write to file 14 | const now = new Date(); 15 | messageLog.write(`${now} - ${message}\n\n`); 16 | // Update the HUD 17 | writeLine(18, message); 18 | if(timeoutId){ 19 | clearTimeout(timeoutId); 20 | } 21 | timeoutId = setTimeout(function(){ 22 | writeLine(18, ''); 23 | }, duration); 24 | } 25 | -------------------------------------------------------------------------------- /app/hud/numLines.js: -------------------------------------------------------------------------------- 1 | module.exports = 20; 2 | -------------------------------------------------------------------------------- /app/hud/progress.js: -------------------------------------------------------------------------------- 1 | module.exports = progress; 2 | 3 | 4 | const numLines = require('./numLines.js'); 5 | const initOnce = require('./initOnce.js'); 6 | const term = require('terminal-kit').terminal; 7 | const moment = require('moment'); 8 | 9 | 10 | let session; 11 | let maxHistoryMs = 1*60*1000; // 1 minute 12 | 13 | 14 | function progress(completed, total){ 15 | initOnce(); 16 | calculateSessionTotals(completed, total); 17 | showProgressMeter(); 18 | showVelocityAndEta(); 19 | } 20 | 21 | 22 | function calculateSessionTotals(completed, total){ 23 | const now = Date.now(); 24 | let props = { completed, total, timestamp: now }; 25 | if(!session){ 26 | session = { 27 | started: now, 28 | initial: props, 29 | history: [], 30 | }; 31 | } 32 | // Basics 33 | session.completed = props.completed; 34 | session.total = props.total; 35 | session.timestamp = now; 36 | session.duration = now - session.initial.timestamp; 37 | session.addedToTotal = total - session.initial.total; 38 | session.newlyCompleted = completed - session.initial.completed; 39 | session.remaining = total - completed; 40 | session.pctComplete = total ? (completed/total) : 0; 41 | // Manage history 42 | session.history.push(props); 43 | session.history = session.history.filter(function(item){ 44 | return item.timestamp > (now - maxHistoryMs); // Trim older history items 45 | }); 46 | // Velocity (time to complete one item, ms) - compare the oldest and newest history items 47 | const dTimestamp = props.timestamp - session.history[0].timestamp; 48 | const dCompleted = props.completed - session.history[0].completed; 49 | session.velocity = false; 50 | if(dCompleted > 0){ 51 | session.velocity = (dTimestamp/dCompleted); 52 | } 53 | // Eta (time to complete the remaining items, ms) 54 | session.eta = false; 55 | if(session.velocity){ 56 | session.eta = new Date(now+(session.remaining*session.velocity)); 57 | } 58 | } 59 | 60 | 61 | // Progress [---------------------------------------------------------------------] 1% [362 / 27,534] 62 | function showProgressMeter(){ 63 | // The meter 64 | var len = 70; 65 | var left = Array(Math.round(len*session.pctComplete)).join('='); 66 | var right = Array(1+len-Math.round(len*session.pctComplete)).join('-'); 67 | var completed = session.completed.toLocaleString(); 68 | var total = session.total.toLocaleString(); 69 | // The string 70 | var pctStr = Math.floor(session.pctComplete * 100); 71 | var str = `Progress [^g${left}^:^-${right}^:] ${pctStr}% [^g${completed}^: / ${total}]`; 72 | // Output 73 | term.restoreCursor(); 74 | term.column(0).move(0, 3-numLines).eraseLine(); 75 | term(str); 76 | term.restoreCursor(); 77 | } 78 | 79 | 80 | // Completed: 1,234, Velocity: 100 / hour, ETA: 2h 40 81 | function showVelocityAndEta(){ 82 | const started = moment(session.started).format('D MMM HH:mm'); 83 | const completed = session.newlyCompleted.toLocaleString(); 84 | var ttc = '---'; 85 | if(session.eta){ 86 | ttc = moment().to(moment(session.eta), true); 87 | } 88 | var rate = '---'; 89 | if(session.velocity){ 90 | rate = Math.round((60*60*1000)/session.velocity).toLocaleString() + ' / hour'; 91 | } 92 | var str = `^-Started: ^:${started} ^-Completed: ^:${completed} ^-Rate: ^:${rate} ^-TTC: ^:${ttc}`; 93 | // Output 94 | term.restoreCursor(); 95 | term.column(0).move(0, 4-numLines).eraseLine(); 96 | term(str); 97 | term.restoreCursor(); 98 | } 99 | -------------------------------------------------------------------------------- /app/hud/title.js: -------------------------------------------------------------------------------- 1 | module.exports = title; 2 | 3 | 4 | const writeLine = require('./writeLine.js'); 5 | const term = require('terminal-kit').terminal; 6 | 7 | 8 | function title(title){ 9 | writeLine(0, title); 10 | title = title.replace(/\^./g, ''); // Strip "Style Markup" ie: ^bBLUE^: becomes BLUE 11 | term.windowTitle(title); 12 | } 13 | -------------------------------------------------------------------------------- /app/hud/urlState.js: -------------------------------------------------------------------------------- 1 | module.exports = urlState; 2 | 3 | 4 | const writeLine = require('./writeLine.js'); 5 | const term = require('terminal-kit').terminal; 6 | 7 | 8 | let firstLine = 6; 9 | let numLines = 10; 10 | 11 | 12 | let allUrls = {}; // url: state 13 | let visibleUrls = new Array(numLines).fill(null); // url 14 | 15 | 16 | function urlState(url, state){ 17 | var index = getVisibleIndex(url); 18 | if(state){ 19 | allUrls[url] = state; 20 | if(index !== -1){ 21 | visibleUrls[index] = url; 22 | let paddedState = (state+' ').substr(0, 11); // Fixed length 23 | writeLine(index+firstLine, `^-${paddedState}^: ${url}`); 24 | } 25 | } else { 26 | delete allUrls[url]; 27 | if(index !== -1){ 28 | visibleUrls[index] = null; 29 | writeLine(index+firstLine, '^-------- http://----.--^:'); 30 | } 31 | } 32 | renderTotals(); 33 | } 34 | 35 | 36 | // Returns the line that this url is already allocated to, or the first empty line (or -1) 37 | function getVisibleIndex(url){ 38 | const index = visibleUrls.indexOf(url); 39 | if(index !== -1){ 40 | return index; 41 | } 42 | return visibleUrls.indexOf(null); 43 | } 44 | 45 | 46 | // Render totals 47 | function renderTotals(){ 48 | // Total threads 49 | const totalThreads = Object.keys(allUrls).length; 50 | let message = `^-${totalThreads} threads`; 51 | // Hidden threads 52 | const hiddenThreads = totalThreads - visibleUrls.filter(Boolean).length; 53 | if(hiddenThreads > 0){ 54 | message += ` (${hiddenThreads} threads are hidden)`; 55 | } 56 | writeLine(numLines+firstLine, message); 57 | } 58 | -------------------------------------------------------------------------------- /app/hud/writeLine.js: -------------------------------------------------------------------------------- 1 | module.exports = writeLine; 2 | 3 | 4 | const numLines = require('./numLines.js'); 5 | const initOnce = require('./initOnce.js'); 6 | const term = require('terminal-kit').terminal; 7 | 8 | 9 | const maxLength = 100; 10 | 11 | 12 | function writeLine(n, message){ 13 | // Sanitize the message 14 | message = message.replace(/[\r\n]/g, ''); // Strip newlines 15 | message = message.substring(0, maxLength); // Clip the length 16 | // Write 17 | initOnce(); 18 | term.restoreCursor(); 19 | term.column(0).move(0, n-numLines).eraseLine(); 20 | term(message); 21 | term.restoreCursor(); 22 | } 23 | -------------------------------------------------------------------------------- /app/mongo/README.md: -------------------------------------------------------------------------------- 1 | # MongoDB 2 | 3 | Don't run MongoDB locally, a decentralised database is more useful; a managed service is often easier. 4 | 5 | There are services available with a free tier: 6 | 7 | * [MongoDB Atlas](https://www.mongodb.com/cloud/atlas) - Free tier 8 | * [MLab](https://mlab.com/) - Sandbox 9 | 10 | Each come with a their own limitations, usually around the amount of data that can be stored, number of connections and replication. 11 | 12 | Regardless, they're enough for testing and development. 13 | 14 | # Setup 15 | 16 | Once you've created a remote database (above), run this command to set up your collection indexes. 17 | 18 | ```shell 19 | node app/mongo/setup/index.js 20 | ``` 21 | 22 | You can update this script to add more collections and indexes, as you see fit. 23 | 24 | The script will prompt you for your [MongoDB Connection String](https://docs.mongodb.com/manual/reference/connection-string/), this will be saved to `creds.json`, and is in the format: 25 | 26 | `mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]]` 27 | 28 | ## Webgraph Data Structure 29 | 30 | We have one collection to represent the webgraph; `urls`. 31 | 32 | Each document will have the basic structure below, with indexes on: 33 | 34 | * url (unique) 35 | * domain, status 36 | * hash 37 | 38 | ```json 39 | { 40 | _id: 'MongoObjectId', 41 | url: 'http://example.com/scraped-url', 42 | domain: 'example.com', 43 | status: 200, 44 | hash: '865A922C002FCA69CF14B0818FCBEB05', 45 | html: '....', 46 | links: [ 47 | 'http://example.com/about-us', 48 | 'http://example.com/contact', 49 | 'http://blog.example.com/', 50 | 'http://otherdomain.com/home' 51 | ] 52 | } 53 | ``` 54 | 55 | # Handy Queries 56 | 57 | ```js 58 | // Total number of URLs in the system (for any domain) 59 | db.urls.find().count(); 60 | 61 | // Get the highest priority url to parse 62 | db.urls.find({ status: { $exists: false }, domain: 'www.checkatrade.com' }).sort({ weight: -1 }).limit(1).pretty(); 63 | 64 | // Get the number of processed URLs 65 | db.urls.find({ status: { $exists: true, $ne: 'wip' }, domain: 'www.checkatrade.com' }).count(); 66 | 67 | // Get all URLs with data matching JamesPond 68 | db.urls.find({ 'data.id': 'JamesPond' }, { url: 1, domain: 1, data: 1 }).pretty() 69 | 70 | // Count the URLs of each state 71 | // https://docs.mongodb.com/manual/reference/operator/aggregation/sum/#grp._S_sum 72 | db.urls.aggregate([{"$group": {"_id": "$status", "count": {"$sum":1} }}]) 73 | ``` 74 | 75 | ### References 76 | 77 | * https://docs.mongodb.com/v2.8/tutorial/create-an-index/ 78 | * http://thecodebarbarian.com/common-async-await-design-patterns-in-node.js.html 79 | -------------------------------------------------------------------------------- /app/mongo/getDb.js: -------------------------------------------------------------------------------- 1 | module.exports = getDb; 2 | 3 | const args = require('../args/index.js'); 4 | const MongoClient = require('mongodb').MongoClient; 5 | 6 | 7 | let db; 8 | 9 | 10 | async function getDb(){ 11 | // Initial connect 12 | if(!db){ 13 | const connectionString = await args.getMongoConnectionString(); 14 | db = await MongoClient.connect(connectionString); 15 | } 16 | // Return from memory 17 | return db; 18 | } 19 | -------------------------------------------------------------------------------- /app/mongo/index.js: -------------------------------------------------------------------------------- 1 | exports.getDb = require('./getDb.js'); 2 | exports.init = require('./init.js'); 3 | 4 | 5 | exports.upsertNewUrls = require('./webgraph/upsertNewUrls.js'); 6 | exports.getUncrawledUrl = require('./webgraph/getUncrawledUrl.js'); 7 | exports.updateUrl = require('./webgraph/updateUrl.js'); 8 | exports.getProgress = require('./webgraph/getProgress.js'); 9 | -------------------------------------------------------------------------------- /app/mongo/init.js: -------------------------------------------------------------------------------- 1 | module.exports = init; 2 | 3 | 4 | const getDb = require('./getDb.js'); 5 | const term = require('terminal-kit').terminal; 6 | 7 | 8 | async function init(){ 9 | try { 10 | const db = await getDb(); 11 | term(`^r❯^: Connected to MongoDB^:\n`); 12 | } catch (e){ 13 | console.log(e); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /app/mongo/setup/index.js: -------------------------------------------------------------------------------- 1 | const getDb = require('../getDb.js'); 2 | 3 | 4 | async function setup(){ 5 | try { 6 | const db = await getDb(); 7 | // All URLS must be unique 8 | await db.collection('urls').createIndex({ url: 1 }, { unique: true }); 9 | // Optimizes the query "find urls from domain with status, in weight order" 10 | await db.collection('urls').createIndex({ domain: 1, status: 1, weight: -1 }); 11 | // Optimizes the query "does this url give a duplicate hash?" 12 | await db.collection('urls').createIndex({ hash: 1 }); 13 | // Allows us to Map Reduce our scraped data 14 | await db.collection('urls').createIndex({ 'data.id': 1 }); 15 | // All done 16 | db.close(); 17 | console.log('Mongo Setup Complete (you only need to run this once)'); 18 | } catch (e){ 19 | console.log(e); 20 | } 21 | } 22 | 23 | 24 | setup(); 25 | -------------------------------------------------------------------------------- /app/mongo/webgraph/getProgress.js: -------------------------------------------------------------------------------- 1 | module.exports = getProgress; 2 | 3 | 4 | const getDb = require('../getDb.js'); 5 | const hud = require('../../hud/'); 6 | 7 | 8 | async function getProgress(domain){ 9 | try { 10 | const db = await getDb(); 11 | // Response 12 | let response = { crawled: 0, total: 0 }; 13 | // Total 14 | const totalCursor = await db.collection('urls').aggregate([ 15 | { $match: { domain: domain } }, 16 | { $count: "total" } 17 | ]).toArray(); 18 | response.total = totalCursor.length ? totalCursor[0].total : 0; 19 | // Crawled 20 | const crawledCursor = await db.collection('urls').aggregate([ 21 | { $match: { status: { $exists: true, $ne: 'wip' }, domain: domain } }, 22 | { $count: "crawled" } 23 | ]).toArray(); 24 | response.crawled = crawledCursor.length ? crawledCursor[0].crawled : 0; 25 | // Return 26 | return response; 27 | } catch (e){ 28 | hud.error(e); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /app/mongo/webgraph/getUncrawledUrl.js: -------------------------------------------------------------------------------- 1 | module.exports = getUncrawledUrl; 2 | 3 | 4 | const getDb = require('../getDb.js'); 5 | const hud = require('../../hud/'); 6 | 7 | 8 | async function getUncrawledUrl(domain){ 9 | try { 10 | const db = await getDb(); 11 | // Get an URL with no status, update it, and return 12 | const filter = { status: { $exists: false }, domain: domain }; 13 | const update = { $set: { status: 'wip' } }; 14 | const options = { returnNewDocument: true, sort: { weight: -1 } }; 15 | const response = await db.collection('urls').findOneAndUpdate(filter, update, options); 16 | return response.value; // This may be null! 17 | } catch (e){ 18 | hud.error(e); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /app/mongo/webgraph/updateUrl.js: -------------------------------------------------------------------------------- 1 | module.exports = updateUrl; 2 | 3 | 4 | const getDb = require('../getDb.js'); 5 | const upsertNewUrls = require('./upsertNewUrls.js'); 6 | const hud = require('../../hud/'); 7 | 8 | 9 | // Where urlObject is an object of format { _id, url, domain, status, html, hash, links, data } 10 | async function updateUrl(domainConfig, urlObject){ 11 | try { 12 | const db = await getDb(); 13 | // Update the object 14 | urlObject.updated = new Date(); 15 | const filter = { _id: urlObject._id }; 16 | const update = { $set: urlObject }; 17 | await db.collection('urls').updateOne(filter, update); 18 | // Upsert the links 19 | await upsertNewUrls(domainConfig, urlObject.links); 20 | } catch (e){ 21 | hud.error(e); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /app/mongo/webgraph/upsertNewUrls.js: -------------------------------------------------------------------------------- 1 | module.exports = upsertNewUrls; 2 | 3 | 4 | const parseUrl = require('url').parse; 5 | const getDb = require('../getDb.js'); 6 | const hud = require('../../hud/'); 7 | 8 | 9 | async function upsertNewUrls(domainConfig, urls){ 10 | try { 11 | const db = await getDb(); 12 | // Insert the URLs 13 | const options = { 14 | ordered: false, // This means that, when a single write fails, the operation continues with the remaining writes 15 | }; 16 | let docs = parseUrlsIntoMongoDocs(domainConfig, urls); 17 | if(docs.length){ 18 | await db.collection('urls').insertMany(docs, options); 19 | } 20 | } catch (e){ 21 | // Ignore errors relating to duplicates 22 | if(!(e.name === 'MongoError' && e.code === 11000)){ 23 | hud.error(e); 24 | } 25 | } 26 | } 27 | 28 | 29 | function parseUrlsIntoMongoDocs(domainConfig, urls){ 30 | return urls 31 | .filter(function(url){ 32 | return url.match(/^http/i); // Must start with HTTP (case insensitive) 33 | }) 34 | .map(function(url){ 35 | return url.split('#')[0]; // Strip hashes 36 | }) 37 | .filter(function(url, index, self){ 38 | return self.indexOf(url) === index; // Only unique URLs 39 | }) 40 | .map(function(url){ 41 | let weight = 0.5; 42 | if(typeof domainConfig.getUrlWeight === 'function'){ 43 | weight = domainConfig.getUrlWeight(url); 44 | } 45 | return { 46 | url, 47 | domain: parseUrl(url).host, // Extract the domain 48 | weight, 49 | created: new Date() 50 | }; 51 | }); 52 | } 53 | -------------------------------------------------------------------------------- /app/mysql/README.md: -------------------------------------------------------------------------------- 1 | ## MySQL (localhost) installation 2 | 3 | 1 - Go here 4 | https://dev.mysql.com/downloads/mysql/ 5 | ... 6 | n - After running the installer you will see a message like this: 7 | > A temporary password is generated for root@localhost: abcd1234ABCD 8 | > If you lose this password, please consult the section How to Reset the Root Password in the MySQL reference manual. 9 | ...make note of the password! 10 | n - Test it is running with 11 | `mysqladmin -u root version` 12 | 13 | ### Automatically Starting 14 | 15 | MySQL will automatically start. 16 | 17 | You can access the server (on OSX) via **System Preferences**. 18 | 19 | https://dev.mysql.com/doc/refman/5.7/en/osx-installation-launchd.html 20 | 21 | ## Create our user 22 | 23 | ``` 24 | mysql -u root -p 25 | --- 26 | CREATE DATABASE scraper; 27 | CREATE USER 'charlotte'@'localhost' IDENTIFIED BY 'Wilbur'; 28 | GRANT ALL PRIVILEGES ON scraper.* TO 'charlotte'@'localhost'; 29 | FLUSH PRIVILEGES; 30 | exit 31 | -- 32 | mysql -u charlotte -p -D scraper 33 | SELECT DATABASE(); 34 | exit 35 | ``` 36 | 37 | ## Workbench installation 38 | 39 | 1 - Go here 40 | https://www.mysql.com/products/workbench/ 41 | 2 - Then here 42 | https://dev.mysql.com/downloads/workbench/ 43 | Select OS and click download 44 | 3 - No need to register; 45 | No thanks, just start my download. 46 | 47 | Exporting our EER Diagram `File > Export > Forward Engineer SQL CREATE Script` 48 | 49 | 50 | ## Create our database 51 | 52 | mysql -u charlotte -p scraper < ./scraper.sql 53 | 54 | 55 | # Useful statements 56 | 57 | ```mysql 58 | -- Recently updated URLs 59 | SELECT id, domain, url, status, created, updated, CHAR_LENGTH(html), hash FROM urls WHERE status IS NOT NULL and status != 'wip' ORDER BY updated DESC LIMIT 10; 60 | 61 | -- How many URLs per domain 62 | SELECT domain, COUNT(*) FROM urls GROUP BY domain; 63 | 64 | -- How many URLs of each status per domain 65 | SELECT domain, status, COUNT(*) FROM urls GROUP BY domain, status ORDER BY COUNT(*) DESC LIMIT 30; 66 | 67 | -- Finding duplicate pages for a given domain 68 | SELECT hash, COUNT(*) AS num FROM urls WHERE domain = 'domain.com' AND hash IS NOT NULL GROUP BY hash HAVING num > 1; 69 | 70 | -- Reset WIP urls 71 | UPDATE urls SET status = NULL WHERE status = 'wip'; 72 | ``` 73 | -------------------------------------------------------------------------------- /app/mysql/index.js: -------------------------------------------------------------------------------- 1 | exports.query = require('./query.js'); 2 | exports.init = require('./init.js'); 3 | 4 | 5 | exports.upsertNewUrls = require('./webgraph/upsertNewUrls.js'); 6 | exports.getUncrawledUrl = require('./webgraph/getUncrawledUrl.js'); 7 | exports.updateUrl = require('./webgraph/updateUrl.js'); 8 | exports.getProgress = require('./webgraph/getProgress.js'); 9 | -------------------------------------------------------------------------------- /app/mysql/init.js: -------------------------------------------------------------------------------- 1 | module.exports = init; 2 | 3 | 4 | const query = require('./query.js'); 5 | const term = require('terminal-kit').terminal; 6 | 7 | 8 | async function init(){ 9 | let rows = await query('SELECT DATABASE() AS db, USER() AS user;'); 10 | term(`^r❯^: Connected to ^c${rows[0]['db']}^: with ^c${rows[0]['user']}^:\n`); 11 | } 12 | -------------------------------------------------------------------------------- /app/mysql/query.js: -------------------------------------------------------------------------------- 1 | module.exports = query; 2 | 3 | 4 | const Promise = require('bluebird'); 5 | const mysql = require('promise-mysql'); 6 | const args = require('../args/index.js'); 7 | 8 | 9 | let pool; 10 | 11 | 12 | async function query(sql, values){ 13 | return await Promise.using(getConnection(), function(connection) { 14 | return connection 15 | .query(sql, values); 16 | }); 17 | } 18 | 19 | 20 | async function getConnection(){ 21 | const pool = await getPool(); 22 | return pool 23 | .getConnection() 24 | .disposer(function(connection) { 25 | pool.releaseConnection(connection); 26 | }); 27 | } 28 | 29 | 30 | async function getPool() { 31 | // From memory 32 | if(pool){ 33 | return pool; 34 | } 35 | // Create the pool from settings 36 | let settings = await args.getMysqlSettings(); 37 | settings.multipleStatements = true; 38 | pool = mysql.createPool(settings); 39 | return pool; 40 | } 41 | -------------------------------------------------------------------------------- /app/mysql/setup/scraper.mwb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oodavid/puppeteer-scraper/286326c380eb026c730fccf337a0ff1a2bcda0f9/app/mysql/setup/scraper.mwb -------------------------------------------------------------------------------- /app/mysql/setup/scraper.sql: -------------------------------------------------------------------------------- 1 | -- MySQL Script generated by MySQL Workbench 2 | -- Sat Nov 18 17:17:03 2017 3 | -- Model: New Model Version: 1.0 4 | SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0; 5 | SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0; 6 | SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='TRADITIONAL,ALLOW_INVALID_DATES'; 7 | 8 | -- ----------------------------------------------------- 9 | -- Schema scraper 10 | -- ----------------------------------------------------- 11 | CREATE SCHEMA IF NOT EXISTS `scraper` DEFAULT CHARACTER SET utf8mb4 ; 12 | USE `scraper` ; 13 | 14 | -- ----------------------------------------------------- 15 | -- Table `scraper`.`urls` 16 | -- ----------------------------------------------------- 17 | CREATE TABLE IF NOT EXISTS `scraper`.`urls` ( 18 | `id` INT NOT NULL AUTO_INCREMENT, 19 | `domain` VARCHAR(100) NULL, 20 | `url` VARCHAR(750) NULL, 21 | `hash` VARCHAR(32) NULL, 22 | `status` VARCHAR(3) NULL, 23 | `html` LONGTEXT NULL, 24 | `created` DATETIME NULL DEFAULT CURRENT_TIMESTAMP, 25 | `updated` DATETIME NULL, 26 | PRIMARY KEY (`id`), 27 | UNIQUE INDEX `url_UNIQUE` (`url` ASC), 28 | INDEX `domain_INDEX` (`domain` ASC), 29 | INDEX `hash_INDEX` (`hash` ASC), 30 | INDEX `status_INDEX` (`status` ASC), 31 | INDEX `created_INDEX` (`created` ASC), 32 | INDEX `updated_INDEX` (`updated` ASC)) 33 | ENGINE = InnoDB; 34 | 35 | 36 | -- ----------------------------------------------------- 37 | -- Table `scraper`.`links` 38 | -- ----------------------------------------------------- 39 | CREATE TABLE IF NOT EXISTS `scraper`.`links` ( 40 | `from_url_id` INT NOT NULL, 41 | `to_url_id` INT NOT NULL, 42 | INDEX `fk_links_from_url_id_idx` (`from_url_id` ASC), 43 | INDEX `fk_links_to_url_id_idx` (`to_url_id` ASC), 44 | UNIQUE INDEX `link_UNIQUE` (`from_url_id` ASC, `to_url_id` ASC), 45 | CONSTRAINT `fk_links_from_url_id` 46 | FOREIGN KEY (`from_url_id`) 47 | REFERENCES `scraper`.`urls` (`id`) 48 | ON DELETE NO ACTION 49 | ON UPDATE NO ACTION, 50 | CONSTRAINT `fk_links_to_url_id` 51 | FOREIGN KEY (`to_url_id`) 52 | REFERENCES `scraper`.`urls` (`id`) 53 | ON DELETE NO ACTION 54 | ON UPDATE NO ACTION) 55 | ENGINE = InnoDB; 56 | 57 | 58 | SET SQL_MODE=@OLD_SQL_MODE; 59 | SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS; 60 | SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS; 61 | -------------------------------------------------------------------------------- /app/mysql/webgraph/getProgress.js: -------------------------------------------------------------------------------- 1 | module.exports = getProgress; 2 | 3 | 4 | const mysql = require('../mysql/'); 5 | 6 | 7 | async function getProgress(domain){ 8 | const sql = ` 9 | SELECT COUNT(*) as total FROM urls WHERE domain = ?; 10 | SELECT COUNT(*) as crawled FROM urls WHERE domain = ? AND status IS NOT NULL AND status != 'wip'; 11 | `; 12 | const values = [ domain, domain ]; 13 | let rows = await mysql.query(sql, values); 14 | if(rows && rows[0] && rows[0][0] && rows[1] && rows[1][0]){ 15 | return { 16 | total: rows[0][0].total, 17 | crawled: rows[1][0].crawled, 18 | }; 19 | } else { 20 | return { 21 | total: 0, 22 | crawled: 0, 23 | }; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /app/mysql/webgraph/getUncrawledUrl.js: -------------------------------------------------------------------------------- 1 | module.exports = getUncrawledUrl; 2 | 3 | 4 | const mysql = require('../mysql/'); 5 | const err = require('../hud/error.js'); 6 | 7 | 8 | async function getUncrawledUrl(domain){ 9 | const sql = ` 10 | SELECT id, url, @id:=id FROM urls WHERE status IS NULL AND domain = ? LIMIT 1 FOR UPDATE; 11 | UPDATE urls SET status='wip', updated = NOW() WHERE id=@id; 12 | `; 13 | const values = [ domain ]; 14 | let rows = await mysql.query(sql, values); 15 | if(rows && rows[0] && rows[0][0] && rows[0][0].id){ 16 | return { 17 | id: rows[0][0].id, 18 | url: rows[0][0].url, 19 | }; 20 | } else { 21 | return false; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /app/mysql/webgraph/updateUrl.js: -------------------------------------------------------------------------------- 1 | module.exports = updateUrl; 2 | 3 | 4 | const hud = require('../hud/'); 5 | const mysql = require('../mysql/'); 6 | const parseUrl = require('url').parse; 7 | 8 | 9 | // Where urlObject is an object of format { id, status, html, hash, links } 10 | async function updateUrl(domainConfig, urlObject){ 11 | // Build up our SQL statement and values 12 | let sql = ''; 13 | let values = []; 14 | // Run the query 15 | try { 16 | // Update the URL itself 17 | sql += 'UPDATE urls SET status = ?, html = ?, hash = ?, updated = NOW() WHERE id = ?;'; 18 | values.push(... [ urlObject.status, urlObject.html, urlObject.hash, urlObject.id ]); 19 | // Create the links 20 | const links = parseLinks(urlObject.links); 21 | links.forEach(function(link){ 22 | const domain = parseUrl(link).host; 23 | if(domain){ 24 | sql += 'INSERT INTO urls (domain, url) VALUES (?, ?) ON DUPLICATE KEY UPDATE id=id;'; 25 | values.push(... [ domain, link ]); 26 | sql += 'INSERT IGNORE INTO links (from_url_id, to_url_id) VALUES (?, (SELECT id FROM urls WHERE url=? LIMIT 1));'; 27 | values.push(... [ urlObject.id, link ]); 28 | } 29 | }); 30 | await mysql.query(sql, values); 31 | } catch(e){ 32 | hud.error(e); 33 | hud.error(new Error(sql)); 34 | hud.error(new Error(JSON.stringify(values, null, 2))); 35 | } 36 | } 37 | 38 | 39 | function parseLinks(links){ 40 | // Only HTTP (and HTTPS) 41 | links = links.filter(function(link){ 42 | return link.indexOf('http') === 0; 43 | }); 44 | // Strip hashes 45 | links.forEach(function(value, index, arr){ 46 | arr[index] = value.split('#')[0]; 47 | }); 48 | // Make sure the links are unique 49 | links = [... new Set([... links])]; 50 | return links; 51 | } 52 | -------------------------------------------------------------------------------- /app/mysql/webgraph/upsertNewUrls.js: -------------------------------------------------------------------------------- 1 | module.exports = upsertNewUrls; 2 | 3 | 4 | const mysql = require('../mysql/'); 5 | const parseUrl = require('url').parse; 6 | 7 | 8 | async function upsertNewUrls(domainConfig, seedUrls){ 9 | // The `query` module needs `values` in a specific format to trigger a bulk insert, see: https://stackoverflow.com/a/14259347/1122851 10 | let values = [ 11 | seedUrls.map(function(url){ 12 | const domain = parseUrl(url).host; 13 | return [ domain, url ]; 14 | }) 15 | ]; 16 | return await mysql.query('INSERT INTO urls (domain, url) VALUES ? ON DUPLICATE KEY UPDATE id=id;', values); 17 | } 18 | -------------------------------------------------------------------------------- /domains/checkatrade.com/index.js: -------------------------------------------------------------------------------- 1 | exports.domain = 'www.checkatrade.com'; 2 | exports.seedUrls = getSeedUrls(); 3 | exports.parse = parse; 4 | exports.getUrlWeight = getUrlWeight; 5 | 6 | 7 | // Return an array of initial Urls to parse 8 | function getSeedUrls(){ 9 | return [ 10 | 'http://www.checkatrade.com/', 11 | 'http://www.checkatrade.com/JamesPond/', 12 | 'http://www.checkatrade.com/JamesPond/Reviews.aspx', 13 | 'http://www.checkatrade.com/JamesPond/Gallery.aspx', 14 | 'http://www.checkatrade.com/JamesPond/Services.aspx', 15 | 'http://www.checkatrade.com/JamesPond/Checks.aspx', 16 | ]; 17 | }; 18 | 19 | // Parse the page, returning data 20 | async function parse(page, urlObject){ 21 | const data = await page.evaluate(function(){ 22 | var $ = window.$ || window.jQuery; 23 | // Blank data 24 | var data = {}; 25 | // The "id" is simply the first part of the Url, ie: /BusinessName/Services = "BusinessName" 26 | var m = location.pathname.match(/^\/([^\/]*)\//i); 27 | var id = m ? m[1] : false; 28 | if(id){ 29 | data.id = id; 30 | // Rip out every property from the contact card 31 | $('.contact-card [itemprop]').each(function(){ 32 | var prop = $(this).attr('itemprop'); 33 | // Ignore these wrapper elements 34 | if($.inArray(prop, ['aggregateRating']) !== -1){ return true; } 35 | // Values come in different forms 36 | var value; 37 | switch(prop){ 38 | // Rating 39 | case "ratingValue": value = parseFloat($(this).text(), 10); break; 40 | case "reviewCount": value = parseInt($(this).text()); break; 41 | case "worstRating": value = parseInt($(this).attr('content')); break; 42 | case "bestRating": value = parseInt($(this).attr('content')); break; 43 | // By default grab the content 44 | default: value = $.trim($(this).text()); break; 45 | } 46 | // Add to the data, duplicate keys should create arrays (ie: telephone) 47 | if(data[prop] && !Array.isArray(data[prop])){ 48 | data[prop] = [ data[prop] ]; 49 | } 50 | if(Array.isArray(data[prop])){ 51 | data[prop].push(value); 52 | } else { 53 | data[prop] = value; 54 | } 55 | }); 56 | // Can we see the company id? 57 | var companyId = $('[data-company-id]'); 58 | if(companyId.length){ 59 | data.checkatradeId = parseInt(companyId.attr('data-company-id')); 60 | } 61 | // Some pages have a well-formatted "company" object, we'll have that 62 | if(window.cat && window.cat.dataHeaderData && window.cat.dataHeaderData.Companies.length){ 63 | data.company = window.cat.dataHeaderData.Companies[0]; 64 | } 65 | // Checks - I think they really just mean "checks" here, not "checksum" - how silly. 66 | var checks = $('.member-checksum li'); 67 | if(checks.length){ 68 | data.checks = []; 69 | checks.each(function(){ 70 | data.checks.push($.trim($(this).text())); 71 | }); 72 | } 73 | // Skills 74 | var skills = $('.skills-list > li'); 75 | if(skills.length){ 76 | data.skills = {}; 77 | skills.each(function(){ 78 | var heading = $.trim($(this).find('> span').text()); 79 | data.skills[heading] = []; 80 | $(this).find('li').each(function(){ 81 | data.skills[heading].push($.trim($(this).text())); 82 | }); 83 | }); 84 | } 85 | // Services 86 | var services = $('.member-services__serv-list > div'); 87 | if(services.length){ 88 | data.services = []; 89 | services.each(function(){ 90 | data.services.push($.trim($(this).text())); 91 | }); 92 | } 93 | // Map 94 | // http://www.checkatrade.com/BandKElectricalsLtd/Checks.aspx 95 | // VAT Number 96 | // Company Name 97 | // Company ID 98 | // Director 99 | } 100 | return data; 101 | }); 102 | return data; 103 | } 104 | 105 | 106 | // Return the "weight" of an URL from 0-1 107 | function getUrlWeight(url){ 108 | var weights = { 109 | '/Account/': 0, // Customer Sign-In 110 | '/Report.aspx': 0, // Printable report 111 | '/GiveFeedback/': 0, // Give feedback about a member 112 | '/Search/': 0.1, // Good for building the network, but I think there are infinite search variations exposed 113 | '/Gallery.aspx': 0.3, // Just photos of the member's work 114 | '/Reviews.aspx': 0.4, // Reviews about the member 115 | '/Reputation': 0.2 // Can't remember why I downgraded this... 116 | }; 117 | // See if we can match it 118 | for(var key in weights){ 119 | if(weights.hasOwnProperty(key)){ 120 | if(url.indexOf(key) !== -1){ 121 | return weights[key]; 122 | } 123 | } 124 | } 125 | // Everything else is top priority 126 | return 1; 127 | }; 128 | -------------------------------------------------------------------------------- /domains/oodavid.com/index.js: -------------------------------------------------------------------------------- 1 | exports.domain = 'oodavid.com'; 2 | exports.seedUrls = getSeedUrls(); 3 | exports.getUrlWeight = getUrlWeight; 4 | exports.parseForData = parseForData; 5 | exports.throttle = 0; 6 | 7 | // Return an array of initial Urls to parse 8 | function getSeedUrls(){ 9 | return [ 'https://oodavid.com/' ]; 10 | }; 11 | 12 | // Return the "weight" of an URL from 0-1. 13 | // ...this dictates the order that pages are scraped 14 | function getUrlWeight(url){ 15 | var weights = { 16 | '/Account/': 0, // Customer Sign-In 17 | '/Report.aspx': 0, // Printable report 18 | '/GiveFeedback/': 0, // Give feedback about a member 19 | '/Search/': 0.1, // Good for building the network, but I think there are infinite search variations exposed 20 | '/Gallery.aspx': 0.3, // Just photos of the member's work 21 | '/Reviews.aspx': 0.4, // Reviews about the member 22 | '/Reputation': 0.2 // Can't remember why I downgraded this... 23 | }; 24 | // See if we can match it 25 | for(var key in weights){ 26 | if(weights.hasOwnProperty(key)){ 27 | if(url.pathname.indexOf(key) !== -1){ 28 | return weights[key]; 29 | } 30 | } 31 | } 32 | // Everything else is top priority 33 | return 1; 34 | }; 35 | 36 | // Return a { data } object to be stored with the Url 37 | // ...this runs in the window context, so console.log won't output to NodeJS (it operates in the horseman window) 38 | // ...this MUST have an `id` value to uniquely identify the lead 39 | function parseForData(){ 40 | var $ = window.$ || window.jQuery; 41 | // Blank data 42 | var data = {}; 43 | // The "id" is simply the first part of the Url, ie: /BusinessName/Services = "BusinessName" 44 | var m = location.pathname.match(/^\/([^\/]*)\//i); 45 | var id = m ? m[1] : false; 46 | if(id){ 47 | data.id = id; 48 | // Rip out every property from the contact card 49 | $('.contact-card [itemprop]').each(function(){ 50 | var prop = $(this).attr('itemprop'); 51 | // Ignore these wrapper elements 52 | if($.inArray(prop, ['aggregateRating']) !== -1){ return true; } 53 | // Values come in different forms 54 | var value; 55 | switch(prop){ 56 | // Rating 57 | case "ratingValue": value = parseFloat($(this).text(), 10); break; 58 | case "reviewCount": value = parseInt($(this).text()); break; 59 | case "worstRating": value = parseInt($(this).attr('content')); break; 60 | case "bestRating": value = parseInt($(this).attr('content')); break; 61 | // By default grab the content 62 | default: value = $.trim($(this).text()); break; 63 | } 64 | // Add to the data, duplicate keys should create arrays (ie: telephone) 65 | if(data[prop] && !Array.isArray(data[prop])){ 66 | data[prop] = [ data[prop] ]; 67 | } 68 | if(Array.isArray(data[prop])){ 69 | data[prop].push(value); 70 | } else { 71 | data[prop] = value; 72 | } 73 | }); 74 | // Can we see the company id? 75 | var companyId = $('[data-company-id]'); 76 | if(companyId.length){ 77 | data.checkatradeId = parseInt(companyId.attr('data-company-id')); 78 | } 79 | // Some pages have a well-formatted "company" object, we'll have that 80 | if(window.cat && window.cat.dataHeaderData && window.cat.dataHeaderData.Companies.length){ 81 | data.company = window.cat.dataHeaderData.Companies[0]; 82 | } 83 | // Checks - I think they really just mean "checks" here, not "checksum" - how silly. 84 | var checks = $('.member-checksum li'); 85 | if(checks.length){ 86 | data.checks = []; 87 | checks.each(function(){ 88 | data.checks.push($.trim($(this).text())); 89 | }); 90 | } 91 | // Skills 92 | var skills = $('.skills-list > li'); 93 | if(skills){ 94 | data.skills = {}; 95 | skills.each(function(){ 96 | var heading = $.trim($(this).find('> span').text()); 97 | data.skills[heading] = []; 98 | $(this).find('li').each(function(){ 99 | data.skills[heading].push($.trim($(this).text())); 100 | }); 101 | }); 102 | } 103 | // Services 104 | var services = $('.member-services__serv-list > div'); 105 | if(services){ 106 | data.services = []; 107 | services.each(function(){ 108 | data.services.push($.trim($(this).text())); 109 | }); 110 | } 111 | // Map 112 | // http://www.checkatrade.com/BandKElectricalsLtd/Checks.aspx 113 | // VAT Number 114 | // Company Name 115 | // Company ID 116 | // Director 117 | } 118 | return data; 119 | }; 120 | -------------------------------------------------------------------------------- /domains/template/template.js: -------------------------------------------------------------------------------- 1 | /* 2 | regex url matching 3 | getSeedUrls 4 | getUrlThrottle 5 | getUrlPriority 6 | parseHtml 7 | auth 8 | */ 9 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const hud = require('./app/hud/'); 2 | const args = require('./app/args/'); 3 | const datastore = require('./app/mongo/'); 4 | const term = require('terminal-kit').terminal; 5 | 6 | 7 | (async () => { 8 | hud.intro(); 9 | // Connect to MySQL 10 | await datastore.init(); 11 | // Get the arguments 12 | const action = await args.getAction(); 13 | const domainName = await args.getDomain(); 14 | const numThreads = await args.getThreads(); 15 | term(`^r❯^: ^+To run this without prompts, use^:\n`); 16 | term(`^r❯^: ^-node index.js --action ${action} --domain ${domainName} --threads ${numThreads}^:\n`); 17 | // Read the config 18 | const domainConfig = require(`./domains/${domainName}/`); 19 | // Start the action 20 | if(action === 'crawl'){ 21 | const crawler = require('./app/crawler/'); 22 | crawler.start(domainConfig, numThreads); 23 | } else { 24 | const parser = require('./app/parser/'); 25 | parser.start(domainConfig, numThreads); 26 | } 27 | })(); 28 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "puppeteer-scraper", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "@types/mysql": { 8 | "version": "2.15.1", 9 | "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.1.tgz", 10 | "integrity": "sha512-CLdDYJMQjM3yFbCVcNAxplWX0WEPI6RQesJapPo9TbduGbrfielJJdMafslycAeC36OwgtWHE06YnwgzSKl3xw==", 11 | "requires": { 12 | "@types/node": "8.0.51" 13 | } 14 | }, 15 | "@types/node": { 16 | "version": "8.0.51", 17 | "resolved": "https://registry.npmjs.org/@types/node/-/node-8.0.51.tgz", 18 | "integrity": "sha512-El3+WJk2D/ppWNd2X05aiP5l2k4EwF7KwheknQZls+I26eSICoWRhRIJ56jGgw2dqNGQ5LtNajmBU2ajS28EvQ==" 19 | }, 20 | "agent-base": { 21 | "version": "4.1.1", 22 | "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.1.1.tgz", 23 | "integrity": "sha512-yWGUUmCZD/33IRjG2It94PzixT8lX+47Uq8fjmd0cgQWITCMrJuXFaVIMnGDmDnZGGKAGdwTx8UGeU8lMR2urA==", 24 | "requires": { 25 | "es6-promisify": "5.0.0" 26 | } 27 | }, 28 | "ajv": { 29 | "version": "5.2.3", 30 | "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.2.3.tgz", 31 | "integrity": "sha1-wG9Zh3jETGsWGrr+NGa4GtGBTtI=", 32 | "requires": { 33 | "co": "4.6.0", 34 | "fast-deep-equal": "1.0.0", 35 | "json-schema-traverse": "0.3.1", 36 | "json-stable-stringify": "1.0.1" 37 | } 38 | }, 39 | "ansi-escapes": { 40 | "version": "2.0.0", 41 | "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-2.0.0.tgz", 42 | "integrity": "sha1-W65SvkJIeN2Xg+iRDj/Cki6DyBs=" 43 | }, 44 | "ansi-regex": { 45 | "version": "2.1.1", 46 | "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", 47 | "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" 48 | }, 49 | "ansi-styles": { 50 | "version": "2.2.1", 51 | "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", 52 | "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" 53 | }, 54 | "asn1": { 55 | "version": "0.2.3", 56 | "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz", 57 | "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=" 58 | }, 59 | "assert-plus": { 60 | "version": "1.0.0", 61 | "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", 62 | "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" 63 | }, 64 | "async-kit": { 65 | "version": "2.2.3", 66 | "resolved": "https://registry.npmjs.org/async-kit/-/async-kit-2.2.3.tgz", 67 | "integrity": "sha1-JkdRonndxfWbQZY4uAWuLEmFj7c=", 68 | "requires": { 69 | "nextgen-events": "0.9.9", 70 | "tree-kit": "0.5.26" 71 | }, 72 | "dependencies": { 73 | "nextgen-events": { 74 | "version": "0.9.9", 75 | "resolved": "https://registry.npmjs.org/nextgen-events/-/nextgen-events-0.9.9.tgz", 76 | "integrity": "sha1-OaivxKK4RTiMV+LGu5cWcRmGo6A=" 77 | } 78 | } 79 | }, 80 | "async-limiter": { 81 | "version": "1.0.0", 82 | "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz", 83 | "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==" 84 | }, 85 | "asynckit": { 86 | "version": "0.4.0", 87 | "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", 88 | "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" 89 | }, 90 | "aws-sign2": { 91 | "version": "0.7.0", 92 | "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", 93 | "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" 94 | }, 95 | "aws4": { 96 | "version": "1.6.0", 97 | "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz", 98 | "integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4=" 99 | }, 100 | "balanced-match": { 101 | "version": "1.0.0", 102 | "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", 103 | "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" 104 | }, 105 | "bcrypt-pbkdf": { 106 | "version": "1.0.1", 107 | "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz", 108 | "integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=", 109 | "optional": true, 110 | "requires": { 111 | "tweetnacl": "0.14.5" 112 | } 113 | }, 114 | "bignumber.js": { 115 | "version": "4.0.4", 116 | "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-4.0.4.tgz", 117 | "integrity": "sha512-LDXpJKVzEx2/OqNbG9mXBNvHuiRL4PzHCGfnANHMJ+fv68Ads3exDVJeGDJws+AoNEuca93bU3q+S0woeUaCdg==" 118 | }, 119 | "bluebird": { 120 | "version": "3.5.1", 121 | "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz", 122 | "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA==" 123 | }, 124 | "boom": { 125 | "version": "4.3.1", 126 | "resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz", 127 | "integrity": "sha1-T4owBctKfjiJ90kDD9JbluAdLjE=", 128 | "requires": { 129 | "hoek": "4.2.0" 130 | } 131 | }, 132 | "brace-expansion": { 133 | "version": "1.1.8", 134 | "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.8.tgz", 135 | "integrity": "sha1-wHshHHyVLsH479Uad+8NHTmQopI=", 136 | "requires": { 137 | "balanced-match": "1.0.0", 138 | "concat-map": "0.0.1" 139 | } 140 | }, 141 | "browser-request": { 142 | "version": "0.3.3", 143 | "resolved": "https://registry.npmjs.org/browser-request/-/browser-request-0.3.3.tgz", 144 | "integrity": "sha1-ns5bWsqJopkyJC4Yv5M975h2zBc=" 145 | }, 146 | "bson": { 147 | "version": "1.0.4", 148 | "resolved": "https://registry.npmjs.org/bson/-/bson-1.0.4.tgz", 149 | "integrity": "sha1-k8ENOeqltYQVy8QFLz5T5WKwtyw=" 150 | }, 151 | "buffer-shims": { 152 | "version": "1.0.0", 153 | "resolved": "https://registry.npmjs.org/buffer-shims/-/buffer-shims-1.0.0.tgz", 154 | "integrity": "sha1-mXjOMXOIxkmth5MCjDR37wRKi1E=" 155 | }, 156 | "camelcase": { 157 | "version": "4.1.0", 158 | "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz", 159 | "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=" 160 | }, 161 | "caseless": { 162 | "version": "0.12.0", 163 | "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", 164 | "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" 165 | }, 166 | "chalk": { 167 | "version": "1.1.3", 168 | "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", 169 | "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", 170 | "requires": { 171 | "ansi-styles": "2.2.1", 172 | "escape-string-regexp": "1.0.5", 173 | "has-ansi": "2.0.0", 174 | "strip-ansi": "3.0.1", 175 | "supports-color": "2.0.0" 176 | } 177 | }, 178 | "charenc": { 179 | "version": "0.0.2", 180 | "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", 181 | "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=" 182 | }, 183 | "cli-cursor": { 184 | "version": "2.1.0", 185 | "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", 186 | "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", 187 | "requires": { 188 | "restore-cursor": "2.0.0" 189 | } 190 | }, 191 | "cli-width": { 192 | "version": "2.2.0", 193 | "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.0.tgz", 194 | "integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=" 195 | }, 196 | "cliui": { 197 | "version": "3.2.0", 198 | "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz", 199 | "integrity": "sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0=", 200 | "requires": { 201 | "string-width": "1.0.2", 202 | "strip-ansi": "3.0.1", 203 | "wrap-ansi": "2.1.0" 204 | }, 205 | "dependencies": { 206 | "is-fullwidth-code-point": { 207 | "version": "1.0.0", 208 | "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", 209 | "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", 210 | "requires": { 211 | "number-is-nan": "1.0.1" 212 | } 213 | }, 214 | "string-width": { 215 | "version": "1.0.2", 216 | "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", 217 | "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", 218 | "requires": { 219 | "code-point-at": "1.1.0", 220 | "is-fullwidth-code-point": "1.0.0", 221 | "strip-ansi": "3.0.1" 222 | } 223 | } 224 | } 225 | }, 226 | "cloudant-follow": { 227 | "version": "0.13.0", 228 | "resolved": "https://registry.npmjs.org/cloudant-follow/-/cloudant-follow-0.13.0.tgz", 229 | "integrity": "sha1-fs6teQYbADmfXuNUA2jdYGV83Cs=", 230 | "requires": { 231 | "browser-request": "0.3.3", 232 | "debug": "2.6.9", 233 | "request": "2.81.0" 234 | }, 235 | "dependencies": { 236 | "ajv": { 237 | "version": "4.11.8", 238 | "resolved": "https://registry.npmjs.org/ajv/-/ajv-4.11.8.tgz", 239 | "integrity": "sha1-gv+wKynmYq5TvcIK8VlHcGc5xTY=", 240 | "requires": { 241 | "co": "4.6.0", 242 | "json-stable-stringify": "1.0.1" 243 | } 244 | }, 245 | "assert-plus": { 246 | "version": "0.2.0", 247 | "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz", 248 | "integrity": "sha1-104bh+ev/A24qttwIfP+SBAasjQ=" 249 | }, 250 | "aws-sign2": { 251 | "version": "0.6.0", 252 | "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz", 253 | "integrity": "sha1-FDQt0428yU0OW4fXY81jYSwOeU8=" 254 | }, 255 | "boom": { 256 | "version": "2.10.1", 257 | "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz", 258 | "integrity": "sha1-OciRjO/1eZ+D+UkqhI9iWt0Mdm8=", 259 | "requires": { 260 | "hoek": "2.16.3" 261 | } 262 | }, 263 | "cryptiles": { 264 | "version": "2.0.5", 265 | "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz", 266 | "integrity": "sha1-O9/s3GCBR8HGcgL6KR59ylnqo7g=", 267 | "requires": { 268 | "boom": "2.10.1" 269 | } 270 | }, 271 | "form-data": { 272 | "version": "2.1.4", 273 | "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.1.4.tgz", 274 | "integrity": "sha1-M8GDrPGTJ27KqYFDpp6Uv+4XUNE=", 275 | "requires": { 276 | "asynckit": "0.4.0", 277 | "combined-stream": "1.0.5", 278 | "mime-types": "2.1.17" 279 | } 280 | }, 281 | "har-schema": { 282 | "version": "1.0.5", 283 | "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-1.0.5.tgz", 284 | "integrity": "sha1-0mMTX0MwfALGAq/I/pWXDAFRNp4=" 285 | }, 286 | "har-validator": { 287 | "version": "4.2.1", 288 | "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-4.2.1.tgz", 289 | "integrity": "sha1-M0gdDxu/9gDdID11gSpqX7oALio=", 290 | "requires": { 291 | "ajv": "4.11.8", 292 | "har-schema": "1.0.5" 293 | } 294 | }, 295 | "hawk": { 296 | "version": "3.1.3", 297 | "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz", 298 | "integrity": "sha1-B4REvXwWQLD+VA0sm3PVlnjo4cQ=", 299 | "requires": { 300 | "boom": "2.10.1", 301 | "cryptiles": "2.0.5", 302 | "hoek": "2.16.3", 303 | "sntp": "1.0.9" 304 | } 305 | }, 306 | "hoek": { 307 | "version": "2.16.3", 308 | "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz", 309 | "integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0=" 310 | }, 311 | "http-signature": { 312 | "version": "1.1.1", 313 | "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz", 314 | "integrity": "sha1-33LiZwZs0Kxn+3at+OE0qPvPkb8=", 315 | "requires": { 316 | "assert-plus": "0.2.0", 317 | "jsprim": "1.4.1", 318 | "sshpk": "1.13.1" 319 | } 320 | }, 321 | "performance-now": { 322 | "version": "0.2.0", 323 | "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-0.2.0.tgz", 324 | "integrity": "sha1-M+8wxcd9TqIcWlOGnZG1bY8lVeU=" 325 | }, 326 | "qs": { 327 | "version": "6.4.0", 328 | "resolved": "https://registry.npmjs.org/qs/-/qs-6.4.0.tgz", 329 | "integrity": "sha1-E+JtKK1rD/qpExLNO/cI7TUecjM=" 330 | }, 331 | "request": { 332 | "version": "2.81.0", 333 | "resolved": "https://registry.npmjs.org/request/-/request-2.81.0.tgz", 334 | "integrity": "sha1-xpKJRqDgbF+Nb4qTM0af/aRimKA=", 335 | "requires": { 336 | "aws-sign2": "0.6.0", 337 | "aws4": "1.6.0", 338 | "caseless": "0.12.0", 339 | "combined-stream": "1.0.5", 340 | "extend": "3.0.1", 341 | "forever-agent": "0.6.1", 342 | "form-data": "2.1.4", 343 | "har-validator": "4.2.1", 344 | "hawk": "3.1.3", 345 | "http-signature": "1.1.1", 346 | "is-typedarray": "1.0.0", 347 | "isstream": "0.1.2", 348 | "json-stringify-safe": "5.0.1", 349 | "mime-types": "2.1.17", 350 | "oauth-sign": "0.8.2", 351 | "performance-now": "0.2.0", 352 | "qs": "6.4.0", 353 | "safe-buffer": "5.1.1", 354 | "stringstream": "0.0.5", 355 | "tough-cookie": "2.3.3", 356 | "tunnel-agent": "0.6.0", 357 | "uuid": "3.1.0" 358 | } 359 | }, 360 | "sntp": { 361 | "version": "1.0.9", 362 | "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz", 363 | "integrity": "sha1-ZUEYTMkK7qbG57NeJlkIJEPGYZg=", 364 | "requires": { 365 | "hoek": "2.16.3" 366 | } 367 | } 368 | } 369 | }, 370 | "co": { 371 | "version": "4.6.0", 372 | "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", 373 | "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" 374 | }, 375 | "code-point-at": { 376 | "version": "1.1.0", 377 | "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", 378 | "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" 379 | }, 380 | "color-convert": { 381 | "version": "1.9.0", 382 | "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.0.tgz", 383 | "integrity": "sha1-Gsz5fdc5uYO/mU1W/sj5WFNkG3o=", 384 | "requires": { 385 | "color-name": "1.1.3" 386 | } 387 | }, 388 | "color-name": { 389 | "version": "1.1.3", 390 | "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", 391 | "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" 392 | }, 393 | "combined-stream": { 394 | "version": "1.0.5", 395 | "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz", 396 | "integrity": "sha1-k4NwpXtKUd6ix3wV1cX9+JUWQAk=", 397 | "requires": { 398 | "delayed-stream": "1.0.0" 399 | } 400 | }, 401 | "concat-map": { 402 | "version": "0.0.1", 403 | "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", 404 | "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" 405 | }, 406 | "concat-stream": { 407 | "version": "1.6.0", 408 | "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.0.tgz", 409 | "integrity": "sha1-CqxmL9Ur54lk1VMvaUeE5wEQrPc=", 410 | "requires": { 411 | "inherits": "2.0.3", 412 | "readable-stream": "2.3.3", 413 | "typedarray": "0.0.6" 414 | } 415 | }, 416 | "core-util-is": { 417 | "version": "1.0.2", 418 | "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", 419 | "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" 420 | }, 421 | "cross-spawn": { 422 | "version": "5.1.0", 423 | "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", 424 | "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=", 425 | "requires": { 426 | "lru-cache": "4.1.1", 427 | "shebang-command": "1.2.0", 428 | "which": "1.3.0" 429 | } 430 | }, 431 | "crypt": { 432 | "version": "0.0.2", 433 | "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", 434 | "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=" 435 | }, 436 | "cryptiles": { 437 | "version": "3.1.2", 438 | "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-3.1.2.tgz", 439 | "integrity": "sha1-qJ+7Ig9c4l7FboxKqKT9e1sNKf4=", 440 | "requires": { 441 | "boom": "5.2.0" 442 | }, 443 | "dependencies": { 444 | "boom": { 445 | "version": "5.2.0", 446 | "resolved": "https://registry.npmjs.org/boom/-/boom-5.2.0.tgz", 447 | "integrity": "sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw==", 448 | "requires": { 449 | "hoek": "4.2.0" 450 | } 451 | } 452 | } 453 | }, 454 | "cwise-compiler": { 455 | "version": "1.1.3", 456 | "resolved": "https://registry.npmjs.org/cwise-compiler/-/cwise-compiler-1.1.3.tgz", 457 | "integrity": "sha1-9NZnQQ6FDToxOn0tt7HlBbsDTMU=", 458 | "requires": { 459 | "uniq": "1.0.1" 460 | } 461 | }, 462 | "dashdash": { 463 | "version": "1.14.1", 464 | "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", 465 | "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", 466 | "requires": { 467 | "assert-plus": "1.0.0" 468 | } 469 | }, 470 | "data-uri-to-buffer": { 471 | "version": "0.0.3", 472 | "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-0.0.3.tgz", 473 | "integrity": "sha1-GK6XmmoMqZSwYlhTkW0mYruuCxo=" 474 | }, 475 | "dateformat": { 476 | "version": "3.0.2", 477 | "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.2.tgz", 478 | "integrity": "sha1-mk30v/FYrC80vGN6vbFUcWB+Flk=" 479 | }, 480 | "debug": { 481 | "version": "2.6.9", 482 | "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", 483 | "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", 484 | "requires": { 485 | "ms": "2.0.0" 486 | } 487 | }, 488 | "decamelize": { 489 | "version": "1.2.0", 490 | "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", 491 | "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=" 492 | }, 493 | "delayed-stream": { 494 | "version": "1.0.0", 495 | "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", 496 | "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" 497 | }, 498 | "ecc-jsbn": { 499 | "version": "0.1.1", 500 | "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", 501 | "integrity": "sha1-D8c6ntXw1Tw4GTOYUj735UN3dQU=", 502 | "optional": true, 503 | "requires": { 504 | "jsbn": "0.1.1" 505 | } 506 | }, 507 | "errs": { 508 | "version": "0.3.2", 509 | "resolved": "https://registry.npmjs.org/errs/-/errs-0.3.2.tgz", 510 | "integrity": "sha1-eYCZstvTfKK8dJ5TinwTB9C1BJk=" 511 | }, 512 | "es6-promise": { 513 | "version": "4.1.1", 514 | "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.1.1.tgz", 515 | "integrity": "sha512-OaU1hHjgJf+b0NzsxCg7NdIYERD6Hy/PEmFLTjw+b65scuisG3Kt4QoTvJ66BBkPZ581gr0kpoVzKnxniM8nng==" 516 | }, 517 | "es6-promisify": { 518 | "version": "5.0.0", 519 | "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", 520 | "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", 521 | "requires": { 522 | "es6-promise": "4.1.1" 523 | } 524 | }, 525 | "escape-string-regexp": { 526 | "version": "1.0.5", 527 | "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", 528 | "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" 529 | }, 530 | "execa": { 531 | "version": "0.7.0", 532 | "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz", 533 | "integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=", 534 | "requires": { 535 | "cross-spawn": "5.1.0", 536 | "get-stream": "3.0.0", 537 | "is-stream": "1.1.0", 538 | "npm-run-path": "2.0.2", 539 | "p-finally": "1.0.0", 540 | "signal-exit": "3.0.2", 541 | "strip-eof": "1.0.0" 542 | } 543 | }, 544 | "extend": { 545 | "version": "3.0.1", 546 | "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz", 547 | "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ=" 548 | }, 549 | "external-editor": { 550 | "version": "2.0.5", 551 | "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-2.0.5.tgz", 552 | "integrity": "sha512-Msjo64WT5W+NhOpQXh0nOHm+n0RfU1QUwDnKYvJ8dEJ8zlwLrqXNTv5mSUTJpepf41PDJGyhueTw2vNZW+Fr/w==", 553 | "requires": { 554 | "iconv-lite": "0.4.19", 555 | "jschardet": "1.6.0", 556 | "tmp": "0.0.33" 557 | } 558 | }, 559 | "extract-zip": { 560 | "version": "1.6.5", 561 | "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.5.tgz", 562 | "integrity": "sha1-maBnNbbqIOqbcF13ms/8yHz/BEA=", 563 | "requires": { 564 | "concat-stream": "1.6.0", 565 | "debug": "2.2.0", 566 | "mkdirp": "0.5.0", 567 | "yauzl": "2.4.1" 568 | }, 569 | "dependencies": { 570 | "debug": { 571 | "version": "2.2.0", 572 | "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz", 573 | "integrity": "sha1-+HBX6ZWxofauaklgZkE3vFbwOdo=", 574 | "requires": { 575 | "ms": "0.7.1" 576 | } 577 | }, 578 | "ms": { 579 | "version": "0.7.1", 580 | "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz", 581 | "integrity": "sha1-nNE8A62/8ltl7/3nzoZO6VIBcJg=" 582 | } 583 | } 584 | }, 585 | "extsprintf": { 586 | "version": "1.3.0", 587 | "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", 588 | "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" 589 | }, 590 | "fast-deep-equal": { 591 | "version": "1.0.0", 592 | "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz", 593 | "integrity": "sha1-liVqO8l1WV6zbYLpkp0GDYk0Of8=" 594 | }, 595 | "fd-slicer": { 596 | "version": "1.0.1", 597 | "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz", 598 | "integrity": "sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU=", 599 | "requires": { 600 | "pend": "1.2.0" 601 | } 602 | }, 603 | "figures": { 604 | "version": "2.0.0", 605 | "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", 606 | "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", 607 | "requires": { 608 | "escape-string-regexp": "1.0.5" 609 | } 610 | }, 611 | "find-up": { 612 | "version": "2.1.0", 613 | "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", 614 | "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", 615 | "requires": { 616 | "locate-path": "2.0.0" 617 | } 618 | }, 619 | "forever-agent": { 620 | "version": "0.6.1", 621 | "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", 622 | "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" 623 | }, 624 | "form-data": { 625 | "version": "2.3.1", 626 | "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.1.tgz", 627 | "integrity": "sha1-b7lPvXGIUwbXPRXMSX/kzE7NRL8=", 628 | "requires": { 629 | "asynckit": "0.4.0", 630 | "combined-stream": "1.0.5", 631 | "mime-types": "2.1.17" 632 | } 633 | }, 634 | "fs.realpath": { 635 | "version": "1.0.0", 636 | "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", 637 | "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" 638 | }, 639 | "fuzzy": { 640 | "version": "0.1.3", 641 | "resolved": "https://registry.npmjs.org/fuzzy/-/fuzzy-0.1.3.tgz", 642 | "integrity": "sha1-THbsL/CsGjap3M+aAN+GIweNTtg=" 643 | }, 644 | "get-caller-file": { 645 | "version": "1.0.2", 646 | "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.2.tgz", 647 | "integrity": "sha1-9wLmMSfn4jHBYKgMFVSstw1QR+U=" 648 | }, 649 | "get-pixels": { 650 | "version": "3.3.0", 651 | "resolved": "https://registry.npmjs.org/get-pixels/-/get-pixels-3.3.0.tgz", 652 | "integrity": "sha1-jZeVvq4YhQuED3SVgbrcBdPjbkE=", 653 | "requires": { 654 | "data-uri-to-buffer": "0.0.3", 655 | "jpeg-js": "0.1.2", 656 | "mime-types": "2.1.17", 657 | "ndarray": "1.0.18", 658 | "ndarray-pack": "1.2.1", 659 | "node-bitmap": "0.0.1", 660 | "omggif": "1.0.8", 661 | "parse-data-uri": "0.2.0", 662 | "pngjs": "2.3.1", 663 | "request": "2.83.0", 664 | "through": "2.3.8" 665 | } 666 | }, 667 | "get-stream": { 668 | "version": "3.0.0", 669 | "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", 670 | "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" 671 | }, 672 | "getpass": { 673 | "version": "0.1.7", 674 | "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", 675 | "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", 676 | "requires": { 677 | "assert-plus": "1.0.0" 678 | } 679 | }, 680 | "glob": { 681 | "version": "7.1.2", 682 | "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", 683 | "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", 684 | "requires": { 685 | "fs.realpath": "1.0.0", 686 | "inflight": "1.0.6", 687 | "inherits": "2.0.3", 688 | "minimatch": "3.0.4", 689 | "once": "1.4.0", 690 | "path-is-absolute": "1.0.1" 691 | } 692 | }, 693 | "har-schema": { 694 | "version": "2.0.0", 695 | "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", 696 | "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" 697 | }, 698 | "har-validator": { 699 | "version": "5.0.3", 700 | "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz", 701 | "integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=", 702 | "requires": { 703 | "ajv": "5.2.3", 704 | "har-schema": "2.0.0" 705 | } 706 | }, 707 | "has-ansi": { 708 | "version": "2.0.0", 709 | "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", 710 | "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", 711 | "requires": { 712 | "ansi-regex": "2.1.1" 713 | } 714 | }, 715 | "has-flag": { 716 | "version": "2.0.0", 717 | "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz", 718 | "integrity": "sha1-6CB68cx7MNRGzHC3NLXovhj4jVE=" 719 | }, 720 | "hawk": { 721 | "version": "6.0.2", 722 | "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz", 723 | "integrity": "sha512-miowhl2+U7Qle4vdLqDdPt9m09K6yZhkLDTWGoUiUzrQCn+mHHSmfJgAyGaLRZbPmTqfFFjRV1QWCW0VWUJBbQ==", 724 | "requires": { 725 | "boom": "4.3.1", 726 | "cryptiles": "3.1.2", 727 | "hoek": "4.2.0", 728 | "sntp": "2.0.2" 729 | } 730 | }, 731 | "hoek": { 732 | "version": "4.2.0", 733 | "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.0.tgz", 734 | "integrity": "sha512-v0XCLxICi9nPfYrS9RL8HbYnXi9obYAeLbSP00BmnZwCK9+Ih9WOjoZ8YoHCoav2csqn4FOz4Orldsy2dmDwmQ==" 735 | }, 736 | "http-signature": { 737 | "version": "1.2.0", 738 | "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", 739 | "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", 740 | "requires": { 741 | "assert-plus": "1.0.0", 742 | "jsprim": "1.4.1", 743 | "sshpk": "1.13.1" 744 | } 745 | }, 746 | "https-proxy-agent": { 747 | "version": "2.1.0", 748 | "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.1.0.tgz", 749 | "integrity": "sha512-/DTVSUCbRc6AiyOV4DBRvPDpKKCJh4qQJNaCgypX0T41quD9hp/PB5iUyx/60XobuMPQa9ce1jNV9UOUq6PnTg==", 750 | "requires": { 751 | "agent-base": "4.1.1", 752 | "debug": "2.6.9" 753 | } 754 | }, 755 | "iconv-lite": { 756 | "version": "0.4.19", 757 | "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", 758 | "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ==" 759 | }, 760 | "inflight": { 761 | "version": "1.0.6", 762 | "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", 763 | "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", 764 | "requires": { 765 | "once": "1.4.0", 766 | "wrappy": "1.0.2" 767 | } 768 | }, 769 | "inherits": { 770 | "version": "2.0.3", 771 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", 772 | "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" 773 | }, 774 | "inquirer": { 775 | "version": "3.3.0", 776 | "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-3.3.0.tgz", 777 | "integrity": "sha512-h+xtnyk4EwKvFWHrUYsWErEVR+igKtLdchu+o0Z1RL7VU/jVMFbYir2bp6bAj8efFNxWqHX0dIss6fJQ+/+qeQ==", 778 | "requires": { 779 | "ansi-escapes": "3.0.0", 780 | "chalk": "2.3.0", 781 | "cli-cursor": "2.1.0", 782 | "cli-width": "2.2.0", 783 | "external-editor": "2.0.5", 784 | "figures": "2.0.0", 785 | "lodash": "4.17.4", 786 | "mute-stream": "0.0.7", 787 | "run-async": "2.3.0", 788 | "rx-lite": "4.0.8", 789 | "rx-lite-aggregates": "4.0.8", 790 | "string-width": "2.1.1", 791 | "strip-ansi": "4.0.0", 792 | "through": "2.3.8" 793 | }, 794 | "dependencies": { 795 | "ansi-escapes": { 796 | "version": "3.0.0", 797 | "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.0.0.tgz", 798 | "integrity": "sha512-O/klc27mWNUigtv0F8NJWbLF00OcegQalkqKURWdosW08YZKi4m6CnSUSvIZG1otNJbTWhN01Hhz389DW7mvDQ==" 799 | }, 800 | "ansi-regex": { 801 | "version": "3.0.0", 802 | "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", 803 | "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" 804 | }, 805 | "ansi-styles": { 806 | "version": "3.2.0", 807 | "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.0.tgz", 808 | "integrity": "sha512-NnSOmMEYtVR2JVMIGTzynRkkaxtiq1xnFBcdQD/DnNCYPoEPsVJhM98BDyaoNOQIi7p4okdi3E27eN7GQbsUug==", 809 | "requires": { 810 | "color-convert": "1.9.0" 811 | } 812 | }, 813 | "chalk": { 814 | "version": "2.3.0", 815 | "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.0.tgz", 816 | "integrity": "sha512-Az5zJR2CBujap2rqXGaJKaPHyJ0IrUimvYNX+ncCy8PJP4ltOGTrHUIo097ZaL2zMeKYpiCdqDvS6zdrTFok3Q==", 817 | "requires": { 818 | "ansi-styles": "3.2.0", 819 | "escape-string-regexp": "1.0.5", 820 | "supports-color": "4.5.0" 821 | } 822 | }, 823 | "strip-ansi": { 824 | "version": "4.0.0", 825 | "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", 826 | "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", 827 | "requires": { 828 | "ansi-regex": "3.0.0" 829 | } 830 | }, 831 | "supports-color": { 832 | "version": "4.5.0", 833 | "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.5.0.tgz", 834 | "integrity": "sha1-vnoN5ITexcXN34s9WRJQRJEvY1s=", 835 | "requires": { 836 | "has-flag": "2.0.0" 837 | } 838 | } 839 | } 840 | }, 841 | "inquirer-autocomplete-prompt": { 842 | "version": "0.11.1", 843 | "resolved": "https://registry.npmjs.org/inquirer-autocomplete-prompt/-/inquirer-autocomplete-prompt-0.11.1.tgz", 844 | "integrity": "sha512-VM4eNiyRD4CeUc2cyKni+F8qgHwL9WC4LdOr+mEC85qP/QNsDV+ysVqUrJYhw1TmDQu1QVhc8hbaL7wfk8SJxw==", 845 | "requires": { 846 | "ansi-escapes": "2.0.0", 847 | "chalk": "1.1.3", 848 | "figures": "2.0.0", 849 | "inquirer": "3.1.1", 850 | "lodash": "4.17.4", 851 | "run-async": "2.3.0", 852 | "util": "0.10.3" 853 | }, 854 | "dependencies": { 855 | "inquirer": { 856 | "version": "3.1.1", 857 | "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-3.1.1.tgz", 858 | "integrity": "sha512-H50sHQwgvvaTBd3HpKMVtL/u6LoHDvYym51gd7bGQe/+9HkCE+J0/3N5FJLfd6O6oz44hHewC2Pc2LodzWVafQ==", 859 | "requires": { 860 | "ansi-escapes": "2.0.0", 861 | "chalk": "1.1.3", 862 | "cli-cursor": "2.1.0", 863 | "cli-width": "2.2.0", 864 | "external-editor": "2.0.5", 865 | "figures": "2.0.0", 866 | "lodash": "4.17.4", 867 | "mute-stream": "0.0.7", 868 | "run-async": "2.3.0", 869 | "rx-lite": "4.0.8", 870 | "rx-lite-aggregates": "4.0.8", 871 | "string-width": "2.1.1", 872 | "strip-ansi": "3.0.1", 873 | "through": "2.3.8" 874 | } 875 | } 876 | } 877 | }, 878 | "invert-kv": { 879 | "version": "1.0.0", 880 | "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", 881 | "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=" 882 | }, 883 | "iota-array": { 884 | "version": "1.0.0", 885 | "resolved": "https://registry.npmjs.org/iota-array/-/iota-array-1.0.0.tgz", 886 | "integrity": "sha1-ge9X/l0FgUzVjCSDYyqZwwoOgIc=" 887 | }, 888 | "is-buffer": { 889 | "version": "1.1.6", 890 | "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", 891 | "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" 892 | }, 893 | "is-fullwidth-code-point": { 894 | "version": "2.0.0", 895 | "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", 896 | "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" 897 | }, 898 | "is-promise": { 899 | "version": "2.1.0", 900 | "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", 901 | "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=" 902 | }, 903 | "is-stream": { 904 | "version": "1.1.0", 905 | "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", 906 | "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" 907 | }, 908 | "is-typedarray": { 909 | "version": "1.0.0", 910 | "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", 911 | "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" 912 | }, 913 | "isarray": { 914 | "version": "1.0.0", 915 | "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", 916 | "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" 917 | }, 918 | "isexe": { 919 | "version": "2.0.0", 920 | "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", 921 | "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=" 922 | }, 923 | "isstream": { 924 | "version": "0.1.2", 925 | "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", 926 | "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" 927 | }, 928 | "jpeg-js": { 929 | "version": "0.1.2", 930 | "resolved": "https://registry.npmjs.org/jpeg-js/-/jpeg-js-0.1.2.tgz", 931 | "integrity": "sha1-E1uZLAV1yYXPoPSUoyJ+0jhYPs4=" 932 | }, 933 | "jquery": { 934 | "version": "3.2.1", 935 | "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.2.1.tgz", 936 | "integrity": "sha1-XE2d5lKvbNCncBVKYxu6ErAVx4c=" 937 | }, 938 | "jsbn": { 939 | "version": "0.1.1", 940 | "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", 941 | "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", 942 | "optional": true 943 | }, 944 | "jschardet": { 945 | "version": "1.6.0", 946 | "resolved": "https://registry.npmjs.org/jschardet/-/jschardet-1.6.0.tgz", 947 | "integrity": "sha512-xYuhvQ7I9PDJIGBWev9xm0+SMSed3ZDBAmvVjbFR1ZRLAF+vlXcQu6cRI9uAlj81rzikElRVteehwV7DuX2ZmQ==" 948 | }, 949 | "json-schema": { 950 | "version": "0.2.3", 951 | "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", 952 | "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" 953 | }, 954 | "json-schema-traverse": { 955 | "version": "0.3.1", 956 | "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", 957 | "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=" 958 | }, 959 | "json-stable-stringify": { 960 | "version": "1.0.1", 961 | "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz", 962 | "integrity": "sha1-mnWdOcXy/1A/1TAGRu1EX4jE+a8=", 963 | "requires": { 964 | "jsonify": "0.0.0" 965 | } 966 | }, 967 | "json-stringify-safe": { 968 | "version": "5.0.1", 969 | "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", 970 | "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" 971 | }, 972 | "jsonify": { 973 | "version": "0.0.0", 974 | "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz", 975 | "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM=" 976 | }, 977 | "jsprim": { 978 | "version": "1.4.1", 979 | "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", 980 | "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", 981 | "requires": { 982 | "assert-plus": "1.0.0", 983 | "extsprintf": "1.3.0", 984 | "json-schema": "0.2.3", 985 | "verror": "1.10.0" 986 | } 987 | }, 988 | "keypress": { 989 | "version": "0.2.1", 990 | "resolved": "https://registry.npmjs.org/keypress/-/keypress-0.2.1.tgz", 991 | "integrity": "sha1-HoBFQlABjbrUw/6USX1uZ7YmnHc=" 992 | }, 993 | "lcid": { 994 | "version": "1.0.0", 995 | "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", 996 | "integrity": "sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU=", 997 | "requires": { 998 | "invert-kv": "1.0.0" 999 | } 1000 | }, 1001 | "locate-path": { 1002 | "version": "2.0.0", 1003 | "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", 1004 | "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", 1005 | "requires": { 1006 | "p-locate": "2.0.0", 1007 | "path-exists": "3.0.0" 1008 | } 1009 | }, 1010 | "lodash": { 1011 | "version": "4.17.4", 1012 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz", 1013 | "integrity": "sha1-eCA6TRwyiuHYbcpkYONptX9AVa4=" 1014 | }, 1015 | "lru-cache": { 1016 | "version": "4.1.1", 1017 | "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.1.tgz", 1018 | "integrity": "sha512-q4spe4KTfsAS1SUHLO0wz8Qiyf1+vMIAgpRYioFYDMNqKfHQbg+AVDH3i4fvpl71/P1L0dBl+fQi+P37UYf0ew==", 1019 | "requires": { 1020 | "pseudomap": "1.0.2", 1021 | "yallist": "2.1.2" 1022 | } 1023 | }, 1024 | "md5": { 1025 | "version": "2.2.1", 1026 | "resolved": "https://registry.npmjs.org/md5/-/md5-2.2.1.tgz", 1027 | "integrity": "sha1-U6s41f48iJG6RlMp6iP6wFQBJvk=", 1028 | "requires": { 1029 | "charenc": "0.0.2", 1030 | "crypt": "0.0.2", 1031 | "is-buffer": "1.1.6" 1032 | } 1033 | }, 1034 | "mem": { 1035 | "version": "1.1.0", 1036 | "resolved": "https://registry.npmjs.org/mem/-/mem-1.1.0.tgz", 1037 | "integrity": "sha1-Xt1StIXKHZAP5kiVUFOZoN+kX3Y=", 1038 | "requires": { 1039 | "mimic-fn": "1.1.0" 1040 | } 1041 | }, 1042 | "mime": { 1043 | "version": "1.4.1", 1044 | "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz", 1045 | "integrity": "sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ==" 1046 | }, 1047 | "mime-db": { 1048 | "version": "1.30.0", 1049 | "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.30.0.tgz", 1050 | "integrity": "sha1-dMZD2i3Z1qRTmZY0ZbJtXKfXHwE=" 1051 | }, 1052 | "mime-types": { 1053 | "version": "2.1.17", 1054 | "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.17.tgz", 1055 | "integrity": "sha1-Cdejk/A+mVp5+K+Fe3Cp4KsWVXo=", 1056 | "requires": { 1057 | "mime-db": "1.30.0" 1058 | } 1059 | }, 1060 | "mimic-fn": { 1061 | "version": "1.1.0", 1062 | "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.1.0.tgz", 1063 | "integrity": "sha1-5md4PZLonb00KBi1IwudYqZyrRg=" 1064 | }, 1065 | "minimatch": { 1066 | "version": "3.0.4", 1067 | "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", 1068 | "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", 1069 | "requires": { 1070 | "brace-expansion": "1.1.8" 1071 | } 1072 | }, 1073 | "minimist": { 1074 | "version": "0.0.8", 1075 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", 1076 | "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" 1077 | }, 1078 | "mkdirp": { 1079 | "version": "0.5.0", 1080 | "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.0.tgz", 1081 | "integrity": "sha1-HXMHam35hs2TROFecfzAWkyavxI=", 1082 | "requires": { 1083 | "minimist": "0.0.8" 1084 | } 1085 | }, 1086 | "moment": { 1087 | "version": "2.19.2", 1088 | "resolved": "https://registry.npmjs.org/moment/-/moment-2.19.2.tgz", 1089 | "integrity": "sha512-Rf6jiHPEfxp9+dlzxPTmRHbvoFXsh2L/U8hOupUMpnuecHQmI6cF6lUbJl3QqKPko1u6ujO+FxtcajLVfLpAtA==" 1090 | }, 1091 | "mongodb": { 1092 | "version": "2.2.33", 1093 | "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-2.2.33.tgz", 1094 | "integrity": "sha1-tTfEcdNKZlG0jzb9vyl1A0Dgi1A=", 1095 | "requires": { 1096 | "es6-promise": "3.2.1", 1097 | "mongodb-core": "2.1.17", 1098 | "readable-stream": "2.2.7" 1099 | }, 1100 | "dependencies": { 1101 | "es6-promise": { 1102 | "version": "3.2.1", 1103 | "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.2.1.tgz", 1104 | "integrity": "sha1-7FYjOGgDKQkgcXDDlEjiREndH8Q=" 1105 | }, 1106 | "readable-stream": { 1107 | "version": "2.2.7", 1108 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.2.7.tgz", 1109 | "integrity": "sha1-BwV6y+JGeyIELTb5jFrVBwVOlbE=", 1110 | "requires": { 1111 | "buffer-shims": "1.0.0", 1112 | "core-util-is": "1.0.2", 1113 | "inherits": "2.0.3", 1114 | "isarray": "1.0.0", 1115 | "process-nextick-args": "1.0.7", 1116 | "string_decoder": "1.0.3", 1117 | "util-deprecate": "1.0.2" 1118 | } 1119 | } 1120 | } 1121 | }, 1122 | "mongodb-core": { 1123 | "version": "2.1.17", 1124 | "resolved": "https://registry.npmjs.org/mongodb-core/-/mongodb-core-2.1.17.tgz", 1125 | "integrity": "sha1-pBizN6FKFJkPtRC5I97mqBMXPfg=", 1126 | "requires": { 1127 | "bson": "1.0.4", 1128 | "require_optional": "1.0.1" 1129 | } 1130 | }, 1131 | "ms": { 1132 | "version": "2.0.0", 1133 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", 1134 | "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" 1135 | }, 1136 | "mute-stream": { 1137 | "version": "0.0.7", 1138 | "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", 1139 | "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=" 1140 | }, 1141 | "mysql": { 1142 | "version": "2.15.0", 1143 | "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.15.0.tgz", 1144 | "integrity": "sha512-C7tjzWtbN5nzkLIV+E8Crnl9bFyc7d3XJcBAvHKEVkjrYjogz3llo22q6s/hw+UcsE4/844pDob9ac+3dVjQSA==", 1145 | "requires": { 1146 | "bignumber.js": "4.0.4", 1147 | "readable-stream": "2.3.3", 1148 | "safe-buffer": "5.1.1", 1149 | "sqlstring": "2.3.0" 1150 | } 1151 | }, 1152 | "nano": { 1153 | "version": "6.4.2", 1154 | "resolved": "https://registry.npmjs.org/nano/-/nano-6.4.2.tgz", 1155 | "integrity": "sha1-ubD6geuRS4ucY0odVeh44PRbCb8=", 1156 | "requires": { 1157 | "cloudant-follow": "0.13.0", 1158 | "debug": "2.6.9", 1159 | "errs": "0.3.2", 1160 | "request": "2.83.0", 1161 | "underscore": "1.8.3" 1162 | } 1163 | }, 1164 | "ndarray": { 1165 | "version": "1.0.18", 1166 | "resolved": "https://registry.npmjs.org/ndarray/-/ndarray-1.0.18.tgz", 1167 | "integrity": "sha1-tg06cyJOxVXQ+qeXEeUCRI/T95M=", 1168 | "requires": { 1169 | "iota-array": "1.0.0", 1170 | "is-buffer": "1.1.6" 1171 | } 1172 | }, 1173 | "ndarray-pack": { 1174 | "version": "1.2.1", 1175 | "resolved": "https://registry.npmjs.org/ndarray-pack/-/ndarray-pack-1.2.1.tgz", 1176 | "integrity": "sha1-jK6+qqJNXs9w/4YCBjeXfajuWFo=", 1177 | "requires": { 1178 | "cwise-compiler": "1.1.3", 1179 | "ndarray": "1.0.18" 1180 | } 1181 | }, 1182 | "nextgen-events": { 1183 | "version": "0.10.2", 1184 | "resolved": "https://registry.npmjs.org/nextgen-events/-/nextgen-events-0.10.2.tgz", 1185 | "integrity": "sha512-P6efDoVOOJjVLOhwINq+aqhC2B3a9IxojbWMn9fTv2coDiyRaaGLSgWsm84wQHlQeuqVgqiLEE+xiHXf3EVN6w==" 1186 | }, 1187 | "node-bitmap": { 1188 | "version": "0.0.1", 1189 | "resolved": "https://registry.npmjs.org/node-bitmap/-/node-bitmap-0.0.1.tgz", 1190 | "integrity": "sha1-GA6scAPgxwdhjvMTaPYvhLKmkJE=" 1191 | }, 1192 | "node-couchdb": { 1193 | "version": "1.2.0", 1194 | "resolved": "https://registry.npmjs.org/node-couchdb/-/node-couchdb-1.2.0.tgz", 1195 | "integrity": "sha1-6APixyNiF9CR3jfeIW2CgYOfCnw=", 1196 | "requires": { 1197 | "request": "2.83.0" 1198 | } 1199 | }, 1200 | "npm-run-path": { 1201 | "version": "2.0.2", 1202 | "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", 1203 | "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=", 1204 | "requires": { 1205 | "path-key": "2.0.1" 1206 | } 1207 | }, 1208 | "number-is-nan": { 1209 | "version": "1.0.1", 1210 | "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", 1211 | "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" 1212 | }, 1213 | "oauth-sign": { 1214 | "version": "0.8.2", 1215 | "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", 1216 | "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=" 1217 | }, 1218 | "omggif": { 1219 | "version": "1.0.8", 1220 | "resolved": "https://registry.npmjs.org/omggif/-/omggif-1.0.8.tgz", 1221 | "integrity": "sha1-F483sqsLPXtG7ToORr0HkLWNNTA=" 1222 | }, 1223 | "once": { 1224 | "version": "1.4.0", 1225 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", 1226 | "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", 1227 | "requires": { 1228 | "wrappy": "1.0.2" 1229 | } 1230 | }, 1231 | "onetime": { 1232 | "version": "2.0.1", 1233 | "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", 1234 | "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", 1235 | "requires": { 1236 | "mimic-fn": "1.1.0" 1237 | } 1238 | }, 1239 | "os-locale": { 1240 | "version": "2.1.0", 1241 | "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-2.1.0.tgz", 1242 | "integrity": "sha512-3sslG3zJbEYcaC4YVAvDorjGxc7tv6KVATnLPZONiljsUncvihe9BQoVCEs0RZ1kmf4Hk9OBqlZfJZWI4GanKA==", 1243 | "requires": { 1244 | "execa": "0.7.0", 1245 | "lcid": "1.0.0", 1246 | "mem": "1.1.0" 1247 | } 1248 | }, 1249 | "os-tmpdir": { 1250 | "version": "1.0.2", 1251 | "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", 1252 | "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=" 1253 | }, 1254 | "p-finally": { 1255 | "version": "1.0.0", 1256 | "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", 1257 | "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" 1258 | }, 1259 | "p-limit": { 1260 | "version": "1.1.0", 1261 | "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.1.0.tgz", 1262 | "integrity": "sha1-sH/y2aXYi+yAYDWJWiurZqJ5iLw=" 1263 | }, 1264 | "p-locate": { 1265 | "version": "2.0.0", 1266 | "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", 1267 | "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", 1268 | "requires": { 1269 | "p-limit": "1.1.0" 1270 | } 1271 | }, 1272 | "parse-data-uri": { 1273 | "version": "0.2.0", 1274 | "resolved": "https://registry.npmjs.org/parse-data-uri/-/parse-data-uri-0.2.0.tgz", 1275 | "integrity": "sha1-vwTYUd1ch7CrI45dAazklLYEtMk=", 1276 | "requires": { 1277 | "data-uri-to-buffer": "0.0.3" 1278 | } 1279 | }, 1280 | "path-exists": { 1281 | "version": "3.0.0", 1282 | "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", 1283 | "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=" 1284 | }, 1285 | "path-is-absolute": { 1286 | "version": "1.0.1", 1287 | "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", 1288 | "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" 1289 | }, 1290 | "path-key": { 1291 | "version": "2.0.1", 1292 | "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", 1293 | "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=" 1294 | }, 1295 | "pend": { 1296 | "version": "1.2.0", 1297 | "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", 1298 | "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" 1299 | }, 1300 | "performance-now": { 1301 | "version": "2.1.0", 1302 | "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", 1303 | "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" 1304 | }, 1305 | "pngjs": { 1306 | "version": "2.3.1", 1307 | "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-2.3.1.tgz", 1308 | "integrity": "sha1-EdHhK5y2TWPjDBQ6Mw9MH1Z9qF8=" 1309 | }, 1310 | "process-nextick-args": { 1311 | "version": "1.0.7", 1312 | "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", 1313 | "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" 1314 | }, 1315 | "progress": { 1316 | "version": "2.0.0", 1317 | "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.0.tgz", 1318 | "integrity": "sha1-ihvjZr+Pwj2yvSPxDG/pILQ4nR8=" 1319 | }, 1320 | "promise-mysql": { 1321 | "version": "3.1.3", 1322 | "resolved": "https://registry.npmjs.org/promise-mysql/-/promise-mysql-3.1.3.tgz", 1323 | "integrity": "sha512-rtFSHKSLiT6x9MC2V1TXhXQw1++QY0O9t+iZIFf6tPcMF06EgUYlw7PD2aqZK6PxF4NyNrcUsqoJIRLRoCsxhQ==", 1324 | "requires": { 1325 | "@types/mysql": "2.15.1", 1326 | "bluebird": "3.5.1", 1327 | "mysql": "2.15.0" 1328 | } 1329 | }, 1330 | "proxy-from-env": { 1331 | "version": "1.0.0", 1332 | "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz", 1333 | "integrity": "sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4=" 1334 | }, 1335 | "pseudomap": { 1336 | "version": "1.0.2", 1337 | "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", 1338 | "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=" 1339 | }, 1340 | "punycode": { 1341 | "version": "1.4.1", 1342 | "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", 1343 | "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" 1344 | }, 1345 | "puppeteer": { 1346 | "version": "0.11.0", 1347 | "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-0.11.0.tgz", 1348 | "integrity": "sha512-/o8uY6VChG74B1HYBZkLDU6aIekWeOd85wez9YxB9SNbDnFNsUddv2F4xWuhwvQ4ab84vb+1VX4fxs2kBz3u8g==", 1349 | "requires": { 1350 | "debug": "2.6.9", 1351 | "extract-zip": "1.6.5", 1352 | "https-proxy-agent": "2.1.0", 1353 | "mime": "1.4.1", 1354 | "progress": "2.0.0", 1355 | "proxy-from-env": "1.0.0", 1356 | "rimraf": "2.6.2", 1357 | "ws": "3.2.0" 1358 | } 1359 | }, 1360 | "qs": { 1361 | "version": "6.5.1", 1362 | "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", 1363 | "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==" 1364 | }, 1365 | "readable-stream": { 1366 | "version": "2.3.3", 1367 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz", 1368 | "integrity": "sha512-m+qzzcn7KUxEmd1gMbchF+Y2eIUbieUaxkWtptyHywrX0rE8QEYqPC07Vuy4Wm32/xE16NcdBctb8S0Xe/5IeQ==", 1369 | "requires": { 1370 | "core-util-is": "1.0.2", 1371 | "inherits": "2.0.3", 1372 | "isarray": "1.0.0", 1373 | "process-nextick-args": "1.0.7", 1374 | "safe-buffer": "5.1.1", 1375 | "string_decoder": "1.0.3", 1376 | "util-deprecate": "1.0.2" 1377 | } 1378 | }, 1379 | "request": { 1380 | "version": "2.83.0", 1381 | "resolved": "https://registry.npmjs.org/request/-/request-2.83.0.tgz", 1382 | "integrity": "sha512-lR3gD69osqm6EYLk9wB/G1W/laGWjzH90t1vEa2xuxHD5KUrSzp9pUSfTm+YC5Nxt2T8nMPEvKlhbQayU7bgFw==", 1383 | "requires": { 1384 | "aws-sign2": "0.7.0", 1385 | "aws4": "1.6.0", 1386 | "caseless": "0.12.0", 1387 | "combined-stream": "1.0.5", 1388 | "extend": "3.0.1", 1389 | "forever-agent": "0.6.1", 1390 | "form-data": "2.3.1", 1391 | "har-validator": "5.0.3", 1392 | "hawk": "6.0.2", 1393 | "http-signature": "1.2.0", 1394 | "is-typedarray": "1.0.0", 1395 | "isstream": "0.1.2", 1396 | "json-stringify-safe": "5.0.1", 1397 | "mime-types": "2.1.17", 1398 | "oauth-sign": "0.8.2", 1399 | "performance-now": "2.1.0", 1400 | "qs": "6.5.1", 1401 | "safe-buffer": "5.1.1", 1402 | "stringstream": "0.0.5", 1403 | "tough-cookie": "2.3.3", 1404 | "tunnel-agent": "0.6.0", 1405 | "uuid": "3.1.0" 1406 | } 1407 | }, 1408 | "require-directory": { 1409 | "version": "2.1.1", 1410 | "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", 1411 | "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" 1412 | }, 1413 | "require-main-filename": { 1414 | "version": "1.0.1", 1415 | "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz", 1416 | "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE=" 1417 | }, 1418 | "require_optional": { 1419 | "version": "1.0.1", 1420 | "resolved": "https://registry.npmjs.org/require_optional/-/require_optional-1.0.1.tgz", 1421 | "integrity": "sha512-qhM/y57enGWHAe3v/NcwML6a3/vfESLe/sGM2dII+gEO0BpKRUkWZow/tyloNqJyN6kXSl3RyyM8Ll5D/sJP8g==", 1422 | "requires": { 1423 | "resolve-from": "2.0.0", 1424 | "semver": "5.4.1" 1425 | } 1426 | }, 1427 | "resolve-from": { 1428 | "version": "2.0.0", 1429 | "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-2.0.0.tgz", 1430 | "integrity": "sha1-lICrIOlP+h2egKgEx+oUdhGWa1c=" 1431 | }, 1432 | "restore-cursor": { 1433 | "version": "2.0.0", 1434 | "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", 1435 | "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", 1436 | "requires": { 1437 | "onetime": "2.0.1", 1438 | "signal-exit": "3.0.2" 1439 | } 1440 | }, 1441 | "rimraf": { 1442 | "version": "2.6.2", 1443 | "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz", 1444 | "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==", 1445 | "requires": { 1446 | "glob": "7.1.2" 1447 | } 1448 | }, 1449 | "run-async": { 1450 | "version": "2.3.0", 1451 | "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.3.0.tgz", 1452 | "integrity": "sha1-A3GrSuC91yDUFm19/aZP96RFpsA=", 1453 | "requires": { 1454 | "is-promise": "2.1.0" 1455 | } 1456 | }, 1457 | "rx-lite": { 1458 | "version": "4.0.8", 1459 | "resolved": "https://registry.npmjs.org/rx-lite/-/rx-lite-4.0.8.tgz", 1460 | "integrity": "sha1-Cx4Rr4vESDbwSmQH6S2kJGe3lEQ=" 1461 | }, 1462 | "rx-lite-aggregates": { 1463 | "version": "4.0.8", 1464 | "resolved": "https://registry.npmjs.org/rx-lite-aggregates/-/rx-lite-aggregates-4.0.8.tgz", 1465 | "integrity": "sha1-dTuHqJoRyVRnxKwWJsTvxOBcZ74=", 1466 | "requires": { 1467 | "rx-lite": "4.0.8" 1468 | } 1469 | }, 1470 | "safe-buffer": { 1471 | "version": "5.1.1", 1472 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", 1473 | "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" 1474 | }, 1475 | "sanitize-filename": { 1476 | "version": "1.6.1", 1477 | "resolved": "https://registry.npmjs.org/sanitize-filename/-/sanitize-filename-1.6.1.tgz", 1478 | "integrity": "sha1-YS2hyWRz+gLczaktzVtKsWSmdyo=", 1479 | "requires": { 1480 | "truncate-utf8-bytes": "1.0.2" 1481 | } 1482 | }, 1483 | "semver": { 1484 | "version": "5.4.1", 1485 | "resolved": "https://registry.npmjs.org/semver/-/semver-5.4.1.tgz", 1486 | "integrity": "sha512-WfG/X9+oATh81XtllIo/I8gOiY9EXRdv1cQdyykeXK17YcUW3EXUAi2To4pcH6nZtJPr7ZOpM5OMyWJZm+8Rsg==" 1487 | }, 1488 | "set-blocking": { 1489 | "version": "2.0.0", 1490 | "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", 1491 | "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" 1492 | }, 1493 | "shebang-command": { 1494 | "version": "1.2.0", 1495 | "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", 1496 | "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", 1497 | "requires": { 1498 | "shebang-regex": "1.0.0" 1499 | } 1500 | }, 1501 | "shebang-regex": { 1502 | "version": "1.0.0", 1503 | "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", 1504 | "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=" 1505 | }, 1506 | "signal-exit": { 1507 | "version": "3.0.2", 1508 | "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", 1509 | "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" 1510 | }, 1511 | "sntp": { 1512 | "version": "2.0.2", 1513 | "resolved": "https://registry.npmjs.org/sntp/-/sntp-2.0.2.tgz", 1514 | "integrity": "sha1-UGQRDwr4X3z9t9a2ekACjOUrSys=", 1515 | "requires": { 1516 | "hoek": "4.2.0" 1517 | } 1518 | }, 1519 | "sqlstring": { 1520 | "version": "2.3.0", 1521 | "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.0.tgz", 1522 | "integrity": "sha1-UluKT9Jtb3GqYegipsr5dtMa0qg=" 1523 | }, 1524 | "sshpk": { 1525 | "version": "1.13.1", 1526 | "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz", 1527 | "integrity": "sha1-US322mKHFEMW3EwY/hzx2UBzm+M=", 1528 | "requires": { 1529 | "asn1": "0.2.3", 1530 | "assert-plus": "1.0.0", 1531 | "bcrypt-pbkdf": "1.0.1", 1532 | "dashdash": "1.14.1", 1533 | "ecc-jsbn": "0.1.1", 1534 | "getpass": "0.1.7", 1535 | "jsbn": "0.1.1", 1536 | "tweetnacl": "0.14.5" 1537 | } 1538 | }, 1539 | "string-kit": { 1540 | "version": "0.6.3", 1541 | "resolved": "https://registry.npmjs.org/string-kit/-/string-kit-0.6.3.tgz", 1542 | "integrity": "sha512-G2T92klsuE+S9mqdKQyWurFweNQV5X+FRzSKTqYHRdaVUN/4dL6urbYJJ+xb9ep/4XWm+4RNT8j3acncNhFRBg==", 1543 | "requires": { 1544 | "xregexp": "3.2.0" 1545 | } 1546 | }, 1547 | "string-width": { 1548 | "version": "2.1.1", 1549 | "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", 1550 | "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", 1551 | "requires": { 1552 | "is-fullwidth-code-point": "2.0.0", 1553 | "strip-ansi": "4.0.0" 1554 | }, 1555 | "dependencies": { 1556 | "ansi-regex": { 1557 | "version": "3.0.0", 1558 | "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", 1559 | "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" 1560 | }, 1561 | "strip-ansi": { 1562 | "version": "4.0.0", 1563 | "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", 1564 | "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", 1565 | "requires": { 1566 | "ansi-regex": "3.0.0" 1567 | } 1568 | } 1569 | } 1570 | }, 1571 | "string_decoder": { 1572 | "version": "1.0.3", 1573 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", 1574 | "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==", 1575 | "requires": { 1576 | "safe-buffer": "5.1.1" 1577 | } 1578 | }, 1579 | "stringstream": { 1580 | "version": "0.0.5", 1581 | "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", 1582 | "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=" 1583 | }, 1584 | "strip-ansi": { 1585 | "version": "3.0.1", 1586 | "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", 1587 | "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", 1588 | "requires": { 1589 | "ansi-regex": "2.1.1" 1590 | } 1591 | }, 1592 | "strip-eof": { 1593 | "version": "1.0.0", 1594 | "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", 1595 | "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=" 1596 | }, 1597 | "supports-color": { 1598 | "version": "2.0.0", 1599 | "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", 1600 | "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" 1601 | }, 1602 | "terminal-kit": { 1603 | "version": "1.14.1", 1604 | "resolved": "https://registry.npmjs.org/terminal-kit/-/terminal-kit-1.14.1.tgz", 1605 | "integrity": "sha512-XBHeDrw84ehTejUWSQi1bK98xVr+K154m9LKU0absFbKVO+Q4g2iXxRKMXWS4YfxlgxE4MWlUhemw32TOQK8/w==", 1606 | "requires": { 1607 | "async-kit": "2.2.3", 1608 | "get-pixels": "3.3.0", 1609 | "ndarray": "1.0.18", 1610 | "nextgen-events": "0.10.2", 1611 | "string-kit": "0.6.3", 1612 | "tree-kit": "0.5.26" 1613 | } 1614 | }, 1615 | "through": { 1616 | "version": "2.3.8", 1617 | "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", 1618 | "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" 1619 | }, 1620 | "tmp": { 1621 | "version": "0.0.33", 1622 | "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", 1623 | "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", 1624 | "requires": { 1625 | "os-tmpdir": "1.0.2" 1626 | } 1627 | }, 1628 | "tough-cookie": { 1629 | "version": "2.3.3", 1630 | "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.3.tgz", 1631 | "integrity": "sha1-C2GKVWW23qkL80JdBNVe3EdadWE=", 1632 | "requires": { 1633 | "punycode": "1.4.1" 1634 | } 1635 | }, 1636 | "tree-kit": { 1637 | "version": "0.5.26", 1638 | "resolved": "https://registry.npmjs.org/tree-kit/-/tree-kit-0.5.26.tgz", 1639 | "integrity": "sha1-hXHIb6JNHbdU5bDLOn4J9B50qN8=" 1640 | }, 1641 | "truncate-utf8-bytes": { 1642 | "version": "1.0.2", 1643 | "resolved": "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz", 1644 | "integrity": "sha1-QFkjkJWS1W94pYGENLC3hInKXys=", 1645 | "requires": { 1646 | "utf8-byte-length": "1.0.4" 1647 | } 1648 | }, 1649 | "tunnel-agent": { 1650 | "version": "0.6.0", 1651 | "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", 1652 | "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", 1653 | "requires": { 1654 | "safe-buffer": "5.1.1" 1655 | } 1656 | }, 1657 | "tweetnacl": { 1658 | "version": "0.14.5", 1659 | "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", 1660 | "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", 1661 | "optional": true 1662 | }, 1663 | "typedarray": { 1664 | "version": "0.0.6", 1665 | "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", 1666 | "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" 1667 | }, 1668 | "ultron": { 1669 | "version": "1.1.0", 1670 | "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.0.tgz", 1671 | "integrity": "sha1-sHoualQagV/Go0zNRTO67DB8qGQ=" 1672 | }, 1673 | "underscore": { 1674 | "version": "1.8.3", 1675 | "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz", 1676 | "integrity": "sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI=" 1677 | }, 1678 | "uniq": { 1679 | "version": "1.0.1", 1680 | "resolved": "https://registry.npmjs.org/uniq/-/uniq-1.0.1.tgz", 1681 | "integrity": "sha1-sxxa6CVIRKOoKBVBzisEuGWnNP8=" 1682 | }, 1683 | "utf8-byte-length": { 1684 | "version": "1.0.4", 1685 | "resolved": "https://registry.npmjs.org/utf8-byte-length/-/utf8-byte-length-1.0.4.tgz", 1686 | "integrity": "sha1-9F8VDExm7uloGGUFq5P8u4rWv2E=" 1687 | }, 1688 | "util": { 1689 | "version": "0.10.3", 1690 | "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", 1691 | "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", 1692 | "requires": { 1693 | "inherits": "2.0.1" 1694 | }, 1695 | "dependencies": { 1696 | "inherits": { 1697 | "version": "2.0.1", 1698 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz", 1699 | "integrity": "sha1-sX0I0ya0Qj5Wjv9xn5GwscvfafE=" 1700 | } 1701 | } 1702 | }, 1703 | "util-deprecate": { 1704 | "version": "1.0.2", 1705 | "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", 1706 | "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" 1707 | }, 1708 | "uuid": { 1709 | "version": "3.1.0", 1710 | "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", 1711 | "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==" 1712 | }, 1713 | "verror": { 1714 | "version": "1.10.0", 1715 | "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", 1716 | "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", 1717 | "requires": { 1718 | "assert-plus": "1.0.0", 1719 | "core-util-is": "1.0.2", 1720 | "extsprintf": "1.3.0" 1721 | } 1722 | }, 1723 | "which": { 1724 | "version": "1.3.0", 1725 | "resolved": "https://registry.npmjs.org/which/-/which-1.3.0.tgz", 1726 | "integrity": "sha512-xcJpopdamTuY5duC/KnTTNBraPK54YwpenP4lzxU8H91GudWpFv38u0CKjclE1Wi2EH2EDz5LRcHcKbCIzqGyg==", 1727 | "requires": { 1728 | "isexe": "2.0.0" 1729 | } 1730 | }, 1731 | "which-module": { 1732 | "version": "2.0.0", 1733 | "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", 1734 | "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=" 1735 | }, 1736 | "wrap-ansi": { 1737 | "version": "2.1.0", 1738 | "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", 1739 | "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", 1740 | "requires": { 1741 | "string-width": "1.0.2", 1742 | "strip-ansi": "3.0.1" 1743 | }, 1744 | "dependencies": { 1745 | "is-fullwidth-code-point": { 1746 | "version": "1.0.0", 1747 | "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", 1748 | "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", 1749 | "requires": { 1750 | "number-is-nan": "1.0.1" 1751 | } 1752 | }, 1753 | "string-width": { 1754 | "version": "1.0.2", 1755 | "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", 1756 | "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", 1757 | "requires": { 1758 | "code-point-at": "1.1.0", 1759 | "is-fullwidth-code-point": "1.0.0", 1760 | "strip-ansi": "3.0.1" 1761 | } 1762 | } 1763 | } 1764 | }, 1765 | "wrappy": { 1766 | "version": "1.0.2", 1767 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", 1768 | "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" 1769 | }, 1770 | "ws": { 1771 | "version": "3.2.0", 1772 | "resolved": "https://registry.npmjs.org/ws/-/ws-3.2.0.tgz", 1773 | "integrity": "sha512-hTS3mkXm/j85jTQOIcwVz3yK3up9xHgPtgEhDBOH3G18LDOZmSAG1omJeXejLKJakx+okv8vS1sopgs7rw0kVw==", 1774 | "requires": { 1775 | "async-limiter": "1.0.0", 1776 | "safe-buffer": "5.1.1", 1777 | "ultron": "1.1.0" 1778 | } 1779 | }, 1780 | "xregexp": { 1781 | "version": "3.2.0", 1782 | "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-3.2.0.tgz", 1783 | "integrity": "sha1-yzYBmHv+JpW1hAAMGPHEqMMih44=" 1784 | }, 1785 | "y18n": { 1786 | "version": "3.2.1", 1787 | "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz", 1788 | "integrity": "sha1-bRX7qITAhnnA136I53WegR4H+kE=" 1789 | }, 1790 | "yallist": { 1791 | "version": "2.1.2", 1792 | "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", 1793 | "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" 1794 | }, 1795 | "yargs": { 1796 | "version": "10.0.3", 1797 | "resolved": "https://registry.npmjs.org/yargs/-/yargs-10.0.3.tgz", 1798 | "integrity": "sha512-DqBpQ8NAUX4GyPP/ijDGHsJya4tYqLQrjPr95HNsr1YwL3+daCfvBwg7+gIC6IdJhR2kATh3hb61vjzMWEtjdw==", 1799 | "requires": { 1800 | "cliui": "3.2.0", 1801 | "decamelize": "1.2.0", 1802 | "find-up": "2.1.0", 1803 | "get-caller-file": "1.0.2", 1804 | "os-locale": "2.1.0", 1805 | "require-directory": "2.1.1", 1806 | "require-main-filename": "1.0.1", 1807 | "set-blocking": "2.0.0", 1808 | "string-width": "2.1.1", 1809 | "which-module": "2.0.0", 1810 | "y18n": "3.2.1", 1811 | "yargs-parser": "8.0.0" 1812 | } 1813 | }, 1814 | "yargs-parser": { 1815 | "version": "8.0.0", 1816 | "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-8.0.0.tgz", 1817 | "integrity": "sha1-IdR2Mw5agieaS4gTRb8GYQLiGcY=", 1818 | "requires": { 1819 | "camelcase": "4.1.0" 1820 | } 1821 | }, 1822 | "yauzl": { 1823 | "version": "2.4.1", 1824 | "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz", 1825 | "integrity": "sha1-lSj0QtqxsihOWLQ3m7GU4i4MQAU=", 1826 | "requires": { 1827 | "fd-slicer": "1.0.1" 1828 | } 1829 | } 1830 | } 1831 | } 1832 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "puppeteer-scraper", 3 | "version": "1.0.0", 4 | "description": "A Better Scraper, with Puppeteer", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "David 'oodavid' King", 10 | "license": "ISC", 11 | "keywords": [ 12 | "Puppeteer", 13 | "Chrome", 14 | "Headless", 15 | "Scrapping", 16 | "Node", 17 | "Distributed", 18 | "Proxy", 19 | "Proxies", 20 | "Throttle", 21 | "Throttling" 22 | ], 23 | "dependencies": { 24 | "bluebird": "^3.5.1", 25 | "dateformat": "^3.0.2", 26 | "fuzzy": "^0.1.3", 27 | "glob": "^7.1.2", 28 | "inquirer": "^3.3.0", 29 | "inquirer-autocomplete-prompt": "^0.11.1", 30 | "jquery": "^3.2.1", 31 | "keypress": "^0.2.1", 32 | "md5": "^2.2.1", 33 | "moment": "^2.19.2", 34 | "mongodb": "^2.2.33", 35 | "nano": "^6.4.2", 36 | "node-couchdb": "^1.2.0", 37 | "promise-mysql": "^3.1.3", 38 | "puppeteer": "^0.11.0", 39 | "sanitize-filename": "^1.6.1", 40 | "terminal-kit": "^1.14.1", 41 | "yargs": "^10.0.3" 42 | } 43 | } 44 | --------------------------------------------------------------------------------