├── .gitignore ├── src ├── index.js ├── config │ ├── defaultConfig.js │ ├── events.js │ └── selectors.js ├── listeners │ └── LoggerListener.js ├── transformers │ └── parseSERP.js ├── providers │ └── providers.js ├── services │ └── ScraperService.js └── repositories │ └── scraperRepository.js ├── package.json ├── README.md └── yarn.lock /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | const { scraperService } = require('./providers/providers') 2 | 3 | module.exports = scraperService; 4 | -------------------------------------------------------------------------------- /src/config/defaultConfig.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | headers: { 3 | 'User-Agent': 'Node SEO Scraper 1.0.0' 4 | } 5 | } -------------------------------------------------------------------------------- /src/config/events.js: -------------------------------------------------------------------------------- 1 | const events = [ 2 | 'ScraperRepository:fetchInit', 3 | 'ScraperRepository:fetchSuccess', 4 | 'ScraperRepository:fetchError', 5 | 'ScraperRepository:parseInit', 6 | 'ScraperRepository:parseSuccess', 7 | 'ScraperRepository:parseError', 8 | 'ScraperService:scrapeError' 9 | ]; 10 | 11 | module.exports = { 12 | events 13 | } 14 | -------------------------------------------------------------------------------- /src/listeners/LoggerListener.js: -------------------------------------------------------------------------------- 1 | class LoggerListener { 2 | constructor(eventEmitter, events){ 3 | this.eventEmitter = eventEmitter; 4 | this.events = events; 5 | } 6 | 7 | listen() { 8 | this.events.forEach(event => this.eventEmitter.on(event, this._log)) 9 | } 10 | 11 | _log(msg, args) { 12 | console.log(msg); 13 | if(args) { 14 | console.log(`\n${args}`); 15 | } 16 | } 17 | } 18 | 19 | module.exports = { 20 | LoggerListener 21 | }; -------------------------------------------------------------------------------- /src/config/selectors.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | content: { 3 | robots: "meta[name='robots']", 4 | description: "meta[name='description']", 5 | }, 6 | textContent: { 7 | title: "title", 8 | h1: "h1", 9 | h2: "h2", 10 | h3: "h3", 11 | h4: "h4", 12 | h5: "h5", 13 | h6: "h6" 14 | }, 15 | href: { 16 | links: "a", 17 | canonical: "link[rel='canonical']", 18 | alternateMobile: "link[media='only screen and (max-width: 640px)']", 19 | prevPagination: "link[rel='prev']", 20 | nextPagination: "link[rel='next']", 21 | amp: "link[rel='amphtml']" 22 | }, 23 | outerHTML: { 24 | hreflang: "link[hreflang]" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "seo-scraper", 3 | "version": "2.0.2", 4 | "description": "Scrape SEO elements or whatever you need with this scraper built in Node.js", 5 | "main": "src/index.js", 6 | "author": "Nacho Mascort ", 7 | "license": "MIT", 8 | "keywords": [ 9 | "SEO", 10 | "scraper", 11 | "SERP", 12 | "metadata", 13 | "google", 14 | "meta scraper" 15 | ], 16 | "scripts": { 17 | "test": "echo \"No test specified\"" 18 | }, 19 | "repository": { 20 | "type": "git", 21 | "url": "https://github.com/NachoSEO/seo-scraper.git" 22 | }, 23 | "dependencies": { 24 | "axios": "^0.21.1", 25 | "jsdom": "^16.4.0" 26 | }, 27 | "devDependencies": {} 28 | } 29 | -------------------------------------------------------------------------------- /src/transformers/parseSERP.js: -------------------------------------------------------------------------------- 1 | module.exports = function parseSERP(document, selectors, ...args) { 2 | const scrapedArr = [] 3 | const { parent, elements } = selectors; 4 | 5 | document 6 | .querySelectorAll(parent) 7 | .forEach(node => { 8 | const scrapedElement = {} 9 | for (const [method, value] of Object.entries(elements)) { 10 | for (const [selectorName, selector] of Object.entries(value)) { 11 | try { 12 | scrapedElement[selectorName] = node.querySelector(selector)[method]; 13 | } catch (err) { 14 | console.error(err); 15 | scrapedElement[selectorName] = ''; 16 | } 17 | } 18 | } 19 | scrapedArr.push(scrapedElement); 20 | }) 21 | 22 | return scrapedArr; 23 | } -------------------------------------------------------------------------------- /src/providers/providers.js: -------------------------------------------------------------------------------- 1 | const axios = require('axios').default; 2 | const jsdom = require("jsdom"); 3 | const EventEmitter = require('events'); 4 | 5 | const defaultSelectors = require('../config/selectors'); 6 | const defaultConfig = require('../config/defaultConfig'); 7 | const { events } = require('../config/events'); 8 | 9 | const { ScraperRepository } = require('../repositories/scraperRepository'); 10 | 11 | const { ScraperService } = require('../services/ScraperService'); 12 | 13 | const { LoggerListener } = require('../listeners/LoggerListener'); 14 | 15 | const { JSDOM } = jsdom; 16 | const eventEmitter = new EventEmitter(); 17 | 18 | const scraperService = new ScraperService( 19 | new ScraperRepository( 20 | axios, 21 | JSDOM, 22 | eventEmitter 23 | ), 24 | defaultSelectors, 25 | defaultConfig, 26 | eventEmitter 27 | ); 28 | 29 | 30 | const loggerListener = new LoggerListener( 31 | eventEmitter, 32 | events 33 | ); 34 | 35 | loggerListener.listen(); 36 | 37 | module.exports = { 38 | scraperService 39 | } -------------------------------------------------------------------------------- /src/services/ScraperService.js: -------------------------------------------------------------------------------- 1 | class ScraperService { 2 | constructor(scraperRepository, defaultSelectors, defaultConfig, eventEmitter) { 3 | this.scraperRepository = scraperRepository; 4 | this.defaultSelectors = defaultSelectors; 5 | this.defaultConfig = defaultConfig; 6 | this.eventEmitter = eventEmitter; 7 | } 8 | 9 | /** 10 | * Scrape SEO elements or whatever you need from an URL 11 | * @param {String} url - URL to scrape 12 | * @param {Object} selectors - CSS selectors to scrape 13 | * @param {Object} config - Request's configuration 14 | * @param {Object} proxies - Proxies to do the requests 15 | * @param {Function} transformer - Transformer Function to change the parse function 16 | * @param {Array} args - The rest of arguments that you need for the transformer function 17 | * @return {Object} - The scraped data 18 | */ 19 | async scrape({ url, selectors = this.defaultSelectors, config = this.defaultConfig, proxies = null, transformer = null }, ...args) { 20 | 21 | if (proxies) { 22 | const [host, port, username, password] = this.scraperRepository.getRandomProxy(proxies); 23 | config.proxy = { 24 | host, 25 | port: parseInt(port), 26 | auth: { 27 | username, 28 | password 29 | } 30 | } 31 | } 32 | 33 | try { 34 | const dom = await this.scraperRepository.fetch(url, config); 35 | const { document } = dom.window; 36 | return this.scraperRepository.parse(document, selectors, transformer, ...args); 37 | } catch (error) { 38 | this.eventEmitter.emit( 39 | 'ScraperService:scrapeError', 40 | `Error scraping ${url} 41 | ${error.stack}` 42 | ); 43 | } 44 | } 45 | } 46 | 47 | module.exports = { 48 | ScraperService 49 | } -------------------------------------------------------------------------------- /src/repositories/scraperRepository.js: -------------------------------------------------------------------------------- 1 | class ScraperRepository { 2 | constructor(axios, JSDOM, eventEmitter) { 3 | this.axios = axios; 4 | this.JSDOM = JSDOM; 5 | this.eventEmitter = eventEmitter; 6 | } 7 | 8 | /** 9 | * Fetch a page and return the document using a GET request 10 | * @param {String} url - The url to scrape 11 | * @param {Object} config - The configuration object that axios will use 12 | * @return {Promise} - A promise with a JSDOM object 13 | */ 14 | async fetch(url, config = {}) { 15 | this.eventEmitter.emit( 16 | 'ScraperRepository:fetchInit', 17 | `Init fetching ${url} with config:\n ${JSON.stringify(config)}` 18 | ); 19 | 20 | try { 21 | const response = await this.axios.get(url, config); 22 | this.eventEmitter.emit( 23 | 'ScraperRepository:fetchSuccess', 24 | `Success fetching ${url}` 25 | ); 26 | 27 | return new this.JSDOM(response.data); 28 | } catch (error) { 29 | this.eventEmitter.emit( 30 | 'ScraperRepository:fetchError', 31 | `Error fetching ${url} 32 | ${error.stack}` 33 | ); 34 | 35 | return null; 36 | } 37 | } 38 | 39 | /** 40 | * Parse a page and return the data selected using CSS selectors 41 | * @param {String} document - The document to be parsed 42 | * @param {String} selector - The CSS selector to be used 43 | * @param {Function} transformer - A callback function to be 44 | * called in case that you need to change the default parsing behaviour 45 | * @param {args} args - The arguments to be passed to the callback function 46 | * @return {Object} - An Object with the elements selected 47 | */ 48 | parse(document, selectors, transformer, ...args) { 49 | this.eventEmitter.emit( 50 | 'ScraperRepository:parseInit', 51 | `Init parsing` 52 | ); 53 | 54 | if(transformer) { 55 | return transformer(document, selectors, ...args); 56 | } 57 | 58 | const elements = {}; 59 | 60 | for (const [method, value] of Object.entries(selectors)) { 61 | for (const [selectorName, selector] of Object.entries(value)){ 62 | const newArr = []; 63 | try { 64 | const nodeElements = document.querySelectorAll(`${selector}`); 65 | nodeElements.forEach(node => { 66 | newArr.push(node[method]); 67 | }) 68 | } catch(error) { 69 | this.eventEmitter.emit( 70 | 'ScraperRepository:parseError', 71 | `Error parsing ${url} 72 | ${error.stack}` 73 | ); 74 | } 75 | elements[selectorName] = newArr; 76 | } 77 | } 78 | this.eventEmitter.emit( 79 | 'ScraperRepository:parseSuccess', 80 | `Parsing succed` 81 | ); 82 | 83 | return elements; 84 | } 85 | 86 | /** 87 | * Function to get a random proxy in every request 88 | * @param {Array} - An array of proxies to be used 89 | * @return {Array} - An array with the proxy information to be destructured 90 | */ 91 | getRandomProxy(proxies) { 92 | const randomNumber = Math.floor(Math.random() * proxies.whitelist.length); 93 | const [ host, port ] = proxies.whitelist[randomNumber].split(':'); 94 | const { username, password } = proxies.credentials; 95 | return [ host, port, username, password ]; 96 | } 97 | } 98 | 99 | module.exports = { 100 | ScraperRepository 101 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NodeJS SEO Scraper 2 | Scrape SEO elements by default or whatever you need with easy customizations with this Web Scraper built in Node.js 3 | 4 | ## Installing 5 | npm 6 | ``` 7 | npm install seo-scraper 8 | ``` 9 | 10 | yarn 11 | ``` 12 | yarn add seo-scraper 13 | ``` 14 | 15 | 16 | 17 | ## Get started with default config 18 | The default config extract the usual SEO elements without proxies: 19 | * title 20 | * meta description 21 | * meta robots 22 | * canonical 23 | * hreflang 24 | * alternate mobile 25 | * rel next/prev (although is deprecated in some Search Engines) 26 | * amp links 27 | * h1 - h6 28 | * links 29 | 30 | 31 | ### Usage 32 | The process is as easy as use the `scrape` function with an object with a url property as argument/input: 33 | ```js 34 | const scraper = require('seo-scraper'); 35 | 36 | scraper.scrape({ url: 'https://github.com/NachoSEO/seo-scraper' }) 37 | .then(elements => console.log(elements)) 38 | ``` 39 | 40 | The output of that execution will be: 41 | ```js 42 | ` 43 | { 44 | robots: [], 45 | description: [ 46 | 'Scrape SEO elements or whatever you need with this scraper built in Node.js - GitHub - NachoSEO/seo-scraper: Scrape SEO elements or whatever you need with this scraper built in Node.js' 47 | ], 48 | title: [ 49 | 'GitHub - NachoSEO/seo-scraper: Scrape SEO elements or whatever you need with this scraper built in Node.js' 50 | ], 51 | h1: [ 52 | '\n \n \n NachoSEO\n \n /\n \n seo-scraper\n \n \n', 53 | 'NodeJS SEO Scraper' 54 | ], 55 | h2: [ 56 | 'Latest commit', 57 | 'Git stats', 58 | 'Files', 59 | '\n README.md\n ', 60 | 'Installing', 61 | 'Get started with default config', 62 | 'Custom Configuration', 63 | 'Custom scraping example', 64 | 'Transformers', 65 | 'About', 66 | '\n \n Releases\n 4\n', 67 | '\n \n Packages 0\n', 68 | '\n \n Contributors 2\n', 69 | 'Languages' 70 | ], 71 | h3: [ 72 | 'Usage', 73 | 'Fetch', 74 | 'Parsing', 75 | 'Proxies - Optional', 76 | 'Example using a Transformer function', 77 | 'Topics', 78 | 'Resources' 79 | ], 80 | h4: [ 81 | 'Learn and contribute', 82 | 'Connect with others', 83 | 'Launching GitHub Desktop', 84 | 'Launching GitHub Desktop', 85 | 'Launching Xcode', 86 | 'Launching Visual Studio Code' 87 | ], 88 | h5: [], 89 | h6: [], 90 | links: [ 91 | 'about:blank#start-of-content', 92 | 'https://github.com/', 93 | '/signup?ref_cta=Sign+up&ref_loc=header+logged+out&ref_page=%2F%3Cuser-name%3E%2F%3Crepo-name%3E&source=header-repo', 94 | '/features', 95 | '/mobile', 96 | '/features/actions', 97 | '/features/codespaces', 98 | '/features/packages', 99 | '/features/security', 100 | '/features/code-review/', 101 | '/features/issues/', 102 | '/features/integrations', 103 | '/sponsors', 104 | '/customer-stories', 105 | '/team', 106 | '/enterprise', 107 | '/explore', 108 | '/topics', 109 | '/collections', 110 | '/trending', 111 | 'https://lab.github.com/', 112 | 'https://opensource.guide/', 113 | 'https://github.com/readme', 114 | 'https://github.com/events', 115 | 'https://github.community/', 116 | 'https://education.github.com/', 117 | 'https://stars.github.com/', 118 | '/marketplace', 119 | '/pricing', 120 | '/pricing#feature-comparison', 121 | 'https://enterprise.github.com/contact', 122 | 'https://education.github.com/', 123 | '', 124 | '', 125 | '', 126 | '', 127 | '/login?return_to=%2FNachoSEO%2Fseo-scraper', 128 | '/signup?ref_cta=Sign+up&ref_loc=header+logged+out&ref_page=%2F%3Cuser-name%3E%2F%3Crepo-name%3E&source=header-repo&source_repo=NachoSEO%2Fseo-scraper', 129 | '/NachoSEO', 130 | '/NachoSEO/seo-scraper', 131 | '/login?return_to=%2FNachoSEO%2Fseo-scraper', 132 | '/login?return_to=%2FNachoSEO%2Fseo-scraper', 133 | '/NachoSEO/seo-scraper/stargazers', 134 | '/login?return_to=%2FNachoSEO%2Fseo-scraper', 135 | '/NachoSEO/seo-scraper/network/members', 136 | '/NachoSEO/seo-scraper/stargazers', 137 | '/NachoSEO/seo-scraper/network/members', 138 | '/login?return_to=%2FNachoSEO%2Fseo-scraper', 139 | '/login?return_to=%2FNachoSEO%2Fseo-scraper', 140 | '/NachoSEO/seo-scraper', 141 | '/NachoSEO/seo-scraper/issues', 142 | '/NachoSEO/seo-scraper/pulls', 143 | '/NachoSEO/seo-scraper/actions', 144 | '/NachoSEO/seo-scraper/projects', 145 | '/NachoSEO/seo-scraper/wiki', 146 | '/NachoSEO/seo-scraper/security', 147 | '/NachoSEO/seo-scraper/pulse', 148 | '/NachoSEO/seo-scraper', 149 | '/NachoSEO/seo-scraper/issues', 150 | '/NachoSEO/seo-scraper/pulls', 151 | '/NachoSEO/seo-scraper/actions', 152 | '/NachoSEO/seo-scraper/projects', 153 | '/NachoSEO/seo-scraper/wiki', 154 | '/NachoSEO/seo-scraper/security', 155 | '/NachoSEO/seo-scraper/pulse', 156 | '/NachoSEO/seo-scraper/branches', 157 | '/NachoSEO/seo-scraper/tags', 158 | '/NachoSEO/seo-scraper/branches', 159 | '/NachoSEO/seo-scraper/tags', 160 | 'https://docs.github.com/articles/which-remote-url-should-i-use', 161 | 'https://cli.github.com/', 162 | 'https://desktop.github.com/', 163 | '/NachoSEO/seo-scraper/archive/refs/heads/master.zip', 164 | 'https://desktop.github.com/', 165 | 'https://desktop.github.com/', 166 | 'https://developer.apple.com/xcode/', 167 | '/NachoSEO', 168 | '/NachoSEO/seo-scraper/commits?author=NachoSEO', 169 | '/NachoSEO/seo-scraper/commit/eeb016220dacb9f9131772ad3a53ad03aede18d5', 170 | 'https://github.com/NachoSEO/seo-scraper/pull/3', 171 | '/NachoSEO/seo-scraper/commit/eeb016220dacb9f9131772ad3a53ad03aede18d5', 172 | '/NachoSEO/seo-scraper/commit/eeb016220dacb9f9131772ad3a53ad03aede18d5', 173 | '/NachoSEO/seo-scraper/commit/eeb016220dacb9f9131772ad3a53ad03aede18d5', 174 | '/NachoSEO/seo-scraper/commit/eeb016220dacb9f9131772ad3a53ad03aede18d5', 175 | 'https://github.com/NachoSEO/seo-scraper/pull/3', 176 | '/NachoSEO/seo-scraper/commit/eeb016220dacb9f9131772ad3a53ad03aede18d5', 177 | '/NachoSEO/seo-scraper/commits/master', 178 | '/NachoSEO/seo-scraper/tree/eeb016220dacb9f9131772ad3a53ad03aede18d5', 179 | '/NachoSEO/seo-scraper/tree/master/config', 180 | '/NachoSEO/seo-scraper/tree/master/fetch', 181 | '/NachoSEO/seo-scraper/tree/master/parse', 182 | '/NachoSEO/seo-scraper/tree/master/transformers', 183 | '/NachoSEO/seo-scraper/tree/master/utils', 184 | '/NachoSEO/seo-scraper/blob/master/.gitignore', 185 | '/NachoSEO/seo-scraper/blob/master/README.md', 186 | '/NachoSEO/seo-scraper/blob/master/index.js', 187 | '/NachoSEO/seo-scraper/blob/master/package.json', 188 | '/NachoSEO/seo-scraper/blob/master/yarn.lock', 189 | 'about:blank#nodejs-seo-scraper', 190 | 'about:blank#installing', 191 | ... 52 more items 192 | ], 193 | canonical: [ 'https://github.com/NachoSEO/seo-scraper' ], 194 | alternateMobile: [], 195 | prevPagination: [], 196 | nextPagination: [], 197 | amp: [], 198 | hreflang: [] 199 | } 200 | ` 201 | ``` 202 | 203 | 204 | ## Custom Configuration 205 | In order to work properly the SEO Scraper uses several files of configuration, although not all of them are necessary I recommend to use them all. 206 | 207 | 208 | ### Fetch 209 | The fetch function uses [Axios](https://github.com/axios/axios) in order to perform the requests (only the GET requests for now). You can use any of the customization that axios permits. 210 | 211 | In the default config we use this configuration: 212 | ```js 213 | { 214 | headers: { 215 | 'User-Agent': 'Node SEO Scraper 1.0.0' 216 | } 217 | } 218 | ``` 219 | 220 | 221 | ### Parsing 222 | This repo uses [JSDOM](https://github.com/jsdom/jsdom) in order to extract any elements from the DOM. JSDOM is a pure-JavaScript implementation of many web standards, notably the WHATWG DOM and HTML Standards, for use with Node.js. 223 | 224 | In order to extract the elements that you need instead of the default config you need to provide an Object with this structure: 225 | ``` 226 | { 227 | method1: { 228 | selectorName1: CSSselector1, 229 | selectorName2: CSSselector2 230 | }, 231 | method2: { 232 | selectorName3: CSSselector3, 233 | selectorName4: CSSselector4 234 | } 235 | } 236 | ``` 237 | If you need to use a custom parsing function please refer to [Transformers section](#Transformers). 238 | 239 | IMPORTANT: You need to know the web standard API in order to use the custom selectors as the structure of it depend of the methods to extract each tag. If you, for example, want to extract the Meta Robots value and the Canonical value you will need to use this config in selectors: 240 | ```js 241 | { 242 | content: { 243 | robots: "meta[name='robots']", 244 | }, 245 | href: { 246 | canonical: "link[rel='canonical']", 247 | } 248 | } 249 | ``` 250 | 251 | That's because as we are using JSDOM/web API, under the hood the `parse` function is running this: 252 | ```js 253 | 254 | document.querySelector(CSSselector1)[method1] 255 | //document.querySelector("meta[name='robots']").content 256 | 257 | document.querySelector(CSSselector2)[method2] 258 | //document.querySelector("link[rel='canonical']").href 259 | 260 | ``` 261 | 262 | You can check the default config of selectors as an example: 263 | ```js 264 | { 265 | content: { 266 | robots: "meta[name='robots']", 267 | description: "meta[name='description']", 268 | }, 269 | textContent: { 270 | title: "title", 271 | h1: "h1", 272 | h2: "h2", 273 | h3: "h3", 274 | h4: "h4", 275 | h5: "h5", 276 | h6: "h6" 277 | }, 278 | href: { 279 | links: "a", 280 | canonical: "link[rel='canonical']", 281 | alternateMobile: "link[media='only screen and (max-width: 640px)']", 282 | prevPagination: "link[rel='prev']", 283 | nextPagination: "link[rel='next']", 284 | amp: "link[rel='amphtml']" 285 | }, 286 | outerHTML: { 287 | hreflang: "link[hreflang]" 288 | } 289 | } 290 | ``` 291 | 292 | 293 | ### Proxies - Optional 294 | In order to use proxies it's required to pass an Object as argument to the main function `scrape()` with the same format as the example below: 295 | ```js 296 | const proxies = { 297 | "credentials": { 298 | "username": "exampleUser", 299 | "password": "examplePass" 300 | }, 301 | "whitelist": [ 302 | // host:port 303 | "11.11.11.11:21232", 304 | "3.2.1.114:21290" 305 | ] 306 | } 307 | ``` 308 | 309 | It's important to add an array in the whitelist property in order to use the utility of the package that selects a random proxy for every request. 310 | 311 | Example: 312 | ```js 313 | scraper.scrape({ url: 'https://example.com/', proxies }) 314 | ``` 315 | 316 | 317 | ## Custom scraping example 318 | This is an example of a custom scraping. 319 | 320 | We can use several variables and pass it to the functions as arguments or just one object with all the custom properties we need. 321 | 322 | ```js 323 | const scraper = require('seo-scraper'); 324 | 325 | const options = { 326 | url: 'https://github.com/NachoSEO', 327 | selectors: { 328 | textContent: { 329 | title:'title', 330 | }, 331 | content: { 332 | description: "meta[name='description']", 333 | } 334 | }, 335 | config: { 336 | headers: { 337 | 'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)', 338 | } 339 | }, 340 | proxies: { 341 | "credentials": { 342 | "username": "exampleUser", 343 | "password": "examplePass" 344 | }, 345 | "whitelist": [ 346 | "11.11.11.11:21232", 347 | "3.2.1.114:21290" 348 | ] 349 | } 350 | } 351 | 352 | scraper.scrape(options) 353 | .then(elements => console.log(elements)) 354 | .catch(err => console.error(err)); 355 | 356 | // Output 357 | ` 358 | { 359 | title: [ 'NachoSEO (Nacho Mascort) · GitHub' ], 360 | description: [ 361 | 'SEO Product Owner @ Softonic. NachoSEO has 13 repositories available. Follow their code on GitHub.' 362 | ] 363 | } 364 | ` 365 | ``` 366 | 367 | ## Transformers 368 | Maybe the `parse` function doesn't solve all your needs regarding Scraping and that's why exist the Transformer. The Transformer is a function that you could pass as a parameter in order to substitute the logic of the `parse` function. 369 | 370 | By default, the Transformer function uses: `document`, `selectors` & `...args` parameters in order to add any logic that you could need. 371 | 372 | In order to use the `...args` you will need to pass any extra argument in the `scrape`function (the main one from the examples). 373 | 374 | Like this: 375 | ```js 376 | scraper.scrape(options, arg1, arg2, arg3); 377 | ``` 378 | 379 | ### Example using a Transformer function 380 | Imagine an scenario where you can to extract some SERP data from Google and you need another logic with the `selectors` file and the `parse` function. 381 | 382 | DISCLAIMER: You should not scrape companies that don't allow that practice (as Google), this is just an example to ilustrate how works this kind of functionality (I'm using Google because the vast audience of this repo will be SEO and it's a familiar place to all of us ^^). 383 | 384 | First, let's define a `selectors.js` file: 385 | 386 | ```js 387 | module.exports = { 388 | parent: '#ires .g, #res .g:not(.kno-kp)', 389 | elements: { 390 | textContent: { 391 | title: '.r a h3', 392 | description: '.s' 393 | }, 394 | href: { 395 | url: '.r a', 396 | sitelinks: 'table a' 397 | } 398 | } 399 | } 400 | ``` 401 | As you see, I'm using a parent selector and the current `parse`function doesn't allow that so we'll need a Transformer in order to use that file. 402 | 403 | Let's define the Transformer function in a file called: `parseSERP.js` (you could find the same function in the transformers folders): 404 | ```js 405 | module.exports = function parseSERP(document, selectors, ...args) { 406 | const scrapedArr = [] 407 | const { parent, elements } = selectors; 408 | 409 | document 410 | .querySelectorAll(parent) 411 | .forEach(node => { 412 | const scrapedElement = {} 413 | for (const [method, value] of Object.entries(elements)) { 414 | for (const [selectorName, selector] of Object.entries(value)) { 415 | try { 416 | scrapedElement[selectorName] = node.querySelector(selector)[method]; 417 | } catch (err) { 418 | console.error(err); 419 | scrapedElement[selectorName] = ''; 420 | } 421 | } 422 | } 423 | scrapedArr.push(scrapedElement); 424 | }) 425 | 426 | return scrapedArr; 427 | } 428 | ``` 429 | Finally we need to add those files into our entrypoint and pass them as arguments: 430 | 431 | ```js 432 | const selectors = require('path/to/selectors'); 433 | const parseSERP = require('transformers/parseSERP') 434 | 435 | const options = { 436 | url: 'https://www.google.com/search?q=hello+google', 437 | selectors, 438 | transformer: parseSERP 439 | } 440 | 441 | scraper.scrape(options) 442 | .then(elements => console.log(elements)) 443 | .catch(err => console.error(err)); 444 | 445 | //Output 446 | ` 447 | [ 448 | { 449 | title: 'Google Assistant - Get things done, hands-free - Apps on ...', 450 | description: 'Get the Google Assistant for hands-free help. Your Google Assistant is ready to help when and where you need it. Manage your schedule , get help with ... Rating: 4.1 - ‎257,718 votes - ‎Free - ‎Android - ‎Business/Productivity', 451 | url: 'https://play.google.com/store/apps/details?id=com.google.android.apps.googleassistant&hl=en_US', 452 | sitelinks: '' 453 | }, 454 | { 455 | title: 'Google – Apps on Google Play', 456 | description: 'The Google app keeps you in the know about the things that you care about. Find quick answers, explore your interests and get a feed of updates on what ... Rating: 4.2 - ‎16,073,194 votes - ‎Free - ‎Android - ‎Utilities/Tools', 457 | url: 'https://play.google.com/store/apps/details?id=com.google.android.googlequicksearchbox&hl=en_GB', 458 | sitelinks: '' 459 | }, 460 | { 461 | title: 'Google Assistant, your own personal Google', 462 | description: "Meet your Google Assistant. Ask it questions. Tell it to do things. It's your own personal Google, always ready to help whenever you need it.‎Get Google Assistant · ‎What it can do · ‎News and resources", 463 | url: 'https://assistant.google.com/', 464 | sitelinks: '' 465 | }, 466 | ... 467 | ] 468 | ` 469 | ``` 470 | -------------------------------------------------------------------------------- /yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | "@tootallnate/once@1": 6 | version "1.1.2" 7 | resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" 8 | integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== 9 | 10 | abab@^2.0.3, abab@^2.0.5: 11 | version "2.0.5" 12 | resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.5.tgz#c0b678fb32d60fc1219c784d6a826fe385aeb79a" 13 | integrity sha512-9IK9EadsbHo6jLWIpxpR6pL0sazTXV6+SQv25ZB+F7Bj9mJNaOc4nCRabwd5M/JwmUa8idz6Eci6eKfJryPs6Q== 14 | 15 | acorn-globals@^6.0.0: 16 | version "6.0.0" 17 | resolved "https://registry.yarnpkg.com/acorn-globals/-/acorn-globals-6.0.0.tgz#46cdd39f0f8ff08a876619b55f5ac8a6dc770b45" 18 | integrity sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg== 19 | dependencies: 20 | acorn "^7.1.1" 21 | acorn-walk "^7.1.1" 22 | 23 | acorn-walk@^7.1.1: 24 | version "7.2.0" 25 | resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc" 26 | integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA== 27 | 28 | acorn@^7.1.1: 29 | version "7.4.1" 30 | resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" 31 | integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== 32 | 33 | acorn@^8.2.4: 34 | version "8.4.1" 35 | resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.4.1.tgz#56c36251fc7cabc7096adc18f05afe814321a28c" 36 | integrity sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA== 37 | 38 | agent-base@6: 39 | version "6.0.2" 40 | resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" 41 | integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== 42 | dependencies: 43 | debug "4" 44 | 45 | asynckit@^0.4.0: 46 | version "0.4.0" 47 | resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" 48 | integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= 49 | 50 | axios@^0.21.1: 51 | version "0.21.2" 52 | resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.2.tgz#21297d5084b2aeeb422f5d38e7be4fbb82239017" 53 | integrity sha512-87otirqUw3e8CzHTMO+/9kh/FSgXt/eVDvipijwDtEuwbkySWZ9SBm6VEubmJ/kLKEoLQV/POhxXFb66bfekfg== 54 | dependencies: 55 | follow-redirects "^1.14.0" 56 | 57 | browser-process-hrtime@^1.0.0: 58 | version "1.0.0" 59 | resolved "https://registry.yarnpkg.com/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz#3c9b4b7d782c8121e56f10106d84c0d0ffc94626" 60 | integrity sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow== 61 | 62 | combined-stream@^1.0.8: 63 | version "1.0.8" 64 | resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" 65 | integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== 66 | dependencies: 67 | delayed-stream "~1.0.0" 68 | 69 | cssom@^0.4.4: 70 | version "0.4.4" 71 | resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.4.4.tgz#5a66cf93d2d0b661d80bf6a44fb65f5c2e4e0a10" 72 | integrity sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw== 73 | 74 | cssom@~0.3.6: 75 | version "0.3.8" 76 | resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.3.8.tgz#9f1276f5b2b463f2114d3f2c75250af8c1a36f4a" 77 | integrity sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg== 78 | 79 | cssstyle@^2.3.0: 80 | version "2.3.0" 81 | resolved "https://registry.yarnpkg.com/cssstyle/-/cssstyle-2.3.0.tgz#ff665a0ddbdc31864b09647f34163443d90b0852" 82 | integrity sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A== 83 | dependencies: 84 | cssom "~0.3.6" 85 | 86 | data-urls@^2.0.0: 87 | version "2.0.0" 88 | resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-2.0.0.tgz#156485a72963a970f5d5821aaf642bef2bf2db9b" 89 | integrity sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ== 90 | dependencies: 91 | abab "^2.0.3" 92 | whatwg-mimetype "^2.3.0" 93 | whatwg-url "^8.0.0" 94 | 95 | debug@4: 96 | version "4.3.2" 97 | resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b" 98 | integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw== 99 | dependencies: 100 | ms "2.1.2" 101 | 102 | decimal.js@^10.2.1: 103 | version "10.3.1" 104 | resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.3.1.tgz#d8c3a444a9c6774ba60ca6ad7261c3a94fd5e783" 105 | integrity sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ== 106 | 107 | deep-is@~0.1.3: 108 | version "0.1.3" 109 | resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" 110 | integrity sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ= 111 | 112 | delayed-stream@~1.0.0: 113 | version "1.0.0" 114 | resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" 115 | integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= 116 | 117 | domexception@^2.0.1: 118 | version "2.0.1" 119 | resolved "https://registry.yarnpkg.com/domexception/-/domexception-2.0.1.tgz#fb44aefba793e1574b0af6aed2801d057529f304" 120 | integrity sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg== 121 | dependencies: 122 | webidl-conversions "^5.0.0" 123 | 124 | escodegen@^2.0.0: 125 | version "2.0.0" 126 | resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-2.0.0.tgz#5e32b12833e8aa8fa35e1bf0befa89380484c7dd" 127 | integrity sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw== 128 | dependencies: 129 | esprima "^4.0.1" 130 | estraverse "^5.2.0" 131 | esutils "^2.0.2" 132 | optionator "^0.8.1" 133 | optionalDependencies: 134 | source-map "~0.6.1" 135 | 136 | esprima@^4.0.1: 137 | version "4.0.1" 138 | resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" 139 | integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== 140 | 141 | estraverse@^5.2.0: 142 | version "5.2.0" 143 | resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.2.0.tgz#307df42547e6cc7324d3cf03c155d5cdb8c53880" 144 | integrity sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ== 145 | 146 | esutils@^2.0.2: 147 | version "2.0.3" 148 | resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" 149 | integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== 150 | 151 | fast-levenshtein@~2.0.6: 152 | version "2.0.6" 153 | resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" 154 | integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc= 155 | 156 | follow-redirects@^1.14.0: 157 | version "1.14.9" 158 | resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.9.tgz#dd4ea157de7bfaf9ea9b3fbd85aa16951f78d8d7" 159 | integrity sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w== 160 | 161 | form-data@^3.0.0: 162 | version "3.0.1" 163 | resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.1.tgz#ebd53791b78356a99af9a300d4282c4d5eb9755f" 164 | integrity sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg== 165 | dependencies: 166 | asynckit "^0.4.0" 167 | combined-stream "^1.0.8" 168 | mime-types "^2.1.12" 169 | 170 | html-encoding-sniffer@^2.0.1: 171 | version "2.0.1" 172 | resolved "https://registry.yarnpkg.com/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz#42a6dc4fd33f00281176e8b23759ca4e4fa185f3" 173 | integrity sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ== 174 | dependencies: 175 | whatwg-encoding "^1.0.5" 176 | 177 | http-proxy-agent@^4.0.1: 178 | version "4.0.1" 179 | resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz#8a8c8ef7f5932ccf953c296ca8291b95aa74aa3a" 180 | integrity sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg== 181 | dependencies: 182 | "@tootallnate/once" "1" 183 | agent-base "6" 184 | debug "4" 185 | 186 | https-proxy-agent@^5.0.0: 187 | version "5.0.0" 188 | resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2" 189 | integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA== 190 | dependencies: 191 | agent-base "6" 192 | debug "4" 193 | 194 | iconv-lite@0.4.24: 195 | version "0.4.24" 196 | resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" 197 | integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== 198 | dependencies: 199 | safer-buffer ">= 2.1.2 < 3" 200 | 201 | is-potential-custom-element-name@^1.0.1: 202 | version "1.0.1" 203 | resolved "https://registry.yarnpkg.com/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz#171ed6f19e3ac554394edf78caa05784a45bebb5" 204 | integrity sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ== 205 | 206 | jsdom@^16.4.0: 207 | version "16.7.0" 208 | resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-16.7.0.tgz#918ae71965424b197c819f8183a754e18977b710" 209 | integrity sha512-u9Smc2G1USStM+s/x1ru5Sxrl6mPYCbByG1U/hUmqaVsm4tbNyS7CicOSRyuGQYZhTu0h84qkZZQ/I+dzizSVw== 210 | dependencies: 211 | abab "^2.0.5" 212 | acorn "^8.2.4" 213 | acorn-globals "^6.0.0" 214 | cssom "^0.4.4" 215 | cssstyle "^2.3.0" 216 | data-urls "^2.0.0" 217 | decimal.js "^10.2.1" 218 | domexception "^2.0.1" 219 | escodegen "^2.0.0" 220 | form-data "^3.0.0" 221 | html-encoding-sniffer "^2.0.1" 222 | http-proxy-agent "^4.0.1" 223 | https-proxy-agent "^5.0.0" 224 | is-potential-custom-element-name "^1.0.1" 225 | nwsapi "^2.2.0" 226 | parse5 "6.0.1" 227 | saxes "^5.0.1" 228 | symbol-tree "^3.2.4" 229 | tough-cookie "^4.0.0" 230 | w3c-hr-time "^1.0.2" 231 | w3c-xmlserializer "^2.0.0" 232 | webidl-conversions "^6.1.0" 233 | whatwg-encoding "^1.0.5" 234 | whatwg-mimetype "^2.3.0" 235 | whatwg-url "^8.5.0" 236 | ws "^7.4.6" 237 | xml-name-validator "^3.0.0" 238 | 239 | levn@~0.3.0: 240 | version "0.3.0" 241 | resolved "https://registry.yarnpkg.com/levn/-/levn-0.3.0.tgz#3b09924edf9f083c0490fdd4c0bc4421e04764ee" 242 | integrity sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4= 243 | dependencies: 244 | prelude-ls "~1.1.2" 245 | type-check "~0.3.2" 246 | 247 | lodash@^4.7.0: 248 | version "4.17.21" 249 | resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" 250 | integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== 251 | 252 | mime-db@1.49.0: 253 | version "1.49.0" 254 | resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.49.0.tgz#f3dfde60c99e9cf3bc9701d687778f537001cbed" 255 | integrity sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA== 256 | 257 | mime-types@^2.1.12: 258 | version "2.1.32" 259 | resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.32.tgz#1d00e89e7de7fe02008db61001d9e02852670fd5" 260 | integrity sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A== 261 | dependencies: 262 | mime-db "1.49.0" 263 | 264 | ms@2.1.2: 265 | version "2.1.2" 266 | resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" 267 | integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== 268 | 269 | nwsapi@^2.2.0: 270 | version "2.2.0" 271 | resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.0.tgz#204879a9e3d068ff2a55139c2c772780681a38b7" 272 | integrity sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ== 273 | 274 | optionator@^0.8.1: 275 | version "0.8.3" 276 | resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.3.tgz#84fa1d036fe9d3c7e21d99884b601167ec8fb495" 277 | integrity sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA== 278 | dependencies: 279 | deep-is "~0.1.3" 280 | fast-levenshtein "~2.0.6" 281 | levn "~0.3.0" 282 | prelude-ls "~1.1.2" 283 | type-check "~0.3.2" 284 | word-wrap "~1.2.3" 285 | 286 | parse5@6.0.1: 287 | version "6.0.1" 288 | resolved "https://registry.yarnpkg.com/parse5/-/parse5-6.0.1.tgz#e1a1c085c569b3dc08321184f19a39cc27f7c30b" 289 | integrity sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw== 290 | 291 | prelude-ls@~1.1.2: 292 | version "1.1.2" 293 | resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" 294 | integrity sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ= 295 | 296 | psl@^1.1.33: 297 | version "1.8.0" 298 | resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" 299 | integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== 300 | 301 | punycode@^2.1.1: 302 | version "2.1.1" 303 | resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" 304 | integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== 305 | 306 | "safer-buffer@>= 2.1.2 < 3": 307 | version "2.1.2" 308 | resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" 309 | integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== 310 | 311 | saxes@^5.0.1: 312 | version "5.0.1" 313 | resolved "https://registry.yarnpkg.com/saxes/-/saxes-5.0.1.tgz#eebab953fa3b7608dbe94e5dadb15c888fa6696d" 314 | integrity sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw== 315 | dependencies: 316 | xmlchars "^2.2.0" 317 | 318 | source-map@~0.6.1: 319 | version "0.6.1" 320 | resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" 321 | integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== 322 | 323 | symbol-tree@^3.2.4: 324 | version "3.2.4" 325 | resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2" 326 | integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw== 327 | 328 | tough-cookie@^4.0.0: 329 | version "4.0.0" 330 | resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.0.0.tgz#d822234eeca882f991f0f908824ad2622ddbece4" 331 | integrity sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg== 332 | dependencies: 333 | psl "^1.1.33" 334 | punycode "^2.1.1" 335 | universalify "^0.1.2" 336 | 337 | tr46@^2.1.0: 338 | version "2.1.0" 339 | resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" 340 | integrity sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw== 341 | dependencies: 342 | punycode "^2.1.1" 343 | 344 | type-check@~0.3.2: 345 | version "0.3.2" 346 | resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72" 347 | integrity sha1-WITKtRLPHTVeP7eE8wgEsrUg23I= 348 | dependencies: 349 | prelude-ls "~1.1.2" 350 | 351 | universalify@^0.1.2: 352 | version "0.1.2" 353 | resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" 354 | integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== 355 | 356 | w3c-hr-time@^1.0.2: 357 | version "1.0.2" 358 | resolved "https://registry.yarnpkg.com/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz#0a89cdf5cc15822df9c360543676963e0cc308cd" 359 | integrity sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ== 360 | dependencies: 361 | browser-process-hrtime "^1.0.0" 362 | 363 | w3c-xmlserializer@^2.0.0: 364 | version "2.0.0" 365 | resolved "https://registry.yarnpkg.com/w3c-xmlserializer/-/w3c-xmlserializer-2.0.0.tgz#3e7104a05b75146cc60f564380b7f683acf1020a" 366 | integrity sha512-4tzD0mF8iSiMiNs30BiLO3EpfGLZUT2MSX/G+o7ZywDzliWQ3OPtTZ0PTC3B3ca1UAf4cJMHB+2Bf56EriJuRA== 367 | dependencies: 368 | xml-name-validator "^3.0.0" 369 | 370 | webidl-conversions@^5.0.0: 371 | version "5.0.0" 372 | resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-5.0.0.tgz#ae59c8a00b121543a2acc65c0434f57b0fc11aff" 373 | integrity sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA== 374 | 375 | webidl-conversions@^6.1.0: 376 | version "6.1.0" 377 | resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-6.1.0.tgz#9111b4d7ea80acd40f5270d666621afa78b69514" 378 | integrity sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w== 379 | 380 | whatwg-encoding@^1.0.5: 381 | version "1.0.5" 382 | resolved "https://registry.yarnpkg.com/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz#5abacf777c32166a51d085d6b4f3e7d27113ddb0" 383 | integrity sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw== 384 | dependencies: 385 | iconv-lite "0.4.24" 386 | 387 | whatwg-mimetype@^2.3.0: 388 | version "2.3.0" 389 | resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" 390 | integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== 391 | 392 | whatwg-url@^8.0.0, whatwg-url@^8.5.0: 393 | version "8.7.0" 394 | resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-8.7.0.tgz#656a78e510ff8f3937bc0bcbe9f5c0ac35941b77" 395 | integrity sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg== 396 | dependencies: 397 | lodash "^4.7.0" 398 | tr46 "^2.1.0" 399 | webidl-conversions "^6.1.0" 400 | 401 | word-wrap@~1.2.3: 402 | version "1.2.3" 403 | resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" 404 | integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== 405 | 406 | ws@^7.4.6: 407 | version "7.5.3" 408 | resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.3.tgz#160835b63c7d97bfab418fc1b8a9fced2ac01a74" 409 | integrity sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg== 410 | 411 | xml-name-validator@^3.0.0: 412 | version "3.0.0" 413 | resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" 414 | integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== 415 | 416 | xmlchars@^2.2.0: 417 | version "2.2.0" 418 | resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" 419 | integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw== 420 | --------------------------------------------------------------------------------