├── readme.md ├── web-scraper-node ├── .gitignore ├── package-lock.json ├── package.json └── scrape.js └── web-scraper-python ├── .gitignore ├── requirements.txt └── scrape.py /readme.md: -------------------------------------------------------------------------------- 1 | # Web Scrapers 2 | 3 | This is the code from my web scraping guide on youtube. It includes two scrapers. One with Node.js and Puppeteer and one with Python and Beautiful Soup. 4 | 5 | We scrape the website https://books.toscrape.com/ but you can change the code to scrape something else. 6 | 7 | ## Node.js & Puppeteer Usage 8 | 9 | Install dependencies 10 | 11 | ```bash 12 | cd web-scraper-node 13 | npm install 14 | ``` 15 | 16 | Run the file 17 | 18 | ```bash 19 | node scrape 20 | ``` 21 | 22 | ## Python & BS4 Usage 23 | 24 | Install dependencies 25 | 26 | ```bash 27 | cd web-scraper-python 28 | pip install -r requirements.txt 29 | ``` 30 | 31 | Run the file 32 | 33 | ```bash 34 | python scrape.py 35 | ``` 36 | -------------------------------------------------------------------------------- /web-scraper-node/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /web-scraper-node/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "web-scraper-node", 3 | "version": "1.0.0", 4 | "lockfileVersion": 3, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "web-scraper-node", 9 | "version": "1.0.0", 10 | "license": "ISC", 11 | "dependencies": { 12 | "puppeteer": "^22.15.0" 13 | } 14 | }, 15 | "node_modules/@babel/code-frame": { 16 | "version": "7.24.7", 17 | "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.24.7.tgz", 18 | "integrity": "sha512-BcYH1CVJBO9tvyIZ2jVeXgSIMvGZ2FDRvDdOIVQyuklNKSsx+eppDEBq/g47Ayw+RqNFE+URvOShmf+f/qwAlA==", 19 | "license": "MIT", 20 | "dependencies": { 21 | "@babel/highlight": "^7.24.7", 22 | "picocolors": "^1.0.0" 23 | }, 24 | "engines": { 25 | "node": ">=6.9.0" 26 | } 27 | }, 28 | "node_modules/@babel/helper-validator-identifier": { 29 | "version": "7.24.7", 30 | "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.7.tgz", 31 | "integrity": "sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w==", 32 | "license": "MIT", 33 | "engines": { 34 | "node": ">=6.9.0" 35 | } 36 | }, 37 | "node_modules/@babel/highlight": { 38 | "version": "7.24.7", 39 | "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.24.7.tgz", 40 | "integrity": "sha512-EStJpq4OuY8xYfhGVXngigBJRWxftKX9ksiGDnmlY3o7B/V7KIAc9X4oiK87uPJSc/vs5L869bem5fhZa8caZw==", 41 | "license": "MIT", 42 | "dependencies": { 43 | "@babel/helper-validator-identifier": "^7.24.7", 44 | "chalk": "^2.4.2", 45 | "js-tokens": "^4.0.0", 46 | "picocolors": "^1.0.0" 47 | }, 48 | "engines": { 49 | "node": ">=6.9.0" 50 | } 51 | }, 52 | "node_modules/@puppeteer/browsers": { 53 | "version": "2.3.0", 54 | "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.3.0.tgz", 55 | "integrity": "sha512-ioXoq9gPxkss4MYhD+SFaU9p1IHFUX0ILAWFPyjGaBdjLsYAlZw6j1iLA0N/m12uVHLFDfSYNF7EQccjinIMDA==", 56 | "license": "Apache-2.0", 57 | "dependencies": { 58 | "debug": "^4.3.5", 59 | "extract-zip": "^2.0.1", 60 | "progress": "^2.0.3", 61 | "proxy-agent": "^6.4.0", 62 | "semver": "^7.6.3", 63 | "tar-fs": "^3.0.6", 64 | "unbzip2-stream": "^1.4.3", 65 | "yargs": "^17.7.2" 66 | }, 67 | "bin": { 68 | "browsers": "lib/cjs/main-cli.js" 69 | }, 70 | "engines": { 71 | "node": ">=18" 72 | } 73 | }, 74 | "node_modules/@tootallnate/quickjs-emscripten": { 75 | "version": "0.23.0", 76 | "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", 77 | "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", 78 | "license": "MIT" 79 | }, 80 | "node_modules/@types/node": { 81 | "version": "22.1.0", 82 | "resolved": "https://registry.npmjs.org/@types/node/-/node-22.1.0.tgz", 83 | "integrity": "sha512-AOmuRF0R2/5j1knA3c6G3HOk523Ga+l+ZXltX8SF1+5oqcXijjfTd8fY3XRZqSihEu9XhtQnKYLmkFaoxgsJHw==", 84 | "license": "MIT", 85 | "optional": true, 86 | "dependencies": { 87 | "undici-types": "~6.13.0" 88 | } 89 | }, 90 | "node_modules/@types/yauzl": { 91 | "version": "2.10.3", 92 | "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", 93 | "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", 94 | "license": "MIT", 95 | "optional": true, 96 | "dependencies": { 97 | "@types/node": "*" 98 | } 99 | }, 100 | "node_modules/agent-base": { 101 | "version": "7.1.1", 102 | "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", 103 | "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", 104 | "license": "MIT", 105 | "dependencies": { 106 | "debug": "^4.3.4" 107 | }, 108 | "engines": { 109 | "node": ">= 14" 110 | } 111 | }, 112 | "node_modules/ansi-regex": { 113 | "version": "5.0.1", 114 | "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", 115 | "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", 116 | "license": "MIT", 117 | "engines": { 118 | "node": ">=8" 119 | } 120 | }, 121 | "node_modules/ansi-styles": { 122 | "version": "3.2.1", 123 | "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", 124 | "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", 125 | "license": "MIT", 126 | "dependencies": { 127 | "color-convert": "^1.9.0" 128 | }, 129 | "engines": { 130 | "node": ">=4" 131 | } 132 | }, 133 | "node_modules/argparse": { 134 | "version": "2.0.1", 135 | "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", 136 | "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", 137 | "license": "Python-2.0" 138 | }, 139 | "node_modules/ast-types": { 140 | "version": "0.13.4", 141 | "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", 142 | "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", 143 | "license": "MIT", 144 | "dependencies": { 145 | "tslib": "^2.0.1" 146 | }, 147 | "engines": { 148 | "node": ">=4" 149 | } 150 | }, 151 | "node_modules/b4a": { 152 | "version": "1.6.6", 153 | "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.6.tgz", 154 | "integrity": "sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg==", 155 | "license": "Apache-2.0" 156 | }, 157 | "node_modules/bare-events": { 158 | "version": "2.4.2", 159 | "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.4.2.tgz", 160 | "integrity": "sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q==", 161 | "license": "Apache-2.0", 162 | "optional": true 163 | }, 164 | "node_modules/bare-fs": { 165 | "version": "2.3.1", 166 | "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.3.1.tgz", 167 | "integrity": "sha512-W/Hfxc/6VehXlsgFtbB5B4xFcsCl+pAh30cYhoFyXErf6oGrwjh8SwiPAdHgpmWonKuYpZgGywN0SXt7dgsADA==", 168 | "license": "Apache-2.0", 169 | "optional": true, 170 | "dependencies": { 171 | "bare-events": "^2.0.0", 172 | "bare-path": "^2.0.0", 173 | "bare-stream": "^2.0.0" 174 | } 175 | }, 176 | "node_modules/bare-os": { 177 | "version": "2.4.0", 178 | "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.4.0.tgz", 179 | "integrity": "sha512-v8DTT08AS/G0F9xrhyLtepoo9EJBJ85FRSMbu1pQUlAf6A8T0tEEQGMVObWeqpjhSPXsE0VGlluFBJu2fdoTNg==", 180 | "license": "Apache-2.0", 181 | "optional": true 182 | }, 183 | "node_modules/bare-path": { 184 | "version": "2.1.3", 185 | "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-2.1.3.tgz", 186 | "integrity": "sha512-lh/eITfU8hrj9Ru5quUp0Io1kJWIk1bTjzo7JH1P5dWmQ2EL4hFUlfI8FonAhSlgIfhn63p84CDY/x+PisgcXA==", 187 | "license": "Apache-2.0", 188 | "optional": true, 189 | "dependencies": { 190 | "bare-os": "^2.1.0" 191 | } 192 | }, 193 | "node_modules/bare-stream": { 194 | "version": "2.1.3", 195 | "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.1.3.tgz", 196 | "integrity": "sha512-tiDAH9H/kP+tvNO5sczyn9ZAA7utrSMobyDchsnyyXBuUe2FSQWbxhtuHB8jwpHYYevVo2UJpcmvvjrbHboUUQ==", 197 | "license": "Apache-2.0", 198 | "optional": true, 199 | "dependencies": { 200 | "streamx": "^2.18.0" 201 | } 202 | }, 203 | "node_modules/base64-js": { 204 | "version": "1.5.1", 205 | "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", 206 | "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", 207 | "funding": [ 208 | { 209 | "type": "github", 210 | "url": "https://github.com/sponsors/feross" 211 | }, 212 | { 213 | "type": "patreon", 214 | "url": "https://www.patreon.com/feross" 215 | }, 216 | { 217 | "type": "consulting", 218 | "url": "https://feross.org/support" 219 | } 220 | ], 221 | "license": "MIT" 222 | }, 223 | "node_modules/basic-ftp": { 224 | "version": "5.0.5", 225 | "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", 226 | "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", 227 | "license": "MIT", 228 | "engines": { 229 | "node": ">=10.0.0" 230 | } 231 | }, 232 | "node_modules/buffer": { 233 | "version": "5.7.1", 234 | "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", 235 | "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", 236 | "funding": [ 237 | { 238 | "type": "github", 239 | "url": "https://github.com/sponsors/feross" 240 | }, 241 | { 242 | "type": "patreon", 243 | "url": "https://www.patreon.com/feross" 244 | }, 245 | { 246 | "type": "consulting", 247 | "url": "https://feross.org/support" 248 | } 249 | ], 250 | "license": "MIT", 251 | "dependencies": { 252 | "base64-js": "^1.3.1", 253 | "ieee754": "^1.1.13" 254 | } 255 | }, 256 | "node_modules/buffer-crc32": { 257 | "version": "0.2.13", 258 | "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", 259 | "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", 260 | "license": "MIT", 261 | "engines": { 262 | "node": "*" 263 | } 264 | }, 265 | "node_modules/callsites": { 266 | "version": "3.1.0", 267 | "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", 268 | "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", 269 | "license": "MIT", 270 | "engines": { 271 | "node": ">=6" 272 | } 273 | }, 274 | "node_modules/chalk": { 275 | "version": "2.4.2", 276 | "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", 277 | "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", 278 | "license": "MIT", 279 | "dependencies": { 280 | "ansi-styles": "^3.2.1", 281 | "escape-string-regexp": "^1.0.5", 282 | "supports-color": "^5.3.0" 283 | }, 284 | "engines": { 285 | "node": ">=4" 286 | } 287 | }, 288 | "node_modules/chromium-bidi": { 289 | "version": "0.6.3", 290 | "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-0.6.3.tgz", 291 | "integrity": "sha512-qXlsCmpCZJAnoTYI83Iu6EdYQpMYdVkCfq08KDh2pmlVqK5t5IA9mGs4/LwCwp4fqisSOMXZxP3HIh8w8aRn0A==", 292 | "license": "Apache-2.0", 293 | "dependencies": { 294 | "mitt": "3.0.1", 295 | "urlpattern-polyfill": "10.0.0", 296 | "zod": "3.23.8" 297 | }, 298 | "peerDependencies": { 299 | "devtools-protocol": "*" 300 | } 301 | }, 302 | "node_modules/cliui": { 303 | "version": "8.0.1", 304 | "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", 305 | "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", 306 | "license": "ISC", 307 | "dependencies": { 308 | "string-width": "^4.2.0", 309 | "strip-ansi": "^6.0.1", 310 | "wrap-ansi": "^7.0.0" 311 | }, 312 | "engines": { 313 | "node": ">=12" 314 | } 315 | }, 316 | "node_modules/color-convert": { 317 | "version": "1.9.3", 318 | "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", 319 | "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", 320 | "license": "MIT", 321 | "dependencies": { 322 | "color-name": "1.1.3" 323 | } 324 | }, 325 | "node_modules/color-name": { 326 | "version": "1.1.3", 327 | "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", 328 | "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", 329 | "license": "MIT" 330 | }, 331 | "node_modules/cosmiconfig": { 332 | "version": "9.0.0", 333 | "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.0.tgz", 334 | "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", 335 | "license": "MIT", 336 | "dependencies": { 337 | "env-paths": "^2.2.1", 338 | "import-fresh": "^3.3.0", 339 | "js-yaml": "^4.1.0", 340 | "parse-json": "^5.2.0" 341 | }, 342 | "engines": { 343 | "node": ">=14" 344 | }, 345 | "funding": { 346 | "url": "https://github.com/sponsors/d-fischer" 347 | }, 348 | "peerDependencies": { 349 | "typescript": ">=4.9.5" 350 | }, 351 | "peerDependenciesMeta": { 352 | "typescript": { 353 | "optional": true 354 | } 355 | } 356 | }, 357 | "node_modules/data-uri-to-buffer": { 358 | "version": "6.0.2", 359 | "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", 360 | "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", 361 | "license": "MIT", 362 | "engines": { 363 | "node": ">= 14" 364 | } 365 | }, 366 | "node_modules/debug": { 367 | "version": "4.3.6", 368 | "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.6.tgz", 369 | "integrity": "sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==", 370 | "license": "MIT", 371 | "dependencies": { 372 | "ms": "2.1.2" 373 | }, 374 | "engines": { 375 | "node": ">=6.0" 376 | }, 377 | "peerDependenciesMeta": { 378 | "supports-color": { 379 | "optional": true 380 | } 381 | } 382 | }, 383 | "node_modules/degenerator": { 384 | "version": "5.0.1", 385 | "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", 386 | "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", 387 | "license": "MIT", 388 | "dependencies": { 389 | "ast-types": "^0.13.4", 390 | "escodegen": "^2.1.0", 391 | "esprima": "^4.0.1" 392 | }, 393 | "engines": { 394 | "node": ">= 14" 395 | } 396 | }, 397 | "node_modules/devtools-protocol": { 398 | "version": "0.0.1312386", 399 | "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1312386.tgz", 400 | "integrity": "sha512-DPnhUXvmvKT2dFA/j7B+riVLUt9Q6RKJlcppojL5CoRywJJKLDYnRlw0gTFKfgDPHP5E04UoB71SxoJlVZy8FA==", 401 | "license": "BSD-3-Clause" 402 | }, 403 | "node_modules/emoji-regex": { 404 | "version": "8.0.0", 405 | "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", 406 | "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", 407 | "license": "MIT" 408 | }, 409 | "node_modules/end-of-stream": { 410 | "version": "1.4.4", 411 | "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", 412 | "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", 413 | "license": "MIT", 414 | "dependencies": { 415 | "once": "^1.4.0" 416 | } 417 | }, 418 | "node_modules/env-paths": { 419 | "version": "2.2.1", 420 | "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", 421 | "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", 422 | "license": "MIT", 423 | "engines": { 424 | "node": ">=6" 425 | } 426 | }, 427 | "node_modules/error-ex": { 428 | "version": "1.3.2", 429 | "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", 430 | "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", 431 | "license": "MIT", 432 | "dependencies": { 433 | "is-arrayish": "^0.2.1" 434 | } 435 | }, 436 | "node_modules/escalade": { 437 | "version": "3.1.2", 438 | "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", 439 | "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==", 440 | "license": "MIT", 441 | "engines": { 442 | "node": ">=6" 443 | } 444 | }, 445 | "node_modules/escape-string-regexp": { 446 | "version": "1.0.5", 447 | "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", 448 | "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", 449 | "license": "MIT", 450 | "engines": { 451 | "node": ">=0.8.0" 452 | } 453 | }, 454 | "node_modules/escodegen": { 455 | "version": "2.1.0", 456 | "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", 457 | "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", 458 | "license": "BSD-2-Clause", 459 | "dependencies": { 460 | "esprima": "^4.0.1", 461 | "estraverse": "^5.2.0", 462 | "esutils": "^2.0.2" 463 | }, 464 | "bin": { 465 | "escodegen": "bin/escodegen.js", 466 | "esgenerate": "bin/esgenerate.js" 467 | }, 468 | "engines": { 469 | "node": ">=6.0" 470 | }, 471 | "optionalDependencies": { 472 | "source-map": "~0.6.1" 473 | } 474 | }, 475 | "node_modules/esprima": { 476 | "version": "4.0.1", 477 | "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", 478 | "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", 479 | "license": "BSD-2-Clause", 480 | "bin": { 481 | "esparse": "bin/esparse.js", 482 | "esvalidate": "bin/esvalidate.js" 483 | }, 484 | "engines": { 485 | "node": ">=4" 486 | } 487 | }, 488 | "node_modules/estraverse": { 489 | "version": "5.3.0", 490 | "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", 491 | "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", 492 | "license": "BSD-2-Clause", 493 | "engines": { 494 | "node": ">=4.0" 495 | } 496 | }, 497 | "node_modules/esutils": { 498 | "version": "2.0.3", 499 | "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", 500 | "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", 501 | "license": "BSD-2-Clause", 502 | "engines": { 503 | "node": ">=0.10.0" 504 | } 505 | }, 506 | "node_modules/extract-zip": { 507 | "version": "2.0.1", 508 | "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", 509 | "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", 510 | "license": "BSD-2-Clause", 511 | "dependencies": { 512 | "debug": "^4.1.1", 513 | "get-stream": "^5.1.0", 514 | "yauzl": "^2.10.0" 515 | }, 516 | "bin": { 517 | "extract-zip": "cli.js" 518 | }, 519 | "engines": { 520 | "node": ">= 10.17.0" 521 | }, 522 | "optionalDependencies": { 523 | "@types/yauzl": "^2.9.1" 524 | } 525 | }, 526 | "node_modules/fast-fifo": { 527 | "version": "1.3.2", 528 | "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", 529 | "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", 530 | "license": "MIT" 531 | }, 532 | "node_modules/fd-slicer": { 533 | "version": "1.1.0", 534 | "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", 535 | "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", 536 | "license": "MIT", 537 | "dependencies": { 538 | "pend": "~1.2.0" 539 | } 540 | }, 541 | "node_modules/fs-extra": { 542 | "version": "11.2.0", 543 | "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", 544 | "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", 545 | "license": "MIT", 546 | "dependencies": { 547 | "graceful-fs": "^4.2.0", 548 | "jsonfile": "^6.0.1", 549 | "universalify": "^2.0.0" 550 | }, 551 | "engines": { 552 | "node": ">=14.14" 553 | } 554 | }, 555 | "node_modules/get-caller-file": { 556 | "version": "2.0.5", 557 | "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", 558 | "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", 559 | "license": "ISC", 560 | "engines": { 561 | "node": "6.* || 8.* || >= 10.*" 562 | } 563 | }, 564 | "node_modules/get-stream": { 565 | "version": "5.2.0", 566 | "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", 567 | "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", 568 | "license": "MIT", 569 | "dependencies": { 570 | "pump": "^3.0.0" 571 | }, 572 | "engines": { 573 | "node": ">=8" 574 | }, 575 | "funding": { 576 | "url": "https://github.com/sponsors/sindresorhus" 577 | } 578 | }, 579 | "node_modules/get-uri": { 580 | "version": "6.0.3", 581 | "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.3.tgz", 582 | "integrity": "sha512-BzUrJBS9EcUb4cFol8r4W3v1cPsSyajLSthNkz5BxbpDcHN5tIrM10E2eNvfnvBn3DaT3DUgx0OpsBKkaOpanw==", 583 | "license": "MIT", 584 | "dependencies": { 585 | "basic-ftp": "^5.0.2", 586 | "data-uri-to-buffer": "^6.0.2", 587 | "debug": "^4.3.4", 588 | "fs-extra": "^11.2.0" 589 | }, 590 | "engines": { 591 | "node": ">= 14" 592 | } 593 | }, 594 | "node_modules/graceful-fs": { 595 | "version": "4.2.11", 596 | "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", 597 | "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", 598 | "license": "ISC" 599 | }, 600 | "node_modules/has-flag": { 601 | "version": "3.0.0", 602 | "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", 603 | "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", 604 | "license": "MIT", 605 | "engines": { 606 | "node": ">=4" 607 | } 608 | }, 609 | "node_modules/http-proxy-agent": { 610 | "version": "7.0.2", 611 | "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", 612 | "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", 613 | "license": "MIT", 614 | "dependencies": { 615 | "agent-base": "^7.1.0", 616 | "debug": "^4.3.4" 617 | }, 618 | "engines": { 619 | "node": ">= 14" 620 | } 621 | }, 622 | "node_modules/https-proxy-agent": { 623 | "version": "7.0.5", 624 | "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", 625 | "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", 626 | "license": "MIT", 627 | "dependencies": { 628 | "agent-base": "^7.0.2", 629 | "debug": "4" 630 | }, 631 | "engines": { 632 | "node": ">= 14" 633 | } 634 | }, 635 | "node_modules/ieee754": { 636 | "version": "1.2.1", 637 | "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", 638 | "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", 639 | "funding": [ 640 | { 641 | "type": "github", 642 | "url": "https://github.com/sponsors/feross" 643 | }, 644 | { 645 | "type": "patreon", 646 | "url": "https://www.patreon.com/feross" 647 | }, 648 | { 649 | "type": "consulting", 650 | "url": "https://feross.org/support" 651 | } 652 | ], 653 | "license": "BSD-3-Clause" 654 | }, 655 | "node_modules/import-fresh": { 656 | "version": "3.3.0", 657 | "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", 658 | "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", 659 | "license": "MIT", 660 | "dependencies": { 661 | "parent-module": "^1.0.0", 662 | "resolve-from": "^4.0.0" 663 | }, 664 | "engines": { 665 | "node": ">=6" 666 | }, 667 | "funding": { 668 | "url": "https://github.com/sponsors/sindresorhus" 669 | } 670 | }, 671 | "node_modules/ip-address": { 672 | "version": "9.0.5", 673 | "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", 674 | "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", 675 | "license": "MIT", 676 | "dependencies": { 677 | "jsbn": "1.1.0", 678 | "sprintf-js": "^1.1.3" 679 | }, 680 | "engines": { 681 | "node": ">= 12" 682 | } 683 | }, 684 | "node_modules/is-arrayish": { 685 | "version": "0.2.1", 686 | "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", 687 | "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", 688 | "license": "MIT" 689 | }, 690 | "node_modules/is-fullwidth-code-point": { 691 | "version": "3.0.0", 692 | "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", 693 | "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", 694 | "license": "MIT", 695 | "engines": { 696 | "node": ">=8" 697 | } 698 | }, 699 | "node_modules/js-tokens": { 700 | "version": "4.0.0", 701 | "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", 702 | "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", 703 | "license": "MIT" 704 | }, 705 | "node_modules/js-yaml": { 706 | "version": "4.1.0", 707 | "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", 708 | "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", 709 | "license": "MIT", 710 | "dependencies": { 711 | "argparse": "^2.0.1" 712 | }, 713 | "bin": { 714 | "js-yaml": "bin/js-yaml.js" 715 | } 716 | }, 717 | "node_modules/jsbn": { 718 | "version": "1.1.0", 719 | "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", 720 | "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==", 721 | "license": "MIT" 722 | }, 723 | "node_modules/json-parse-even-better-errors": { 724 | "version": "2.3.1", 725 | "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", 726 | "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", 727 | "license": "MIT" 728 | }, 729 | "node_modules/jsonfile": { 730 | "version": "6.1.0", 731 | "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", 732 | "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", 733 | "license": "MIT", 734 | "dependencies": { 735 | "universalify": "^2.0.0" 736 | }, 737 | "optionalDependencies": { 738 | "graceful-fs": "^4.1.6" 739 | } 740 | }, 741 | "node_modules/lines-and-columns": { 742 | "version": "1.2.4", 743 | "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", 744 | "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", 745 | "license": "MIT" 746 | }, 747 | "node_modules/lru-cache": { 748 | "version": "7.18.3", 749 | "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", 750 | "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", 751 | "license": "ISC", 752 | "engines": { 753 | "node": ">=12" 754 | } 755 | }, 756 | "node_modules/mitt": { 757 | "version": "3.0.1", 758 | "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", 759 | "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", 760 | "license": "MIT" 761 | }, 762 | "node_modules/ms": { 763 | "version": "2.1.2", 764 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", 765 | "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", 766 | "license": "MIT" 767 | }, 768 | "node_modules/netmask": { 769 | "version": "2.0.2", 770 | "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", 771 | "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", 772 | "license": "MIT", 773 | "engines": { 774 | "node": ">= 0.4.0" 775 | } 776 | }, 777 | "node_modules/once": { 778 | "version": "1.4.0", 779 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", 780 | "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", 781 | "license": "ISC", 782 | "dependencies": { 783 | "wrappy": "1" 784 | } 785 | }, 786 | "node_modules/pac-proxy-agent": { 787 | "version": "7.0.2", 788 | "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.2.tgz", 789 | "integrity": "sha512-BFi3vZnO9X5Qt6NRz7ZOaPja3ic0PhlsmCRYLOpN11+mWBCR6XJDqW5RF3j8jm4WGGQZtBA+bTfxYzeKW73eHg==", 790 | "license": "MIT", 791 | "dependencies": { 792 | "@tootallnate/quickjs-emscripten": "^0.23.0", 793 | "agent-base": "^7.0.2", 794 | "debug": "^4.3.4", 795 | "get-uri": "^6.0.1", 796 | "http-proxy-agent": "^7.0.0", 797 | "https-proxy-agent": "^7.0.5", 798 | "pac-resolver": "^7.0.1", 799 | "socks-proxy-agent": "^8.0.4" 800 | }, 801 | "engines": { 802 | "node": ">= 14" 803 | } 804 | }, 805 | "node_modules/pac-resolver": { 806 | "version": "7.0.1", 807 | "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", 808 | "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", 809 | "license": "MIT", 810 | "dependencies": { 811 | "degenerator": "^5.0.0", 812 | "netmask": "^2.0.2" 813 | }, 814 | "engines": { 815 | "node": ">= 14" 816 | } 817 | }, 818 | "node_modules/parent-module": { 819 | "version": "1.0.1", 820 | "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", 821 | "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", 822 | "license": "MIT", 823 | "dependencies": { 824 | "callsites": "^3.0.0" 825 | }, 826 | "engines": { 827 | "node": ">=6" 828 | } 829 | }, 830 | "node_modules/parse-json": { 831 | "version": "5.2.0", 832 | "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", 833 | "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", 834 | "license": "MIT", 835 | "dependencies": { 836 | "@babel/code-frame": "^7.0.0", 837 | "error-ex": "^1.3.1", 838 | "json-parse-even-better-errors": "^2.3.0", 839 | "lines-and-columns": "^1.1.6" 840 | }, 841 | "engines": { 842 | "node": ">=8" 843 | }, 844 | "funding": { 845 | "url": "https://github.com/sponsors/sindresorhus" 846 | } 847 | }, 848 | "node_modules/pend": { 849 | "version": "1.2.0", 850 | "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", 851 | "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", 852 | "license": "MIT" 853 | }, 854 | "node_modules/picocolors": { 855 | "version": "1.0.1", 856 | "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", 857 | "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==", 858 | "license": "ISC" 859 | }, 860 | "node_modules/progress": { 861 | "version": "2.0.3", 862 | "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", 863 | "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", 864 | "license": "MIT", 865 | "engines": { 866 | "node": ">=0.4.0" 867 | } 868 | }, 869 | "node_modules/proxy-agent": { 870 | "version": "6.4.0", 871 | "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.4.0.tgz", 872 | "integrity": "sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ==", 873 | "license": "MIT", 874 | "dependencies": { 875 | "agent-base": "^7.0.2", 876 | "debug": "^4.3.4", 877 | "http-proxy-agent": "^7.0.1", 878 | "https-proxy-agent": "^7.0.3", 879 | "lru-cache": "^7.14.1", 880 | "pac-proxy-agent": "^7.0.1", 881 | "proxy-from-env": "^1.1.0", 882 | "socks-proxy-agent": "^8.0.2" 883 | }, 884 | "engines": { 885 | "node": ">= 14" 886 | } 887 | }, 888 | "node_modules/proxy-from-env": { 889 | "version": "1.1.0", 890 | "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", 891 | "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", 892 | "license": "MIT" 893 | }, 894 | "node_modules/pump": { 895 | "version": "3.0.0", 896 | "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", 897 | "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", 898 | "license": "MIT", 899 | "dependencies": { 900 | "end-of-stream": "^1.1.0", 901 | "once": "^1.3.1" 902 | } 903 | }, 904 | "node_modules/puppeteer": { 905 | "version": "22.15.0", 906 | "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-22.15.0.tgz", 907 | "integrity": "sha512-XjCY1SiSEi1T7iSYuxS82ft85kwDJUS7wj1Z0eGVXKdtr5g4xnVcbjwxhq5xBnpK/E7x1VZZoJDxpjAOasHT4Q==", 908 | "hasInstallScript": true, 909 | "license": "Apache-2.0", 910 | "dependencies": { 911 | "@puppeteer/browsers": "2.3.0", 912 | "cosmiconfig": "^9.0.0", 913 | "devtools-protocol": "0.0.1312386", 914 | "puppeteer-core": "22.15.0" 915 | }, 916 | "bin": { 917 | "puppeteer": "lib/esm/puppeteer/node/cli.js" 918 | }, 919 | "engines": { 920 | "node": ">=18" 921 | } 922 | }, 923 | "node_modules/puppeteer-core": { 924 | "version": "22.15.0", 925 | "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-22.15.0.tgz", 926 | "integrity": "sha512-cHArnywCiAAVXa3t4GGL2vttNxh7GqXtIYGym99egkNJ3oG//wL9LkvO4WE8W1TJe95t1F1ocu9X4xWaGsOKOA==", 927 | "license": "Apache-2.0", 928 | "dependencies": { 929 | "@puppeteer/browsers": "2.3.0", 930 | "chromium-bidi": "0.6.3", 931 | "debug": "^4.3.6", 932 | "devtools-protocol": "0.0.1312386", 933 | "ws": "^8.18.0" 934 | }, 935 | "engines": { 936 | "node": ">=18" 937 | } 938 | }, 939 | "node_modules/queue-tick": { 940 | "version": "1.0.1", 941 | "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", 942 | "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==", 943 | "license": "MIT" 944 | }, 945 | "node_modules/require-directory": { 946 | "version": "2.1.1", 947 | "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", 948 | "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", 949 | "license": "MIT", 950 | "engines": { 951 | "node": ">=0.10.0" 952 | } 953 | }, 954 | "node_modules/resolve-from": { 955 | "version": "4.0.0", 956 | "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", 957 | "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", 958 | "license": "MIT", 959 | "engines": { 960 | "node": ">=4" 961 | } 962 | }, 963 | "node_modules/semver": { 964 | "version": "7.6.3", 965 | "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", 966 | "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", 967 | "license": "ISC", 968 | "bin": { 969 | "semver": "bin/semver.js" 970 | }, 971 | "engines": { 972 | "node": ">=10" 973 | } 974 | }, 975 | "node_modules/smart-buffer": { 976 | "version": "4.2.0", 977 | "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", 978 | "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", 979 | "license": "MIT", 980 | "engines": { 981 | "node": ">= 6.0.0", 982 | "npm": ">= 3.0.0" 983 | } 984 | }, 985 | "node_modules/socks": { 986 | "version": "2.8.3", 987 | "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", 988 | "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", 989 | "license": "MIT", 990 | "dependencies": { 991 | "ip-address": "^9.0.5", 992 | "smart-buffer": "^4.2.0" 993 | }, 994 | "engines": { 995 | "node": ">= 10.0.0", 996 | "npm": ">= 3.0.0" 997 | } 998 | }, 999 | "node_modules/socks-proxy-agent": { 1000 | "version": "8.0.4", 1001 | "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.4.tgz", 1002 | "integrity": "sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==", 1003 | "license": "MIT", 1004 | "dependencies": { 1005 | "agent-base": "^7.1.1", 1006 | "debug": "^4.3.4", 1007 | "socks": "^2.8.3" 1008 | }, 1009 | "engines": { 1010 | "node": ">= 14" 1011 | } 1012 | }, 1013 | "node_modules/source-map": { 1014 | "version": "0.6.1", 1015 | "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", 1016 | "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", 1017 | "license": "BSD-3-Clause", 1018 | "optional": true, 1019 | "engines": { 1020 | "node": ">=0.10.0" 1021 | } 1022 | }, 1023 | "node_modules/sprintf-js": { 1024 | "version": "1.1.3", 1025 | "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", 1026 | "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", 1027 | "license": "BSD-3-Clause" 1028 | }, 1029 | "node_modules/streamx": { 1030 | "version": "2.18.0", 1031 | "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.18.0.tgz", 1032 | "integrity": "sha512-LLUC1TWdjVdn1weXGcSxyTR3T4+acB6tVGXT95y0nGbca4t4o/ng1wKAGTljm9VicuCVLvRlqFYXYy5GwgM7sQ==", 1033 | "license": "MIT", 1034 | "dependencies": { 1035 | "fast-fifo": "^1.3.2", 1036 | "queue-tick": "^1.0.1", 1037 | "text-decoder": "^1.1.0" 1038 | }, 1039 | "optionalDependencies": { 1040 | "bare-events": "^2.2.0" 1041 | } 1042 | }, 1043 | "node_modules/string-width": { 1044 | "version": "4.2.3", 1045 | "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", 1046 | "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", 1047 | "license": "MIT", 1048 | "dependencies": { 1049 | "emoji-regex": "^8.0.0", 1050 | "is-fullwidth-code-point": "^3.0.0", 1051 | "strip-ansi": "^6.0.1" 1052 | }, 1053 | "engines": { 1054 | "node": ">=8" 1055 | } 1056 | }, 1057 | "node_modules/strip-ansi": { 1058 | "version": "6.0.1", 1059 | "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", 1060 | "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", 1061 | "license": "MIT", 1062 | "dependencies": { 1063 | "ansi-regex": "^5.0.1" 1064 | }, 1065 | "engines": { 1066 | "node": ">=8" 1067 | } 1068 | }, 1069 | "node_modules/supports-color": { 1070 | "version": "5.5.0", 1071 | "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", 1072 | "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", 1073 | "license": "MIT", 1074 | "dependencies": { 1075 | "has-flag": "^3.0.0" 1076 | }, 1077 | "engines": { 1078 | "node": ">=4" 1079 | } 1080 | }, 1081 | "node_modules/tar-fs": { 1082 | "version": "3.0.6", 1083 | "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.6.tgz", 1084 | "integrity": "sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==", 1085 | "license": "MIT", 1086 | "dependencies": { 1087 | "pump": "^3.0.0", 1088 | "tar-stream": "^3.1.5" 1089 | }, 1090 | "optionalDependencies": { 1091 | "bare-fs": "^2.1.1", 1092 | "bare-path": "^2.1.0" 1093 | } 1094 | }, 1095 | "node_modules/tar-stream": { 1096 | "version": "3.1.7", 1097 | "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", 1098 | "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", 1099 | "license": "MIT", 1100 | "dependencies": { 1101 | "b4a": "^1.6.4", 1102 | "fast-fifo": "^1.2.0", 1103 | "streamx": "^2.15.0" 1104 | } 1105 | }, 1106 | "node_modules/text-decoder": { 1107 | "version": "1.1.1", 1108 | "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.1.1.tgz", 1109 | "integrity": "sha512-8zll7REEv4GDD3x4/0pW+ppIxSNs7H1J10IKFZsuOMscumCdM2a+toDGLPA3T+1+fLBql4zbt5z83GEQGGV5VA==", 1110 | "license": "Apache-2.0", 1111 | "dependencies": { 1112 | "b4a": "^1.6.4" 1113 | } 1114 | }, 1115 | "node_modules/through": { 1116 | "version": "2.3.8", 1117 | "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", 1118 | "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", 1119 | "license": "MIT" 1120 | }, 1121 | "node_modules/tslib": { 1122 | "version": "2.6.3", 1123 | "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", 1124 | "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==", 1125 | "license": "0BSD" 1126 | }, 1127 | "node_modules/unbzip2-stream": { 1128 | "version": "1.4.3", 1129 | "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", 1130 | "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", 1131 | "license": "MIT", 1132 | "dependencies": { 1133 | "buffer": "^5.2.1", 1134 | "through": "^2.3.8" 1135 | } 1136 | }, 1137 | "node_modules/undici-types": { 1138 | "version": "6.13.0", 1139 | "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.13.0.tgz", 1140 | "integrity": "sha512-xtFJHudx8S2DSoujjMd1WeWvn7KKWFRESZTMeL1RptAYERu29D6jphMjjY+vn96jvN3kVPDNxU/E13VTaXj6jg==", 1141 | "license": "MIT", 1142 | "optional": true 1143 | }, 1144 | "node_modules/universalify": { 1145 | "version": "2.0.1", 1146 | "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", 1147 | "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", 1148 | "license": "MIT", 1149 | "engines": { 1150 | "node": ">= 10.0.0" 1151 | } 1152 | }, 1153 | "node_modules/urlpattern-polyfill": { 1154 | "version": "10.0.0", 1155 | "resolved": "https://registry.npmjs.org/urlpattern-polyfill/-/urlpattern-polyfill-10.0.0.tgz", 1156 | "integrity": "sha512-H/A06tKD7sS1O1X2SshBVeA5FLycRpjqiBeqGKmBwBDBy28EnRjORxTNe269KSSr5un5qyWi1iL61wLxpd+ZOg==", 1157 | "license": "MIT" 1158 | }, 1159 | "node_modules/wrap-ansi": { 1160 | "version": "7.0.0", 1161 | "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", 1162 | "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", 1163 | "license": "MIT", 1164 | "dependencies": { 1165 | "ansi-styles": "^4.0.0", 1166 | "string-width": "^4.1.0", 1167 | "strip-ansi": "^6.0.0" 1168 | }, 1169 | "engines": { 1170 | "node": ">=10" 1171 | }, 1172 | "funding": { 1173 | "url": "https://github.com/chalk/wrap-ansi?sponsor=1" 1174 | } 1175 | }, 1176 | "node_modules/wrap-ansi/node_modules/ansi-styles": { 1177 | "version": "4.3.0", 1178 | "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", 1179 | "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", 1180 | "license": "MIT", 1181 | "dependencies": { 1182 | "color-convert": "^2.0.1" 1183 | }, 1184 | "engines": { 1185 | "node": ">=8" 1186 | }, 1187 | "funding": { 1188 | "url": "https://github.com/chalk/ansi-styles?sponsor=1" 1189 | } 1190 | }, 1191 | "node_modules/wrap-ansi/node_modules/color-convert": { 1192 | "version": "2.0.1", 1193 | "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", 1194 | "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", 1195 | "license": "MIT", 1196 | "dependencies": { 1197 | "color-name": "~1.1.4" 1198 | }, 1199 | "engines": { 1200 | "node": ">=7.0.0" 1201 | } 1202 | }, 1203 | "node_modules/wrap-ansi/node_modules/color-name": { 1204 | "version": "1.1.4", 1205 | "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", 1206 | "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", 1207 | "license": "MIT" 1208 | }, 1209 | "node_modules/wrappy": { 1210 | "version": "1.0.2", 1211 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", 1212 | "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", 1213 | "license": "ISC" 1214 | }, 1215 | "node_modules/ws": { 1216 | "version": "8.18.0", 1217 | "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", 1218 | "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", 1219 | "license": "MIT", 1220 | "engines": { 1221 | "node": ">=10.0.0" 1222 | }, 1223 | "peerDependencies": { 1224 | "bufferutil": "^4.0.1", 1225 | "utf-8-validate": ">=5.0.2" 1226 | }, 1227 | "peerDependenciesMeta": { 1228 | "bufferutil": { 1229 | "optional": true 1230 | }, 1231 | "utf-8-validate": { 1232 | "optional": true 1233 | } 1234 | } 1235 | }, 1236 | "node_modules/y18n": { 1237 | "version": "5.0.8", 1238 | "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", 1239 | "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", 1240 | "license": "ISC", 1241 | "engines": { 1242 | "node": ">=10" 1243 | } 1244 | }, 1245 | "node_modules/yargs": { 1246 | "version": "17.7.2", 1247 | "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", 1248 | "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", 1249 | "license": "MIT", 1250 | "dependencies": { 1251 | "cliui": "^8.0.1", 1252 | "escalade": "^3.1.1", 1253 | "get-caller-file": "^2.0.5", 1254 | "require-directory": "^2.1.1", 1255 | "string-width": "^4.2.3", 1256 | "y18n": "^5.0.5", 1257 | "yargs-parser": "^21.1.1" 1258 | }, 1259 | "engines": { 1260 | "node": ">=12" 1261 | } 1262 | }, 1263 | "node_modules/yargs-parser": { 1264 | "version": "21.1.1", 1265 | "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", 1266 | "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", 1267 | "license": "ISC", 1268 | "engines": { 1269 | "node": ">=12" 1270 | } 1271 | }, 1272 | "node_modules/yauzl": { 1273 | "version": "2.10.0", 1274 | "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", 1275 | "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", 1276 | "license": "MIT", 1277 | "dependencies": { 1278 | "buffer-crc32": "~0.2.3", 1279 | "fd-slicer": "~1.1.0" 1280 | } 1281 | }, 1282 | "node_modules/zod": { 1283 | "version": "3.23.8", 1284 | "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", 1285 | "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", 1286 | "license": "MIT", 1287 | "funding": { 1288 | "url": "https://github.com/sponsors/colinhacks" 1289 | } 1290 | } 1291 | } 1292 | } 1293 | -------------------------------------------------------------------------------- /web-scraper-node/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "web-scraper-node", 3 | "version": "1.0.0", 4 | "main": "index.js", 5 | "type": "module", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "keywords": [], 10 | "author": "", 11 | "license": "ISC", 12 | "description": "", 13 | "dependencies": { 14 | "puppeteer": "^22.15.0" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /web-scraper-node/scrape.js: -------------------------------------------------------------------------------- 1 | import puppeteer from 'puppeteer'; 2 | import fs from 'fs'; 3 | 4 | const scrape = async () => { 5 | const browser = await puppeteer.launch(); 6 | const page = await browser.newPage(); 7 | 8 | const allBooks = []; 9 | let currentPage = 1; 10 | const maxPages = 10; 11 | 12 | while (currentPage <= maxPages) { 13 | const url = `https://books.toscrape.com/catalogue/page-${currentPage}.html`; 14 | 15 | await page.goto(url); 16 | 17 | const books = await page.evaluate(() => { 18 | const bookElements = document.querySelectorAll('.product_pod'); 19 | return Array.from(bookElements).map((book) => { 20 | const title = book.querySelector('h3 a').getAttribute('title'); 21 | const price = book.querySelector('.price_color').textContent; 22 | const stock = book.querySelector('.instock.availability') 23 | ? 'In Stock' 24 | : 'Out Of Stock'; 25 | const rating = book 26 | .querySelector('.star-rating') 27 | .className.split(' ')[1]; 28 | const link = book.querySelector('h3 a').getAttribute('href'); 29 | 30 | return { 31 | title, 32 | price, 33 | stock, 34 | rating, 35 | link, 36 | }; 37 | }); 38 | }); 39 | 40 | allBooks.push(...books); 41 | console.log(`Books on page ${currentPage}: `, books); 42 | currentPage++; 43 | } 44 | 45 | fs.writeFileSync('books.json', JSON.stringify(allBooks, null, 2)); 46 | 47 | console.log('Data saved to books.json'); 48 | 49 | await browser.close(); 50 | }; 51 | 52 | scrape(); 53 | -------------------------------------------------------------------------------- /web-scraper-python/.gitignore: -------------------------------------------------------------------------------- 1 | env -------------------------------------------------------------------------------- /web-scraper-python/requirements.txt: -------------------------------------------------------------------------------- 1 | beautifulsoup4==4.12.3 2 | certifi==2024.7.4 3 | charset-normalizer==3.3.2 4 | idna==3.7 5 | requests==2.32.3 6 | soupsieve==2.5 7 | urllib3==2.2.2 8 | -------------------------------------------------------------------------------- /web-scraper-python/scrape.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from bs4 import BeautifulSoup 3 | import json 4 | 5 | def fetch_books(page_number): 6 | url = f"https://books.toscrape.com/catalogue/page-{page_number}.html" 7 | response = requests.get(url) 8 | soup = BeautifulSoup(response.text, 'html.parser') 9 | 10 | books = [] 11 | book_elements = soup.find_all('article', class_='product_pod') 12 | 13 | for book in book_elements: 14 | title = book.find('h3').find('a')['title'] 15 | price = book.find('p', class_='price_color').text 16 | stock = 'In stock' if 'In stock' in book.find('p', class_='instock availability').text else 'Out of stock' 17 | rating = book.find('p', class_='star-rating')['class'][1] 18 | link = book.find('h3').find('a')['href'] 19 | 20 | books.append({ 21 | 'title': title, 22 | 'price': price, 23 | 'stock': stock, 24 | 'rating': rating, 25 | 'link': f"https://books.toscrape.com/catalogue/{link}" 26 | }) 27 | 28 | return books 29 | 30 | def main(): 31 | all_books = [] 32 | max_pages = 10 33 | 34 | for current_page in range(1, max_pages + 1): 35 | books_on_page = fetch_books(current_page) 36 | all_books.extend(books_on_page) 37 | print(f"Books on page {current_page}: {books_on_page}") 38 | 39 | # Save data to file 40 | with open('books.json', 'w') as f: 41 | json.dump(all_books, f, indent=2) 42 | 43 | print('Data is saved to books.json') 44 | 45 | if __name__ == "__main__": 46 | main() --------------------------------------------------------------------------------