├── .env-cmdrc.json ├── .eslintrc.json ├── .gitignore ├── .vscode ├── launch.json └── tasks.json ├── LICENSE ├── README.md ├── contributing └── development.md ├── jest.babelrc ├── jest.config.js ├── package.json ├── src ├── index.ts ├── stealthy-scraper │ ├── helpers.ts │ ├── index.spec.ts │ ├── index.ts │ └── types.ts └── types.ts ├── tsconfig.all.json ├── tsconfig.dist.json ├── tsconfig.examples.json ├── tsconfig.integration-tests.json ├── tsconfig.json └── tsconfig.unit-tests.json /.env-cmdrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "dev": { 3 | "NODE_ENV": "development" 4 | }, 5 | "prod": { 6 | "NODE_ENV": "production" 7 | }, 8 | "integration-tests": { 9 | "NODE_ENV": "development", 10 | "DATABASE_NAME": "ts-pg-orm-integration-tests", 11 | "DATABASE_USER": "postgres", 12 | "DATABASE_PASSWORD": "postgres", 13 | "DATABASE_HOST": "localhost", 14 | "DATABASE_PORT": "5432" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": true, 4 | "es2021": true 5 | }, 6 | "extends": [ 7 | "airbnb" 8 | ], 9 | "ignorePatterns": [ 10 | "node_modules/**/*", 11 | "build/**/*", 12 | "build-test/**/*", 13 | "dist**/*", 14 | "package.lock.json", 15 | "LICENSE", 16 | "*.yaml", 17 | "*.go", 18 | "*.sh", 19 | "*.md", 20 | "*.sql", 21 | "*.json", 22 | "*.scss", 23 | "*.html", 24 | "*.ico", 25 | "*.ttf", 26 | "*.woff2", 27 | "*.txt", 28 | "*.env", 29 | "*.md", 30 | "*.bat" 31 | ], 32 | "parser": "@typescript-eslint/parser", 33 | "parserOptions": { 34 | "ecmaVersion": 13, 35 | "sourceType": "module" 36 | }, 37 | "plugins": [ 38 | "@typescript-eslint" 39 | ], 40 | "settings": { 41 | "import/resolver": { 42 | "node": { 43 | "extensions": [".ts", ".js"] 44 | } 45 | } 46 | }, 47 | "rules": { 48 | "semi": [2, "never"], 49 | "prefer-destructuring": "off", 50 | "import/extensions": "off", 51 | "radix": "off", 52 | "object-curly-newline": "off", 53 | "comma-dangle": ["error", "always-multiline"], 54 | "nonblock-statement-body-position": ["error", "any"], 55 | "curly": ["error", "multi-or-nest", "consistent"], 56 | "max-len": ["error", { "code": 150, "ignoreTemplateLiterals": true }], 57 | "@typescript-eslint/no-unused-vars": "warn", 58 | "no-undef": "off", 59 | "no-unused-vars": "off", 60 | "arrow-parens": [2, "as-needed"], 61 | "no-return-assign": [0], 62 | "brace-style": [2, "stroustrup"], 63 | "no-underscore-dangle": [0], 64 | "no-nested-ternary": [0], 65 | "import/prefer-default-export": "off", 66 | "no-use-before-define": "off", 67 | "@typescript-eslint/no-use-before-define": ["error"], 68 | "no-shadow": "off", 69 | "@typescript-eslint/no-shadow": "error", 70 | "class-methods-use-this": "off", 71 | "no-useless-constructor": "off", 72 | "no-empty-function": "off", 73 | "lines-between-class-members": "off", 74 | "no-param-reassign": "warn", 75 | "object-shorthand": ["error", "properties"], 76 | "space-before-function-paren": "off" 77 | } 78 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | /build 3 | /dist 4 | /build-test 5 | /build-ts 6 | /build-integration-test 7 | .DS_Store 8 | /debug.log 9 | /package-lock.json 10 | /build-examples 11 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "name": "Run Unit Tests", 5 | "type": "node", 6 | "request": "launch", 7 | "program": "${workspaceRoot}\\node_modules\\jest\\bin\\jest.js", 8 | "args": [ 9 | "--runInBand", 10 | "--no-cache" 11 | ], 12 | "preLaunchTask": "unitTestsDebugPreLaunchTask" 13 | }, 14 | { 15 | "name": "Run Integration Tests", 16 | "type": "node", 17 | "request": "launch", 18 | "program": "./build-integration-test/integrationTests/index.js", 19 | "args": [ 20 | "--runInBand", 21 | "--no-cache" 22 | ], 23 | "env": { 24 | "DATABASE_NAME": "ts-pg-orm-integration-tests", 25 | "DATABASE_USER": "postgres", 26 | "DATABASE_PASSWORD": "postgres", 27 | "DATABASE_HOST": "localhost", 28 | "DATABASE_PORT": "5432" 29 | }, 30 | "preLaunchTask": "IntegrationTestsDebugPreLaunchTask" 31 | }, 32 | ] 33 | } -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "tasks": [ 3 | { 4 | "label": "unitTestsDebugPreLaunchTask", 5 | "type": "shell", 6 | "command": "npm run build-unit-tests", 7 | "presentation": { 8 | "reveal": "silent", 9 | "revealProblems": "onProblem", 10 | "close": true 11 | } 12 | }, 13 | { 14 | "label": "IntegrationTestsDebugPreLaunchTask", 15 | "type": "shell", 16 | "command": "npm run build-integration-tests", 17 | "presentation": { 18 | "reveal": "silent", 19 | "revealProblems": "onProblem", 20 | "close": true 21 | } 22 | }, 23 | ] 24 | } 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 - 2022 Sam Hughes 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

stealthy-scraper

2 |

3 | Extra stealthy web scraper in Typescript 4 |

5 | 6 |

7 | 8 | license 9 | 10 | 11 | npm version 12 | 13 |

14 | 15 | ## Overview 16 | 17 | stealthy-scraper is a wrapper around [puppeteer-extra](https://github.com/berstend/puppeteer-extra) that adds additional stealth functionality and other helpful features. 18 | 19 | ## Why to use 20 | 21 | * If puppeteer's `Page.goto` and `Browser.newPage` is being detected. stealthy-scraper has a `newBrowser` function as an alternative way to navigate to a new url which is more reliable. 22 | * If puppeteer's default word typing is being detected. stealthy-scraper has a `safeType` function that better mimicks human typing behavior. 23 | * When you want to more neatly centralize all of the puppeteer, puppeteer-extra, and puppeteer-extra's plugin dependencies into one package. 24 | 25 | ## Usage Overview 26 | 27 | `npm i --save stealthy-scraper` 28 | 29 | ```typescript 30 | import { createScraper } from 'stealthy-scraper' 31 | const scraper = await createScraper({ 32 | puppeteerOptions: { 33 | headless: true, 34 | ... 35 | }, 36 | snapshotsDirPath: './scraper-snapshots', 37 | }) 38 | await scraper.page.goto('difficultoscrape.com') 39 | const searchTextInput = await scraper.page.waitForSelector('...') 40 | await scraper.safeType(searchTextInput, 'my search term') 41 | // ... 42 | await scraper.newBrowser(newUrlFromSearchResults) 43 | await scraper.close() 44 | ``` 45 | 46 | ## Development 47 | 48 | See [./contributing/development.md](./contributing/development.md) 49 | 50 | ## Disclaimer 51 | 52 | I do not condone the usage of this package for malevolent purposes. Please be very curtious and a good citizen when using it. I do not take any responsibility for any damages you incur on yourself (e.g. IP blacklisted) or others (e.g. DoS) through any use of this package. 53 | 54 | --- 55 | 56 | If you found this package delightful, feel free to buy me a coffee ✨ 57 | 58 | Buy Me A Coffee 59 | -------------------------------------------------------------------------------- /contributing/development.md: -------------------------------------------------------------------------------- 1 | # Development 2 | 3 | This document describes the process for setting up and running this package on your local computer. 4 | 5 | ## Prerequisites 6 | 7 | This package is for Node.js. 8 | 9 | You can install Node.js [here](https://nodejs.org/en/), or if you are using Chocolatey run `choco install nodejs`. 10 | 11 | It runs on MacOS, Windows, and Linux environments. 12 | 13 | It runs on many versions of Node.js, tested back to version 12.x. 14 | 15 | ## Linting 16 | 17 | ESLint is used for Typescript linting. To lint the Typescript code, run `npm run lint`. To only lint for errors (excluding warnings), run `npm run lint-errors-only`. 18 | 19 | ## Building 20 | 21 | To build all of the Typescript code, run `npm run build-ts`. 22 | 23 | To build the Typescript code that is published to npm, run `npm run build-ts-dist`. 24 | 25 | ## Testing 26 | 27 | ### Unit Tests 28 | 29 | Jest is used for unit testing. To build and run the unit tests, run `npm run unit-tests`. 30 | 31 | The unit tests can be debugged with Visual Studio Code by running the **Run Unit Tests** debug task. 32 | 33 | ### Integration Tests 34 | 35 | Integration tests connect to a real PostgreSQL server and use ts-pg-orm as a user would. 36 | 37 | The database connectivity configuration is at `.env-cmdrc.json`. 38 | 39 | To run the integration tests, run `npm run integration-tests`. 40 | 41 | The integration tests can be debugged with Visual Studio Code by running the **Run Integration Tests** debug task. 42 | 43 | ## Miscellaneous Scripts 44 | 45 | `npm run check` - Useful to run before committing to check the full validity of a change. This runs linting, Typescript build, unit tests and integration tests. 46 | 47 | ## Pull Requests 48 | 49 | Pull requests automatically run a CI pipeline that checks various criteria: 50 | 51 | * Linting 52 | * Typescript build 53 | * Unit tests 54 | * Integration tests 55 | 56 | These must pass for a pull request to be approved and merged. 57 | 58 | ## NPM Publishing 59 | 60 | Run `npm run check` to check local validity of code. 61 | 62 | Run `npm run build-ts-dist` to build the Typescript code to produce the Javascript code that is published to npm. 63 | 64 | Run `npm publish` to publish to npm. -------------------------------------------------------------------------------- /jest.babelrc: -------------------------------------------------------------------------------- 1 | // This is used for jest unit tests to make breakpoints not move. 2 | // Source: https://stackoverflow.com/questions/52191409 3 | { 4 | "sourceMap": "inline", 5 | "retainLines": true 6 | } -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('ts-jest/dist/types').InitialOptionsTsJest} */ 2 | module.exports = { 3 | testEnvironment: 'node', 4 | rootDir: './build-test', 5 | transform: { 6 | '\\.[jt]sx?$': ['babel-jest', { configFile: './jest.babelrc' }], 7 | }, 8 | verbose: true, 9 | collectCoverage: true, 10 | coverageReporters: ['text', 'cobertura'], 11 | } 12 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "stealthy-scraper", 3 | "version": "1.0.4", 4 | "description": "Extra stealthy web scraper in Typescript", 5 | "main": "dist/index.js", 6 | "types": "dist/index.d.ts", 7 | "files": [ 8 | "dist/**/*" 9 | ], 10 | "scripts": { 11 | "clean-unit-tests": "rimraf ./build-test", 12 | "build-unit-tests": "tsc -p ./tsconfig.unit-tests.json", 13 | "run-unit-tests": "jest", 14 | "unit-tests": "npm run clean-unit-tests && npm run build-unit-tests && npm run run-unit-tests", 15 | "clean-integration-tests": "rimraf ./build-integration-test", 16 | "build-integration-tests": "tsc -p ./tsconfig.integration-tests.json", 17 | "run-integration-tests-ci": "node ./build-integration-test/integrationTests/index.js", 18 | "run-integration-tests": "env-cmd -e integration-tests node ./build-integration-test/integrationTests/index.js", 19 | "integration-tests-ci": "npm run clean-integration-tests && npm run build-integration-tests && npm run run-integration-tests-ci", 20 | "integration-tests": "npm run clean-integration-tests && npm run build-integration-tests && npm run run-integration-tests", 21 | "clean-tests": "npm run clean-unit-tests && npm run clean-integration-tests", 22 | "build-tests": "npm run build-unit-tests && npm run build-integration-tests", 23 | "run-tests": "npm run run-unit-tests && npm run run-integration-tests", 24 | "run-tests-ci": "npm run run-unit-tests && npm run run-integration-tests-ci", 25 | "tests": "npm run clean-tests && npm run build-tests && npm run run-tests", 26 | "tests-ci": "npm run clean-tests && npm run build-tests && npm run run-tests-ci", 27 | "lint": "eslint -c .eslintrc.json ./src --ext .ts", 28 | "lint-errors-only": "eslint -c .eslintrc.json ./src --ext .ts --quiet", 29 | "clean-ts-dist": "rimraf ./dist", 30 | "build-ts-dist": "npm run clean-ts-dist && tsc -p ./tsconfig.dist.json", 31 | "clean-ts": "rimraf ./build-ts", 32 | "build-ts": "npm run clean-ts && tsc -p ./tsconfig.all.json", 33 | "clean-examples": "rimraf ./build-examples", 34 | "build-examples": "tsc -p ./tsconfig.examples.json", 35 | "clean": "npm run clean-unit-tests && npm run clean-integration-tests && npm run clean-ts && npm run clean-ts-dist && npm run clean-examples", 36 | "check": "npm rum lint-errors-only && npm run build-ts && npm run tests && echo Done!" 37 | }, 38 | "repository": "https://github.com/samhuk/stealthy-scraper", 39 | "author": "", 40 | "license": "MIT", 41 | "publishConfig": { 42 | "access": "public" 43 | }, 44 | "dependencies": { 45 | "puppeteer": "^19.0.0", 46 | "puppeteer-extra": "^3.3.4", 47 | "puppeteer-extra-plugin-adblocker": "^2.13.5", 48 | "puppeteer-extra-plugin-stealth": "^2.11.1" 49 | }, 50 | "devDependencies": { 51 | "@types/jest": "^29.1.2", 52 | "@types/node": "^18.8.3", 53 | "@types/pg": "^8.6.5", 54 | "@typescript-eslint/eslint-plugin": "^5.40.0", 55 | "@typescript-eslint/parser": "^5.40.0", 56 | "babel-jest": "^29.1.2", 57 | "concurrently": "^7.4.0", 58 | "env-cmd": "^10.1.0", 59 | "eslint": "^8.25.0", 60 | "eslint-config-airbnb": "^19.0.4", 61 | "eslint-plugin-import": "^2.26.0", 62 | "eslint-plugin-jsx-a11y": "^6.6.1", 63 | "eslint-plugin-react": "^7.31.9", 64 | "jest": "^29.1.2", 65 | "rimraf": "^3.0.2", 66 | "ts-jest": "^29.0.3", 67 | "typescript": "^4.8.4" 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This file defines the public API of the package. Everything here will be available from 3 | * the top-level package name when importing as an npm package. 4 | * 5 | * E.g. `import { createPackageName, PackageNameOptions } from 'npm-package-name` 6 | */ 7 | 8 | export { createScraper } from './stealthy-scraper' 9 | export * from './types' 10 | -------------------------------------------------------------------------------- /src/stealthy-scraper/helpers.ts: -------------------------------------------------------------------------------- 1 | export const loop = ( 2 | fn: (next: (delayMs?: number) => void, stop: () => void, i: number) => void, 3 | ): (() => void) => { 4 | let currentTimeout: NodeJS.Timeout = null 5 | let stopped = false 6 | const stop = () => { 7 | stopped = true 8 | if (currentTimeout != null) 9 | clearTimeout(currentTimeout) 10 | } 11 | const next = (i: number) => fn(delayMs => { 12 | if (!stopped) { 13 | if (delayMs == null) 14 | next(i + 1) 15 | else 16 | currentTimeout = setTimeout(() => next(i + 1), delayMs) 17 | } 18 | }, stop, i) 19 | next(0) 20 | return stop 21 | } 22 | 23 | // eslint-disable-next-line no-promise-executor-return 24 | export const wait = (s: number) => new Promise(r => setTimeout(r, s * 1000)) 25 | -------------------------------------------------------------------------------- /src/stealthy-scraper/index.spec.ts: -------------------------------------------------------------------------------- 1 | import { createScraper } from '.' 2 | 3 | describe('stealthy-scraper', () => { 4 | describe('createScraper', () => { 5 | const fn = createScraper 6 | 7 | test('basic test', async () => { 8 | const scraper = await fn({ 9 | puppeteerOptions: { 10 | headless: false, 11 | }, 12 | snapshotsDirPath: './scraper-snapshots', 13 | }) 14 | 15 | await scraper.page.goto('http://www.example.com') 16 | 17 | const textEl = await scraper.page.waitForSelector('body>div>h1') 18 | const text = await textEl.evaluate(e => e.innerHTML) 19 | 20 | expect(text).toEqual('Example Domain') 21 | await scraper.close() 22 | }) 23 | }) 24 | }) 25 | -------------------------------------------------------------------------------- /src/stealthy-scraper/index.ts: -------------------------------------------------------------------------------- 1 | import * as fs from 'fs' 2 | import path from 'path' 3 | import puppeteer from 'puppeteer-extra' 4 | import StealthPlugin from 'puppeteer-extra-plugin-stealth' 5 | import AdblockerPlugin from 'puppeteer-extra-plugin-adblocker' 6 | import { ElementHandle, KeyInput } from 'puppeteer' 7 | import { SafeTypeOptions, Scraper, ScraperOptions } from './types' 8 | import { loop, wait } from './helpers' 9 | 10 | const AVERAGE_TYPING_CHARS_PER_MIN = 200 11 | const TYPING_CHARS_PER_MIN_VARIATION = 100 12 | 13 | const safeType = async (el: ElementHandle, text: string, options?: SafeTypeOptions) => { 14 | if (options?.clickFirst ?? true) 15 | await el.click() 16 | 17 | if (options?.waitBeforeS == null || options.waitBeforeS > 0) 18 | await wait(options?.waitBeforeS ?? 1) 19 | 20 | loop((next, stop, i) => { 21 | const char = text.charAt(i) as KeyInput 22 | if (char == null || char.length < 1) { 23 | stop() 24 | return 25 | } 26 | el.press(char).then(() => { 27 | // 200ms +/- 100ms 28 | const delayMs = AVERAGE_TYPING_CHARS_PER_MIN + (TYPING_CHARS_PER_MIN_VARIATION * ((2 * Math.random()) - 1)) 29 | next(delayMs) 30 | }) 31 | }) 32 | } 33 | 34 | export const createScraper = async (options?: ScraperOptions): Promise => { 35 | puppeteer.use(StealthPlugin()) 36 | puppeteer.use(AdblockerPlugin()) 37 | const initialBrowser = await puppeteer.launch(options?.puppeteerOptions) 38 | const initialPage = await initialBrowser.newPage() 39 | const snapshotsDirPath = options?.snapshotsDirPath ?? './' 40 | 41 | let scraper: Scraper 42 | return scraper = { 43 | browser: initialBrowser, 44 | page: initialPage, 45 | newBrowser: async (newUrl, newOptions) => { 46 | await scraper.browser.close() 47 | scraper.browser = await puppeteer.launch(newOptions ?? options?.puppeteerOptions) 48 | scraper.page = await scraper.browser.newPage() 49 | await scraper.page.goto(newUrl) 50 | }, 51 | snapshot: async name => { 52 | await scraper.page.screenshot({ path: path.resolve(snapshotsDirPath, name.concat('.png')), type: 'png' }) 53 | const pageHtml = await scraper.page.content() 54 | fs.writeFileSync(path.resolve(snapshotsDirPath, name.concat('.html')), pageHtml) 55 | }, 56 | safeType: async (el, text, _options) => { 57 | await safeType(el, text, _options) 58 | }, 59 | close: async () => { 60 | await scraper.browser.close() 61 | }, 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/stealthy-scraper/types.ts: -------------------------------------------------------------------------------- 1 | import { Browser, ElementHandle, Page, PuppeteerLaunchOptions } from 'puppeteer' 2 | 3 | export type ScraperOptions = { 4 | /** 5 | * Optional `PuppeteerLaunchOptions` to use to launch the browser. 6 | * 7 | * If not provided, puppeteer's default options are used. 8 | * 9 | * @default undefined 10 | */ 11 | puppeteerOptions?: PuppeteerLaunchOptions 12 | /** 13 | * Optional path to the directory used to store snapshot files created by `Scraper.snapshot`. 14 | * 15 | * @default './' 16 | */ 17 | snapshotsDirPath?: string 18 | } 19 | 20 | export type SafeTypeOptions = { 21 | /** 22 | * Determines if the element is to be clicked on first before typing, which is more 23 | * human-like behavior. 24 | */ 25 | clickFirst?: boolean 26 | /** 27 | * Seconds to wait before starting to type. 28 | * 29 | * @default 1 30 | */ 31 | waitBeforeS?: number 32 | } 33 | 34 | export type Scraper = { 35 | /** 36 | * The puppeteer `Browser`. 37 | */ 38 | browser: Browser 39 | /** 40 | * The puppeteer `Page`. 41 | */ 42 | page: Page 43 | /** 44 | * Opens up a new browser at the given url. 45 | * 46 | * This is useful when `Page.goto` or `Browser.newPage` doesn't work, as they can be detected 47 | * more easily compared to a completely new browser window. 48 | * 49 | * Optionally, new puppeteer options can be provided as an alternative to the original options 50 | * that were provided. 51 | */ 52 | newBrowser: (url: string, newPuppeteerOptions?: PuppeteerLaunchOptions) => Promise 53 | /** 54 | * Takes a png image of the current page and records the current html content of the page, 55 | * saving both files at the defined `snapshotsDir`. 56 | * 57 | * This is useful when scraping sites that prevent dev tools from being open, which is 58 | * un-unexploitable. This allows you to reach a point in your scraper script, take a snapshot, 59 | * and learn from the downloaded image and HTML what to do next. 60 | */ 61 | snapshot: (name: string) => Promise 62 | /** 63 | * Version of `ElementHandle.type` that better mimicks human typing input. 64 | */ 65 | safeType: (el: ElementHandle, text: string, options?: SafeTypeOptions) => Promise 66 | /** 67 | * Closes the browser. 68 | */ 69 | close: () => Promise 70 | } 71 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * This file defines all of the types that will be available in the public API 3 | * of the package. 4 | * 5 | * E.g. `import { PackageNameOptions } from 'npm-package-name` 6 | */ 7 | 8 | export type { Scraper, ScraperOptions } from './stealthy-scraper/types' 9 | -------------------------------------------------------------------------------- /tsconfig.all.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json.schemastore.org/tsconfig", 3 | "extends": "./tsconfig.json", 4 | "include": ["./src/**/*"], 5 | "compilerOptions": { 6 | "declaration": true, 7 | "sourceMap": true, 8 | "noEmit": false, 9 | "outDir": "./build-ts" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tsconfig.dist.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json.schemastore.org/tsconfig", 3 | "extends": "./tsconfig.json", 4 | "include": ["./src/index.ts"], 5 | "compilerOptions": { 6 | "declaration": true, 7 | "sourceMap": true, 8 | "noEmit": false, 9 | "outDir": "./dist" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tsconfig.examples.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json.schemastore.org/tsconfig", 3 | "extends": "./tsconfig.json", 4 | "include": [ 5 | "./src/examples/**/*" 6 | ], 7 | "compilerOptions": { 8 | "module": "CommonJS", 9 | "declaration": false, 10 | "sourceMap": true, 11 | "noEmit": false, 12 | "outDir": "./build-examples", 13 | "skipLibCheck": true 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /tsconfig.integration-tests.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json.schemastore.org/tsconfig", 3 | "extends": "./tsconfig.json", 4 | "include": [ 5 | "./src/integrationTests/**/*" 6 | ], 7 | "compilerOptions": { 8 | "declaration": false, 9 | "sourceMap": true, 10 | "noEmit": false, 11 | "outDir": "./build-integration-test" 12 | } 13 | } -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json.schemastore.org/tsconfig", 3 | "include": ["./src/**/*"], 4 | "compilerOptions": { 5 | "module": "CommonJS", 6 | "target": "es6", 7 | "moduleResolution": "node", 8 | "allowJs": false, 9 | "strictNullChecks": false, 10 | "noEmit": true, 11 | "strict": true, 12 | "isolatedModules": true, 13 | "esModuleInterop": true, 14 | "sourceMap": true 15 | } 16 | } -------------------------------------------------------------------------------- /tsconfig.unit-tests.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json.schemastore.org/tsconfig", 3 | "extends": "./tsconfig.json", 4 | "include": [ 5 | "./src/**/*.spec.ts" 6 | ], 7 | "exclude": [ 8 | "./src/integrationTests/**/*" 9 | ], 10 | "compilerOptions": { 11 | "declaration": false, 12 | "sourceMap": true, 13 | "noEmit": false, 14 | "outDir": "./build-test" 15 | } 16 | } 17 | --------------------------------------------------------------------------------