├── .github
    └── workflows
    │   └── npm.yml
├── .gitignore
├── .npmignore
├── .prettierrc
├── CHANGELOG.md
├── LICENSE.md
├── README.md
├── jest.config.js
├── logo.png
├── package-lock.json
├── package.json
├── src
    ├── index.ts
    ├── scraping-actions
    │   ├── AddCookies.ts
    │   ├── Click.ts
    │   ├── Press.ts
    │   ├── Remove.ts
    │   ├── Scroll.ts
    │   ├── Select.ts
    │   ├── Type.ts
    │   ├── Wait.ts
    │   └── index.ts
    ├── scraping-strategies
    │   ├── ScreenshotToMap.ts
    │   ├── TextContentScraping.ts
    │   └── index.ts
    ├── types.ts
    └── utils
    │   ├── browser-config.ts
    │   └── utils.ts
├── tests
    ├── scraper.test.ts
    ├── screenshotToMap.spec.ts
    └── templates
    │   ├── actions.html
    │   ├── pricing.css
    │   └── pricing.html
└── tsconfig.json


/.github/workflows/npm.yml:
--------------------------------------------------------------------------------
 1 | name: Publish to NPM
 2 | on:
 3 |   push:
 4 |     branches: [ main, master ]
 5 | 
 6 | jobs:
 7 |   test:
 8 |     runs-on: ubuntu-latest
 9 |     steps:
10 |       - uses: actions/checkout@v3
11 |       - uses: actions/setup-node@v3
12 |         with:
13 |           node-version: 16
14 |       - run: npm ci
15 |       - run: npm test
16 |   build:
17 |     needs: test
18 |     runs-on: ubuntu-latest
19 |     steps:
20 |       - uses: actions/checkout@v3
21 |       - uses: actions/setup-node@v3
22 |         with:
23 |           node-version: 16
24 |       - run: npm ci
25 |       - run: npm run build
26 |       - uses: JS-DevTools/npm-publish@v1
27 |         with:
28 |           token: ${{ secrets.NPM_TOKEN }}


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | # Logs
  2 | logs
  3 | *.log
  4 | npm-debug.log*
  5 | yarn-debug.log*
  6 | yarn-error.log*
  7 | lerna-debug.log*
  8 | .pnpm-debug.log*
  9 | 
 10 | # Diagnostic reports (https://nodejs.org/api/report.html)
 11 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
 12 | 
 13 | # Runtime data
 14 | pids
 15 | *.pid
 16 | *.seed
 17 | *.pid.lock
 18 | 
 19 | # Directory for instrumented libs generated by jscoverage/JSCover
 20 | lib-cov
 21 | 
 22 | # Coverage directory used by tools like istanbul
 23 | coverage
 24 | *.lcov
 25 | 
 26 | # nyc test coverage
 27 | .nyc_output
 28 | 
 29 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
 30 | .grunt
 31 | 
 32 | # Bower dependency directory (https://bower.io/)
 33 | bower_components
 34 | 
 35 | # node-waf configuration
 36 | .lock-wscript
 37 | 
 38 | # Compiled binary addons (https://nodejs.org/api/addons.html)
 39 | build/Release
 40 | 
 41 | # Dependency directories
 42 | node_modules/
 43 | jspm_packages/
 44 | 
 45 | # Snowpack dependency directory (https://snowpack.dev/)
 46 | web_modules/
 47 | 
 48 | # TypeScript cache
 49 | *.tsbuildinfo
 50 | 
 51 | # Optional npm cache directory
 52 | .npm
 53 | 
 54 | # Optional eslint cache
 55 | .eslintcache
 56 | 
 57 | # Optional stylelint cache
 58 | .stylelintcache
 59 | 
 60 | # Microbundle cache
 61 | .rpt2_cache/
 62 | .rts2_cache_cjs/
 63 | .rts2_cache_es/
 64 | .rts2_cache_umd/
 65 | 
 66 | # Optional REPL history
 67 | .node_repl_history
 68 | 
 69 | # Output of 'npm pack'
 70 | *.tgz
 71 | 
 72 | # Yarn Integrity file
 73 | .yarn-integrity
 74 | 
 75 | # dotenv environment variable files
 76 | .env
 77 | .env.development.local
 78 | .env.test.local
 79 | .env.production.local
 80 | .env.local
 81 | 
 82 | # parcel-bundler cache (https://parceljs.org/)
 83 | .cache
 84 | .parcel-cache
 85 | 
 86 | # Next.js build output
 87 | .next
 88 | out
 89 | 
 90 | # Nuxt.js build / generate output
 91 | .nuxt
 92 | dist
 93 | 
 94 | # Gatsby files
 95 | .cache/
 96 | # Comment in the public line in if your project uses Gatsby and not Next.js
 97 | # https://nextjs.org/blog/next-9-1#public-directory-support
 98 | # public
 99 | 
100 | # vuepress build output
101 | .vuepress/dist
102 | 
103 | # vuepress v2.x temp and cache directory
104 | .temp
105 | .cache
106 | 
107 | # Docusaurus cache and generated files
108 | .docusaurus
109 | 
110 | # Serverless directories
111 | .serverless/
112 | 
113 | # FuseBox cache
114 | .fusebox/
115 | 
116 | # DynamoDB Local files
117 | .dynamodb/
118 | 
119 | # TernJS port file
120 | .tern-port
121 | 
122 | # Stores VSCode versions used for testing VSCode extensions
123 | .vscode-test
124 | 
125 | # yarn v2
126 | .yarn/cache
127 | .yarn/unplugged
128 | .yarn/build-state.yml
129 | .yarn/install-state.gz
130 | .pnp.*
131 | 


--------------------------------------------------------------------------------
/.npmignore:
--------------------------------------------------------------------------------
1 | .vscode
2 | .github
3 | tests


--------------------------------------------------------------------------------
/.prettierrc:
--------------------------------------------------------------------------------
1 | {
2 |     "printWidth": 100,
3 |     "useTabs": true
4 |   }


--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
 1 | # Changelog
 2 | 
 3 | All notable changes to this project will be documented in this file.
 4 | 
 5 | ## [0.2.0] — 2022-06-13
 6 | 
 7 | ### Added
 8 | 
 9 | - Remove action: it removes an element from the DOM. It can be useful when dealing with consent pop-ups.
10 | 
11 | ## [0.1.0] — 2022-06-11
12 | 
13 | ### Added
14 | 
15 | - ScreenshotToMap strategy (retrieve screenshot and maps with selectors for the specified elements).
16 | 
17 | ### Changed
18 | 
19 | - You can now run the scraper without a strategy. It will only run the actions and return the playwright instances of Browser, BrowserContext, and Page.
20 | 
21 | ## [0.0.1] — 2022-06-10
22 | 
23 | ### Added
24 | 
25 | - First release
26 | 


--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2022 Alex Ferrari
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | ![ScrapeBlocks Logo](logo.png)
  2 | 
  3 | <div align="center">
  4 | 
  5 | [![MIT License](https://img.shields.io/apm/l/atomic-design-ui.svg?)](https://github.com/tterb/atomic-design-ui/blob/master/LICENSEs)
  6 | [![NPM Version](https://img.shields.io/npm/v/scrapeblocks?style=flat-square)](https://www.npmjs.com/package/scrapeblocks)
  7 | [![lgtm Code Quality](https://img.shields.io/lgtm/grade/javascript/github/alexferrari88/scrapeblocks?style=flat-square)](https://lgtm.com/projects/g/alexferrari88/scrapeblocks/)
  8 | [![GitHub Last Commit](https://img.shields.io/github/last-commit/alexferrari88/scrapeblocks?style=flat-square)](https://img.shields.io/github/last-commit/alexferrari88/scrapeblocks?style=flat-square)
  9 | 
 10 | </div>
 11 | 
 12 | # ScrapeBlocks
 13 | 
 14 | ScrapeBlocks is a layer on top of Playwright to make scraping automation easier.
 15 | 
 16 | You can set actions to be performed before starting scraping and you can also decide which scraping strategy to use.
 17 | 
 18 | Start with predefined actions and strategies in a matter of minutes. You can also write your own or use ones from the community.
 19 | 
 20 | ## Who is this for? 🤔
 21 | 
 22 | - I just want to start scraping right now with little effort as possible
 23 | - I have a complicated scraping workflow that I want to simplify with still getting the same results
 24 | - I like to tinker with scraping and build my custom workflows
 25 | 
 26 | With ScrapeBlocks getting started with scraping is a matter of minutes.
 27 | 
 28 | You can use it with its batteries included or as extension to Playwright.
 29 | 
 30 | Whether you are a scraping-hero or just want to monitor the price for that product but you don't know much about scraping, ScrapeBlocks is here for you.
 31 | 
 32 | ## Features 🚀
 33 | 
 34 | - **Pre-scraping actions**: perform actions before running a scraping strategy
 35 |   - Example use-case: you need to click something before your target becomes visible
 36 | - **Plug-n-play**: write your own scraping strategies or use those from the community
 37 |   - Example use-cases: scrape for text of certain elements, get all the images, etc.
 38 | - **Fully customizible (or not)**: you can use it batteries included or use your own Playwright instances
 39 | - **Easy to start with**: it's based on Playwright!
 40 | 
 41 | ### Actions included ⚡
 42 | 
 43 | - [Click on any element](src/scraping-actions/Click.ts)
 44 | - [Add cookie](src/scraping-actions/AddCookies.ts)
 45 | - [Remove an element](src/scraping-actions/Remove.ts)
 46 | - [Type anywhere you can type something](src/scraping-actions/Type.ts)
 47 | - [Press keyboard buttons (e.g. Enter, CTRL+C, etc.)](src/scraping-actions/Press.ts)
 48 | - [Scroll to bottom of the page](src/scraping-actions/Scroll.ts)
 49 | - [Wait a certain amount of time](src/scraping-actions/Wait.ts)
 50 | - [Select](src/scraping-actions/Select.ts) any option from a `<select>` element
 51 | - _(to be continued...)_
 52 | 
 53 | ### Strategies included 🧙🏼
 54 | 
 55 | - [Scrape text element](src/scraping-strategies/TextContentScraping.ts): retrieve the text within any element
 56 | - [Screenshot to map](src/scraping-strategies/ScreenshotToMap.ts): returns a screenshot of the page with a json with the coordinates and xpath/css selector for elements of your choice
 57 | - _(to be continued...)_
 58 | 
 59 | ## Installation 🔧
 60 | 
 61 | Install ScrapeBlocks with npm
 62 | 
 63 | ```bash
 64 |   npm install scrapeblocks
 65 | ```
 66 | 
 67 | ## Usage 🧑🏼‍💻
 68 | 
 69 | ### Using built-in Playwright
 70 | 
 71 | #### Basic textContent strategy
 72 | 
 73 | ```javascript
 74 | import { Scraper, ScrapingStragegies } from "scrapeblocks";
 75 | 
 76 | const URL = "https://webscraper.io/test-sites/e-commerce/allinone";
 77 | const selector = "h4.price";
 78 | 
 79 | const strategy = new ScrapingStragegies.TextContentScraping(selector);
 80 | const result = await new Scraper(URL, strategy).run();
 81 | 
 82 | console.log(result);
 83 | ```
 84 | 
 85 | Output:
 86 | 
 87 | ```bash
 88 | ['$233.99', '$603.99', '$295.99']
 89 | ```
 90 | 
 91 | #### With actions
 92 | 
 93 | ```javascript
 94 | import { Scraper, ScrapingStragegies, Select } from "scrapeblocks";
 95 | 
 96 | const URL = "https://webscraper.io/test-sites/e-commerce/more/product/488";
 97 | const selectElement = "div.dropdown > select";
 98 | const optionToSelect = "Gold";
 99 | const selector = "div.caption > h4:nth-child(2)";
100 | 
101 | const strategy = new ScrapingStragegies.TextContentScraping(selector);
102 | const selectAction = new Select({
103 | 	element: selectElement,
104 | 	value: optionToSelect,
105 | });
106 | const result = await new Scraper(URL, strategy, [selectAction]).run();
107 | 
108 | console.log(result);
109 | ```
110 | 
111 | Output:
112 | 
113 | ```bash
114 | Samsung Galaxy Gold
115 | ```
116 | 
117 | You can chain multiple actions by passing them in the order you want them to be executed as array.
118 | 
119 | Example:
120 | 
121 | ```javascript
122 | const actions = [scrollAction, clickAction, typeAction];
123 | ```
124 | 
125 | Starting from version 0.1.0, you can also just execute actions without providing any strategy.
126 | 
127 | The method will return instances of Playwright Browser, BrowserContext, Page.
128 | 
129 | Example:
130 | 
131 | <!-- prettier-ignore -->
132 | ```javascript
133 | const { browser, context, page } =
134 |         await new Scraper<PlaywrightBlocks>(
135 |           URL, undefined, [clickAction,]
136 |           ).run();
137 | ```
138 | 
139 | ## TODO ✅
140 | 
141 | - Implement more strategies
142 | 
143 | - Implement more actions
144 | 
145 | - Increase test cases
146 | 
147 | - Write more extensive documentation
148 | 
149 | ## Contributing 🤝🏼
150 | 
151 | Feel free to fork this repo and create a PR. I will review them and merge if ok.
152 | The above todos can be a very good place to start.
153 | 
154 | ## License 📝
155 | 
156 | [MIT](https://choosealicense.com/licenses/mit/)
157 | 


--------------------------------------------------------------------------------
/jest.config.js:
--------------------------------------------------------------------------------
1 | /** @type {import('ts-jest/dist/types').InitialOptionsTsJest} */
2 | module.exports = {
3 |   preset: 'ts-jest',
4 |   testEnvironment: 'node',
5 | };


--------------------------------------------------------------------------------
/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alexferrari88/scrapeblocks/9c53411d51de5dcdebbecd0957e2a8768b6ae1d5/logo.png


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "scrapeblocks",
 3 |   "version": "0.2.0",
 4 |   "description": "Scraping automation framework based on Playwright",
 5 |   "main": "./dist/index.js",
 6 |   "scripts": {
 7 |     "test": "jest",
 8 |     "build": "rm -rf ./dist && tsc"
 9 |   },
10 |   "repository": {
11 |     "type": "git",
12 |     "url": "git+https://github.com/alexferrari88/scrapeblocks.git"
13 |   },
14 |   "keywords": [
15 |     "scraping",
16 |     "scraper",
17 |     "modular"
18 |   ],
19 |   "author": "Alex Ferrari",
20 |   "license": "MIT",
21 |   "bugs": {
22 |     "url": "https://github.com/alexferrari88/scrapeblocks/issues"
23 |   },
24 |   "homepage": "https://github.com/alexferrari88/scrapeblocks#readme",
25 |   "devDependencies": {
26 |     "@types/jest": "^28.1.1",
27 |     "jest": "^28.1.1",
28 |     "ts-jest": "^28.0.4",
29 |     "ts-node": "^10.8.1",
30 |     "typescript": "^4.7.3"
31 |   },
32 |   "dependencies": {
33 |     "buffer-image-size": "^0.6.4",
34 |     "image-size": "^1.0.1",
35 |     "playwright": "^1.22.2",
36 |     "playwright-dompath": "^0.0.6"
37 |   }
38 | }
39 | 


--------------------------------------------------------------------------------
/src/index.ts:
--------------------------------------------------------------------------------
 1 | import { Browser, BrowserContext, chromium, Page } from "playwright";
 2 | import { Action, PlaywrightBlocks, ScrapingStrategy } from "./types";
 3 | import { USER_AGENTS } from "./utils/browser-config";
 4 | 
 5 | class Scraper<T = PlaywrightBlocks> {
 6 |   browser: Browser | undefined;
 7 |   context: BrowserContext | undefined;
 8 |   page: Page | undefined;
 9 |   url: string;
10 |   strategy?: ScrapingStrategy<T>;
11 |   actions: Action[] | undefined;
12 | 
13 |   constructor(url: string, strategy?: ScrapingStrategy<T>, actions?: Action[]) {
14 |     this.url = url;
15 |     this.strategy = strategy;
16 |     this.actions = actions;
17 |   }
18 | 
19 |   withBrowser(browser: Browser): Scraper<T> {
20 |     this.browser = browser;
21 |     return this;
22 |   }
23 | 
24 |   withContext(context: BrowserContext): Scraper<T> {
25 |     this.context = context;
26 |     return this;
27 |   }
28 | 
29 |   withPage(page: Page): Scraper<T> {
30 |     this.page = page;
31 |     return this;
32 |   }
33 | 
34 |   async run(): Promise<T> {
35 |     if (!this.browser && !this.context && !this.page)
36 |       this.browser = await chromium.launch();
37 |     if (!this.context && !this.page)
38 |       this.context = await (this.browser as Browser).newContext({
39 |         userAgent: USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)], // Picks a random user agent
40 |       });
41 |     if (!this.page)
42 |       this.page = await (this.context as BrowserContext).newPage();
43 |     if (!this.url) throw new Error("URL is required");
44 |     try {
45 |       if (this.url) await this.page.goto(this.url);
46 |     } catch (error) {
47 |       await this.browser?.close();
48 |       throw error;
49 |     }
50 |     if (this.actions) {
51 |       for (let action of this.actions) {
52 |         try {
53 |           await action.execute(this.page, this.context);
54 |         } catch (error) {
55 |           await this.browser?.close();
56 |           throw error;
57 |         }
58 |       }
59 |     }
60 |     // No strategy provided, just execute the
61 |     // actions and return the PlaywrightBlocks
62 |     if (!this?.strategy) {
63 |       return {
64 |         browser: this.browser as Browser,
65 |         context: this.context as BrowserContext,
66 |         page: this.page,
67 |       } as unknown as T;
68 |     }
69 | 
70 |     let result;
71 |     try {
72 |       result = await this.strategy.execute(this.page);
73 |     } catch (error) {
74 |       await this.browser?.close();
75 |       throw error;
76 |     }
77 |     if (this.browser) await this.browser.close();
78 |     return result;
79 |   }
80 | }
81 | 
82 | export * as ScrapingAction from "./scraping-actions";
83 | export * as ScrapingStrategies from "./scraping-strategies";
84 | export { Scraper };
85 | 


--------------------------------------------------------------------------------
/src/scraping-actions/AddCookies.ts:
--------------------------------------------------------------------------------
 1 | import { BrowserContext, Page } from "playwright";
 2 | import { Action, ActionOptions } from "../types";
 3 | import { delay, validateOptions } from "../utils/utils";
 4 | 
 5 | type CookieOptions = {
 6 |   [P in keyof ActionOptions as Exclude<
 7 |     P,
 8 |     "element" | "value"
 9 |   >]-?: ActionOptions[P];
10 | };
11 | export class AddCookies implements Action {
12 |   options: CookieOptions;
13 | 
14 |   constructor(options: CookieOptions) {
15 |     validateOptions(options, {
16 |       cookies: "Cookie action requires a cookie to add",
17 |     });
18 |     this.options = options;
19 |   }
20 | 
21 |   async execute(_page: Page, context: BrowserContext): Promise<void> {
22 |     await context.addCookies(this.options.cookies);
23 |     await delay(100);
24 |   }
25 | }
26 | 


--------------------------------------------------------------------------------
/src/scraping-actions/Click.ts:
--------------------------------------------------------------------------------
 1 | import { BrowserContext, Page } from "playwright";
 2 | import { Action, ActionOptions } from "../types";
 3 | import { delay, validateOptions } from "../utils/utils";
 4 | 
 5 | type ClickOptions = {
 6 |   [P in keyof ActionOptions as Exclude<
 7 |     P,
 8 |     "cookies" | "value"
 9 |   >]-?: ActionOptions[P];
10 | };
11 | export class Click implements Action {
12 |   options: ClickOptions;
13 | 
14 |   constructor(options: ClickOptions) {
15 |     validateOptions(options, {
16 |       element: "Click action requires an element to click",
17 |     });
18 |     this.options = options;
19 |   }
20 | 
21 |   async execute(page: Page, _context?: BrowserContext): Promise<void> {
22 |     await page.click(this.options.element);
23 |     await delay(100);
24 |   }
25 | }
26 | 


--------------------------------------------------------------------------------
/src/scraping-actions/Press.ts:
--------------------------------------------------------------------------------
 1 | import { BrowserContext, Page } from "playwright";
 2 | import { Action, ActionOptions } from "../types";
 3 | import { delay, validateOptions } from "../utils/utils";
 4 | 
 5 | type PressOptions = {
 6 |   [P in keyof ActionOptions as Exclude<P, "cookies">]-?: ActionOptions[P];
 7 | };
 8 | export class Press implements Action {
 9 |   options: PressOptions;
10 | 
11 |   constructor(options: PressOptions) {
12 |     validateOptions(options, {
13 |       element: "Press action requires an element to press the key in",
14 |       value: "Press action requires a value to press",
15 |     });
16 |     this.options = options;
17 |   }
18 | 
19 |   async execute(page: Page, _context?: BrowserContext): Promise<void> {
20 |     await page.press(this.options.element, this.options.value);
21 |     await delay(100);
22 |   }
23 | }
24 | 


--------------------------------------------------------------------------------
/src/scraping-actions/Remove.ts:
--------------------------------------------------------------------------------
 1 | import { BrowserContext, Page } from "playwright";
 2 | import { Action, ActionOptions } from "../types";
 3 | import { delay, validateOptions } from "../utils/utils";
 4 | 
 5 | type RemoveOptions = {
 6 | 	[P in keyof ActionOptions as Exclude<P, "cookies" | "value">]-?: ActionOptions[P];
 7 | };
 8 | export class Remove implements Action {
 9 | 	options: RemoveOptions;
10 | 
11 | 	constructor(options: RemoveOptions) {
12 | 		validateOptions(options, {
13 | 			element: "Remove action requires an element to remove",
14 | 		});
15 | 		this.options = options;
16 | 	}
17 | 
18 | 	async execute(page: Page, _context?: BrowserContext): Promise<void> {
19 | 		await page.$$eval(this.options.element, (elements) => {
20 | 			elements.forEach((element) => {
21 | 				element.remove();
22 | 			});
23 | 		});
24 | 		await delay(100);
25 | 	}
26 | }
27 | 


--------------------------------------------------------------------------------
/src/scraping-actions/Scroll.ts:
--------------------------------------------------------------------------------
 1 | import { BrowserContext, Page } from "playwright";
 2 | import { Action, ActionOptions } from "../types";
 3 | import { delay } from "../utils/utils";
 4 | 
 5 | export class Scroll implements Action {
 6 |   options: ActionOptions;
 7 | 
 8 |   constructor(options: ActionOptions) {
 9 |     this.options = options;
10 |   }
11 | 
12 |   async execute(page: Page, _context?: BrowserContext): Promise<void> {
13 |     await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
14 |     await delay(100);
15 |   }
16 | }
17 | 


--------------------------------------------------------------------------------
/src/scraping-actions/Select.ts:
--------------------------------------------------------------------------------
 1 | import { BrowserContext, Page } from "playwright";
 2 | import { Action, ActionOptions } from "../types";
 3 | import { delay, validateOptions } from "../utils/utils";
 4 | 
 5 | type SelectOptions = {
 6 |   [P in keyof ActionOptions as Exclude<P, "cookies">]-?: ActionOptions[P];
 7 | };
 8 | export class Select implements Action {
 9 |   options: SelectOptions;
10 | 
11 |   constructor(options: SelectOptions) {
12 |     validateOptions(options, {
13 |       element: "Select action requires an element to select",
14 |       value: "Select action requires a value to select",
15 |     });
16 |     this.options = options;
17 |   }
18 | 
19 |   async execute(page: Page, _context?: BrowserContext): Promise<void> {
20 |     await page.selectOption(
21 |       this.options.element as string,
22 |       this.options.value as string
23 |     );
24 |     await delay(100);
25 |   }
26 | }
27 | 


--------------------------------------------------------------------------------
/src/scraping-actions/Type.ts:
--------------------------------------------------------------------------------
 1 | import { BrowserContext, Page } from "playwright";
 2 | import { Action, ActionOptions } from "../types";
 3 | import { delay, validateOptions } from "../utils/utils";
 4 | 
 5 | type TypeOptions = {
 6 |   [P in keyof ActionOptions as Exclude<P, "cookies">]-?: ActionOptions[P];
 7 | };
 8 | export class Type implements Action {
 9 |   options: TypeOptions;
10 | 
11 |   constructor(options: TypeOptions) {
12 |     validateOptions(options, {
13 |       element: "Type action requires an element to type in",
14 |       value: "Type action requires a value to type",
15 |     });
16 |     this.options = options;
17 |   }
18 | 
19 |   async execute(page: Page, _context?: BrowserContext): Promise<void> {
20 |     await page.type(this.options.element, this.options.value, { delay: 100 });
21 |     await delay(100);
22 |   }
23 | }
24 | 


--------------------------------------------------------------------------------
/src/scraping-actions/Wait.ts:
--------------------------------------------------------------------------------
 1 | import { BrowserContext, Page } from "playwright";
 2 | import { Action, ActionOptions } from "../types";
 3 | import { delay, validateOptions } from "../utils/utils";
 4 | 
 5 | type WaitOptions = {
 6 |   [P in keyof ActionOptions as Exclude<
 7 |     P,
 8 |     "element" | "cookies"
 9 |   >]-?: ActionOptions[P];
10 | };
11 | export class Wait implements Action {
12 |   options: WaitOptions;
13 | 
14 |   constructor(options: WaitOptions) {
15 |     validateOptions(options, {
16 |       value: "Wait action requires a value (in ms) to wait",
17 |     });
18 |     this.options = options;
19 |   }
20 | 
21 |   async execute(_page: Page, _context: BrowserContext): Promise<void> {
22 |     await delay(parseInt(this.options.value));
23 |     await delay(100);
24 |   }
25 | }
26 | 


--------------------------------------------------------------------------------
/src/scraping-actions/index.ts:
--------------------------------------------------------------------------------
1 | export { AddCookies } from "./AddCookies";
2 | export { Click } from "./Click";
3 | export { Press } from "./Press";
4 | export { Remove } from "./Remove";
5 | export { Scroll } from "./Scroll";
6 | export { Select } from "./Select";
7 | export { Type } from "./Type";
8 | export { Wait } from "./Wait";
9 | 


--------------------------------------------------------------------------------
/src/scraping-strategies/ScreenshotToMap.ts:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * This strategy allows to take a screenshot of the page and then parse it to extract the map of the page with selected elements.
 3 |  * This is akin to what visualping.io does.
 4 |  */
 5 | 
 6 | import path from "path";
 7 | import { ElementHandle, Page } from "playwright";
 8 | import { cssPath, xPath as getXPath } from "playwright-dompath";
 9 | import { ScrapingStrategy } from "../types";
10 | import { getRandomFileName } from "../utils/utils";
11 | const sizeOf = require("image-size");
12 | const sizeOfBuffer = require("buffer-image-size");
13 | 
14 | type BoundingBoxReturn = NonNullable<Awaited<ReturnType<ElementHandle["boundingBox"]>>>;
15 | 
16 | export type ScreenshotMap = BoundingBoxReturn & {
17 | 	cssSelector?: string;
18 | 	xPath?: string;
19 | 	zIndex: number;
20 | };
21 | 
22 | export type ImageSizeType = {
23 | 	width: number;
24 | 	height: number;
25 | 	type: string;
26 | };
27 | 
28 | export class ScreenshotToMapStrategy
29 | 	implements ScrapingStrategy<Promise<[Buffer | string, ImageSizeType, ScreenshotMap[]]>>
30 | {
31 | 	selectors: string[];
32 | 	includexpath: boolean = false;
33 | 	includeCSSSelectors: boolean = true;
34 | 	fileName?: string;
35 | 	filePath?: string;
36 | 
37 | 	constructor(selectors: string[], includeCSSSelectors?: boolean) {
38 | 		this.selectors = selectors;
39 | 		this.includeCSSSelectors = includeCSSSelectors || false;
40 | 	}
41 | 
42 | 	asFile(filePath: string, fileName?: string): this {
43 | 		this.fileName = fileName || `${getRandomFileName()}.png`;
44 | 		this.filePath = filePath || "./";
45 | 		return this;
46 | 	}
47 | 
48 | 	getZIndex(xpath: string): number {
49 | 		return xpath.split("/").length;
50 | 	}
51 | 
52 | 	async execute(page: Page): Promise<[Buffer | string, ImageSizeType, ScreenshotMap[]]> {
53 | 		let imgResult: Buffer | string;
54 | 		let imgSize: ImageSizeType | undefined;
55 | 		if (this.fileName) {
56 | 			let savePath = path.join(path.normalize(this.filePath as string), this.fileName);
57 | 			await page.screenshot({
58 | 				path: savePath,
59 | 				fullPage: true,
60 | 			});
61 | 			imgResult = savePath;
62 | 			imgSize = sizeOf(savePath) as ImageSizeType;
63 | 		} else {
64 | 			imgResult = await page.screenshot({
65 | 				fullPage: true,
66 | 			});
67 | 			imgSize = sizeOfBuffer(imgResult) as ImageSizeType;
68 | 		}
69 | 		const toReturn: ScreenshotMap[] = [];
70 | 		for (const selector of this.selectors) {
71 | 			const elements = await page.$$(selector);
72 | 			for (const element of elements) {
73 | 				const boundingBox = await element.boundingBox();
74 | 				if (!boundingBox) continue;
75 | 				const xPath = await getXPath(element);
76 | 				if (!xPath || xPath == "") continue;
77 | 				const cssSelector = this.includeCSSSelectors ? await cssPath(element) : undefined;
78 | 				const zIndex = this.getZIndex(xPath);
79 | 				toReturn.push({ ...boundingBox, cssSelector, xPath, zIndex });
80 | 			}
81 | 		}
82 | 		return [imgResult, imgSize, toReturn];
83 | 	}
84 | }
85 | 


--------------------------------------------------------------------------------
/src/scraping-strategies/TextContentScraping.ts:
--------------------------------------------------------------------------------
 1 | import { Page } from "playwright";
 2 | import { ScrapingStrategy } from "../types";
 3 | 
 4 | export class TextContentScraping
 5 |   implements ScrapingStrategy<Promise<string[] | null>>
 6 | {
 7 |   selector: string;
 8 |   constructor(selector: string) {
 9 |     this.selector = selector;
10 |   }
11 | 
12 |   async execute(page: Page): Promise<string[] | null> {
13 |     return await page.locator(this.selector).allTextContents();
14 |   }
15 | }
16 | 


--------------------------------------------------------------------------------
/src/scraping-strategies/index.ts:
--------------------------------------------------------------------------------
1 | export {
2 |   ScreenshotMap,
3 |   ScreenshotToMapStrategy as ScreenshotToMap,
4 | } from "./ScreenshotToMap";
5 | export { TextContentScraping } from "./TextContentScraping";
6 | 


--------------------------------------------------------------------------------
/src/types.ts:
--------------------------------------------------------------------------------
 1 | import { Browser, BrowserContext, Page } from "playwright";
 2 | 
 3 | export interface Command {
 4 |   execute(): void;
 5 | }
 6 | 
 7 | export interface ScrapingStrategy<T> {
 8 |   execute(page: Page): T;
 9 | }
10 | 
11 | export type ActionType =
12 |   | "click"
13 |   | "type"
14 |   | "scroll"
15 |   | "select"
16 |   | "wait"
17 |   | "cookie";
18 | 
19 | export type ActionOptions = {
20 |   element?: string | undefined;
21 |   value?: string | undefined;
22 |   cookies?: Parameters<BrowserContext["addCookies"]>[0] | undefined;
23 | };
24 | 
25 | export interface Action {
26 |   options: ActionOptions;
27 |   execute(page: Page, context?: BrowserContext): void;
28 | }
29 | 
30 | export type PlaywrightBlocks = {
31 |   browser: Browser;
32 |   context: BrowserContext;
33 |   page: Page;
34 | };
35 | 


--------------------------------------------------------------------------------
/src/utils/browser-config.ts:
--------------------------------------------------------------------------------
 1 | export const USER_AGENTS = [
 2 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36',
 3 |   'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36',
 4 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36',
 5 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
 6 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36',
 7 |   'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36',
 8 |   'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36',
 9 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36',
10 |   'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36',
11 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36',
12 |   'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36',
13 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36',
14 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36',
15 |   'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.71 Safari/537.36',
16 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36',
17 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36',
18 |   'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.83 Safari/537.1',
19 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36',
20 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
21 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
22 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
23 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36',
24 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',
25 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36',
26 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36',
27 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36',
28 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36',
29 |   'Mozilla/5.0 (Windows NT 5.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36',
30 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36',
31 |   'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36',
32 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
33 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36',
34 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36',
35 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36',
36 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36',
37 |   'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
38 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36',
39 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36',
40 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
41 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36',
42 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36',
43 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36',
44 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36',
45 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36',
46 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36',
47 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36',
48 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
49 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36',
50 |   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
51 |   'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36'
52 | ];
53 | 


--------------------------------------------------------------------------------
/src/utils/utils.ts:
--------------------------------------------------------------------------------
 1 | export function delay(ms: number) {
 2 |   return new Promise((resolve) => setTimeout(resolve, ms));
 3 | }
 4 | 
 5 | export type ObjectWithKeysFromGeneric<T, U> = {
 6 |   [key in keyof T]: U;
 7 | };
 8 | 
 9 | export function validateOptions<T>(
10 |   options: T,
11 |   errors: ObjectWithKeysFromGeneric<T, string>
12 | ) {
13 |   for (const key in errors) {
14 |     if (options[key] == undefined) {
15 |       throw new Error(errors[key]);
16 |     }
17 |   }
18 | }
19 | 
20 | export function getRandomFileName(prefix?: string) {
21 |   const timestamp = new Date().toISOString().replace(/[-:.]/g, "");
22 |   const random = ("" + Math.random()).substring(2, 8);
23 |   const random_number = timestamp + random;
24 |   return `${prefix}${random_number}`;
25 | }
26 | 


--------------------------------------------------------------------------------
/tests/scraper.test.ts:
--------------------------------------------------------------------------------
  1 | import { Browser, BrowserContext, chromium, Page } from "playwright";
  2 | import { Scraper } from "../src/index";
  3 | import { Click, Press, Remove, Scroll, Select, Type, Wait } from "../src/scraping-actions";
  4 | import { TextContentScraping } from "../src/scraping-strategies/TextContentScraping";
  5 | import { PlaywrightBlocks } from "../src/types";
  6 | const path = require("path");
  7 | 
  8 | expect.extend({
  9 | 	toBeBrowser(received: Browser) {
 10 | 		const pass = "newContext" in received;
 11 | 		return {
 12 | 			pass,
 13 | 			message: () => `expected ${received} ${pass ? "" : "not"} to be a browser`,
 14 | 		};
 15 | 	},
 16 | 	toBeBrowserContext(received: BrowserContext) {
 17 | 		const pass = "newPage" in received;
 18 | 		return {
 19 | 			pass,
 20 | 			message: () => `expected ${received} ${pass ? "" : "not"} to be a browser context`,
 21 | 		};
 22 | 	},
 23 | 	toBePage(received: Page) {
 24 | 		const pass = "goto" in received;
 25 | 		return {
 26 | 			pass,
 27 | 			message: () => `expected ${received} ${pass ? "" : "not"} to be a page`,
 28 | 		};
 29 | 	},
 30 | });
 31 | 
 32 | interface CustomMatchers<R = unknown> {
 33 | 	toBeBrowser(): R;
 34 | 	toBeBrowserContext(): R;
 35 | 	toBePage(): R;
 36 | }
 37 | 
 38 | declare global {
 39 | 	namespace jest {
 40 | 		interface Expect extends CustomMatchers {}
 41 | 		interface Matchers<R> extends CustomMatchers<R> {}
 42 | 		interface InverseAsymmetricMatchers extends CustomMatchers {}
 43 | 	}
 44 | }
 45 | 
 46 | describe("Scraper", () => {
 47 | 	describe("Batteries-included", () => {
 48 | 		describe("TextContentScraping Basics", () => {
 49 | 			const URL = `file://${path.resolve("tests/templates/pricing.html")}`;
 50 | 			const XPATH_SELECTOR = "xpath=/html/body/section/div[2]/div[2]/div[3]/div[1]/div";
 51 | 			const CSS_SELECTOR =
 52 | 				"#pricing-cards > div.price-card.price-card--hero > div.price-card--price > div.price-card--price-text > div";
 53 | 			const expectedValue = "900";
 54 | 			const expectedNumberOfLiElement = 18;
 55 | 
 56 | 			it("should return the text value from a css selector with text", async () => {
 57 | 				const result = await new Scraper(URL, new TextContentScraping(CSS_SELECTOR)).run();
 58 | 				expect(result).toStrictEqual([expectedValue]);
 59 | 			});
 60 | 
 61 | 			it("should return the text value from an xpath selector with text", async () => {
 62 | 				const result = await new Scraper(URL, new TextContentScraping(XPATH_SELECTOR)).run();
 63 | 				expect(result).toStrictEqual([expectedValue]);
 64 | 			});
 65 | 
 66 | 			it("should return an empty string from a selector without text", async () => {
 67 | 				const result = await new Scraper(URL, new TextContentScraping(".slider")).run();
 68 | 				expect(result).toStrictEqual([""]);
 69 | 			});
 70 | 
 71 | 			it("should return an list of strings from a selector with multiple text", async () => {
 72 | 				const result = await new Scraper(URL, new TextContentScraping("li")).run();
 73 | 				expect(result).toHaveLength(expectedNumberOfLiElement);
 74 | 			});
 75 | 
 76 | 			it("should return an empty array for non existing selector", async () => {
 77 | 				const result = await new Scraper(URL, new TextContentScraping("asdeqda")).run();
 78 | 				expect(result).toStrictEqual([]);
 79 | 			});
 80 | 
 81 | 			it("should return an empty array for non existing xpath selector", async () => {
 82 | 				const result = await new Scraper(URL, new TextContentScraping("xpath=asdasdasd")).run();
 83 | 				expect(result).toStrictEqual([]);
 84 | 			});
 85 | 		});
 86 | 
 87 | 		describe("TextContentScraping with actions", () => {
 88 | 			const URL = `file://${path.resolve("tests/templates/actions.html")}`;
 89 | 			const expectedText = "This is a secret message!";
 90 | 			describe("Click", () => {
 91 | 				test("should click on a button", async () => {
 92 | 					const element = "#btn-revealer";
 93 | 					const selector = "#secret";
 94 | 					const clickAction = new Click({ element });
 95 | 					const result = await new Scraper(URL, new TextContentScraping(selector), [
 96 | 						clickAction,
 97 | 					]).run();
 98 | 					expect(result).toStrictEqual([expectedText]);
 99 | 				});
100 | 
101 | 				test("should type in a input box", async () => {
102 | 					const element = "#input-revealer";
103 | 					const value = "secret";
104 | 					const selector = "#you-typed";
105 | 					const typeAction = new Type({ element, value });
106 | 					const result = await new Scraper(URL, new TextContentScraping(selector), [
107 | 						typeAction,
108 | 					]).run();
109 | 					expect(result).toStrictEqual([value]);
110 | 				});
111 | 
112 | 				test("should wait", async () => {
113 | 					jest.setTimeout(20000);
114 | 					const value = "3000";
115 | 					const selector = "#hint";
116 | 					const expectedHintText = "terces";
117 | 					const noWaitResult = await new Scraper(URL, new TextContentScraping(selector)).run();
118 | 					expect(noWaitResult).toStrictEqual([]);
119 | 					const typeAction = new Wait({ value });
120 | 					const result = await new Scraper(URL, new TextContentScraping(selector), [
121 | 						typeAction,
122 | 					]).run();
123 | 					expect(result).toStrictEqual([expectedHintText]);
124 | 				});
125 | 
126 | 				test("should change option in a select element", async () => {
127 | 					const element = "#select-revealer";
128 | 					const value = "yes";
129 | 					const selector = "#secret";
130 | 					const typeAction = new Select({ element, value });
131 | 					const result = await new Scraper(URL, new TextContentScraping(selector), [
132 | 						typeAction,
133 | 					]).run();
134 | 					expect(result).toStrictEqual([expectedText]);
135 | 				});
136 | 
137 | 				test("should perform multiple actions", async () => {
138 | 					const element = "#input-revealer";
139 | 					const value = "secret";
140 | 					const selector = "#secret";
141 | 					const typeAction = new Type({ element, value });
142 | 					const pressAction = new Press({ element, value: "Enter" });
143 | 					const result = await new Scraper(URL, new TextContentScraping(selector), [
144 | 						typeAction,
145 | 						pressAction,
146 | 					]).run();
147 | 					expect(result).toStrictEqual([expectedText]);
148 | 				});
149 | 
150 | 				test("should scroll", async () => {
151 | 					const selector = "#secret";
152 | 					const scrollAction = new Scroll({});
153 | 					const result = await new Scraper(URL, new TextContentScraping(selector), [
154 | 						scrollAction,
155 | 					]).run();
156 | 					expect(result).toStrictEqual([expectedText]);
157 | 				});
158 | 
159 | 				test("should remove 1 element", async () => {
160 | 					const element = "label[for=select-revealer]";
161 | 					const beforeResult = await new Scraper(URL, new TextContentScraping(element)).run();
162 | 					expect(beforeResult).toHaveLength(1);
163 | 					const removeAction = new Remove({ element });
164 | 					const result = await new Scraper(URL, new TextContentScraping(element), [
165 | 						removeAction,
166 | 					]).run();
167 | 					expect(result).toHaveLength(0);
168 | 				});
169 | 
170 | 				test("should remove 2 elements", async () => {
171 | 					const element = "button";
172 | 					const beforeResult = await new Scraper(URL, new TextContentScraping(element)).run();
173 | 					expect(beforeResult).toHaveLength(2);
174 | 					const removeAction = new Remove({ element });
175 | 					const result = await new Scraper(URL, new TextContentScraping(element), [
176 | 						removeAction,
177 | 					]).run();
178 | 					expect(result).toHaveLength(0);
179 | 				});
180 | 
181 | 				test("should execute actions without a strategy", async () => {
182 | 					const element = "#btn-revealer";
183 | 					const selector = "#secret";
184 | 					const clickAction = new Click({ element });
185 | 					const { browser, context, page } = await new Scraper<PlaywrightBlocks>(URL, undefined, [
186 | 						clickAction,
187 | 					]).run();
188 | 					const result = await page.locator(selector).allTextContents();
189 | 					await browser.close();
190 | 					expect(browser).toBeBrowser();
191 | 					expect(context).toBeBrowserContext();
192 | 					expect(page).toBePage();
193 | 					expect(result).toStrictEqual([expectedText]);
194 | 				});
195 | 
196 | 				// TODO: cannot set cookies on file://
197 | 				// test("should set a cookie", async () => {
198 | 				//   const value = "secret";
199 | 				//   const selector = "#secret";
200 | 				//   const cookies = {
201 | 				//     name: "secretCookie",
202 | 				//     value,
203 | 				//     url: "http://localhost",
204 | 				//   };
205 | 				//   const cookieAction = new AddCookies({ cookies: [cookies] });
206 | 				//   const result = await new Scraper(
207 | 				//     URL,
208 | 				//     new TextContentScraping(selector),
209 | 				//     [cookieAction]
210 | 				//   ).run();
211 | 				//   expect(result).toStrictEqual([expectedText]);
212 | 				// });
213 | 			});
214 | 		});
215 | 	});
216 | 
217 | 	describe("With external Playwright instances", () => {
218 | 		const URL = `file://${path.resolve("tests/templates/pricing.html")}`;
219 | 		const CSS_SELECTOR =
220 | 			"#pricing-cards > div.price-card.price-card--hero > div.price-card--price > div.price-card--price-text > div";
221 | 		const expectedValue = "900";
222 | 
223 | 		describe("withBrowser", () => {
224 | 			test("should be able to pass an existing browser instance", async () => {
225 | 				const browser = await chromium.launch();
226 | 				const spy = jest.spyOn(browser, "newContext");
227 | 				const result = await new Scraper(URL, new TextContentScraping(CSS_SELECTOR))
228 | 					.withBrowser(browser)
229 | 					.run();
230 | 				await browser.close();
231 | 				expect(result).toStrictEqual([expectedValue]);
232 | 				expect(spy).toHaveBeenCalled();
233 | 				spy.mockRestore();
234 | 			});
235 | 		});
236 | 
237 | 		describe("withContext", () => {
238 | 			test("should be able to pass an existing BrowserContext instance", async () => {
239 | 				const browser = await chromium.launch();
240 | 				const context = await browser.newContext();
241 | 				const spy = jest.spyOn(context, "newPage");
242 | 				const result = await new Scraper(URL, new TextContentScraping(CSS_SELECTOR))
243 | 					.withContext(context)
244 | 					.run();
245 | 				await browser.close();
246 | 				expect(result).toStrictEqual([expectedValue]);
247 | 				expect(spy).toHaveBeenCalled();
248 | 				spy.mockRestore();
249 | 			});
250 | 		});
251 | 
252 | 		describe("withPage", () => {
253 | 			test("should be able to pass an existing Page instance", async () => {
254 | 				const browser = await chromium.launch();
255 | 				const context = await browser.newContext();
256 | 				const page = await context.newPage();
257 | 				const spy = jest.spyOn(page, "goto");
258 | 				const result = await new Scraper(URL, new TextContentScraping(CSS_SELECTOR))
259 | 					.withPage(page)
260 | 					.run();
261 | 				await browser.close();
262 | 				expect(result).toStrictEqual([expectedValue]);
263 | 				expect(spy).toHaveBeenCalled();
264 | 				spy.mockRestore();
265 | 			});
266 | 		});
267 | 	});
268 | });
269 | 


--------------------------------------------------------------------------------
/tests/screenshotToMap.spec.ts:
--------------------------------------------------------------------------------
 1 | import * as fs from "fs";
 2 | import { chromium } from "playwright";
 3 | import { Scraper } from "../src";
 4 | import { ScreenshotMap, ScreenshotToMap } from "../src/scraping-strategies";
 5 | import { ImageSizeType } from "../src/scraping-strategies/ScreenshotToMap";
 6 | const path = require("path");
 7 | 
 8 | expect.extend({
 9 | 	toHaveSizes(imgSize: ImageSizeType, width: number, height: number) {
10 | 		return {
11 | 			pass: imgSize.width === width && imgSize.height === height,
12 | 			message: () => `expected ${imgSize.width}x${imgSize.height} to equal ${width}x${height}`,
13 | 		};
14 | 	},
15 | 	toBeScreenshotMap(received: ScreenshotMap) {
16 | 		return {
17 | 			pass:
18 | 				"cssSelector" in received &&
19 | 				"xPath" in received &&
20 | 				"zIndex" in received &&
21 | 				"width" in received &&
22 | 				"height" in received &&
23 | 				"x" in received &&
24 | 				"y" in received,
25 | 			message: () => `expected the item with xPath ${received.xPath} to be a screenshot map`,
26 | 		};
27 | 	},
28 | });
29 | 
30 | interface CustomMatchers<R = unknown> {
31 | 	toHaveSizes(width: number, height: number): R;
32 | 	toBeScreenshotMap(): R;
33 | }
34 | 
35 | declare global {
36 | 	namespace jest {
37 | 		interface Expect extends CustomMatchers {}
38 | 		interface Matchers<R> extends CustomMatchers<R> {}
39 | 		interface InverseAsymmetricMatchers extends CustomMatchers {}
40 | 	}
41 | }
42 | describe("ScreenshotToMap Basics", () => {
43 | 	const screenshotsFolder = path.resolve("tests/screenshots");
44 | 	const imgWidth = 1280;
45 | 	const imgHeight = 846;
46 | 	afterAll(() => {
47 | 		fs.rmSync(screenshotsFolder, { recursive: true, force: true });
48 | 	});
49 | 
50 | 	test("should take a screenshot", async () => {
51 | 		const URL = `file://${path.resolve("tests/templates/pricing.html")}`;
52 | 		const selector = ["div", "p", "a", "table", "img"];
53 | 		const browser = await chromium.launch();
54 | 		const context = await browser.newContext();
55 | 		const page = await context.newPage();
56 | 		const spy = jest.spyOn(page, "screenshot");
57 | 		const _ = await new Scraper(URL, new ScreenshotToMap(selector)).withPage(page).run();
58 | 		await browser.close();
59 | 		expect(spy).toHaveBeenCalled();
60 | 		spy.mockRestore();
61 | 	});
62 | 
63 | 	test("should return a buffer, and map", async () => {
64 | 		const URL = `file://${path.resolve("tests/templates/pricing.html")}`;
65 | 		const selector = ["div", "p", "a", "table", "img"];
66 | 		const numElements = 27;
67 | 		const [fileBuffer, imgSize, screenshotMap] = await new Scraper(
68 | 			URL,
69 | 			new ScreenshotToMap(selector)
70 | 		).run();
71 | 		expect(typeof fileBuffer).toBe("object");
72 | 		expect(imgSize).toHaveSizes(imgWidth, imgHeight);
73 | 		expect(imgSize.type).toStrictEqual("png");
74 | 		screenshotMap.forEach((item) => {
75 | 			expect(item).toBeScreenshotMap();
76 | 		});
77 | 		expect(screenshotMap.length).toStrictEqual(numElements);
78 | 	});
79 | 
80 | 	test("should take a screenshot and save it as file", async () => {
81 | 		const URL = `file://${path.resolve("tests/templates/pricing.html")}`;
82 | 		const selector = ["div", "p", "a", "table", "img"];
83 | 		const [file, imgSize, screenshotMap] = await new Scraper(
84 | 			URL,
85 | 			new ScreenshotToMap(selector).asFile(screenshotsFolder)
86 | 		).run();
87 | 		const fileExists = fs.existsSync(file);
88 | 		expect(fileExists).toBeTruthy();
89 | 		expect(typeof file).toStrictEqual("string");
90 | 		expect(imgSize.type).toStrictEqual("png");
91 | 		expect(imgSize).toHaveSizes(imgWidth, imgHeight);
92 | 		screenshotMap.forEach((item) => {
93 | 			expect(item).toBeScreenshotMap();
94 | 		});
95 | 	});
96 | });
97 | 


--------------------------------------------------------------------------------
/tests/templates/actions.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html lang="en">
  3 | 
  4 | <head>
  5 |     <meta charset="UTF-8">
  6 |     <meta http-equiv="X-UA-Compatible" content="IE=edge">
  7 |     <meta name="viewport" content="width=device-width, initial-scale=1.0">
  8 |     <title>Test Page for Actions</title>
  9 | </head>
 10 | 
 11 | <body>
 12 |     <div class="main" style="position: relative; border: 2px solid blue; height: 2000px;">
 13 |         <button id="btn-revealer" onclick="revealSecret(false)">Reveal!</button>
 14 |         <button id="cookie-btn-revealer" onclick="revealSecret(true)">Reveal with cookie!</button>
 15 |         <input id="input-revealer" type="text" placeholder="Type the secret word" style="display: block;">
 16 | 
 17 |         <div class="container">
 18 |             <p>You typed: <span id="you-typed"></span></p>
 19 |             <label for="select-revealer">Select what to do:</label>
 20 | 
 21 |             <select name="select-revealer" id="select-revealer">
 22 |                 <option value="">--Please choose an option--</option>
 23 |                 <option value="yes">Show password</option>
 24 |                 <option value="no">Do nothing</option>
 25 |             </select>
 26 |         </div>
 27 |     </div>
 28 |     <script>
 29 |         const divContainer = document.querySelector(".container");
 30 |         const inputRevealer = document.getElementById("input-revealer");
 31 |         const selectRevealer = document.getElementById("select-revealer");
 32 |         const youTypedElement = document.getElementById("you-typed");
 33 | 
 34 |         setTimeout(function () {
 35 |             revealHint();
 36 |         }, 2000);
 37 | 
 38 |         window.onscroll = function (ev) {
 39 |             if (isContentScrolledToBottom(document.querySelector(".main"))) {
 40 |                 revealSecret(false)
 41 |             }
 42 |         };
 43 | 
 44 | 
 45 |         inputRevealer.addEventListener('keyup', function (event) {
 46 |             if (event.keyCode === 13) {
 47 |                 event.preventDefault();
 48 |                 revealSecret(true)
 49 |             } else {
 50 |                 youTypedElement.innerHTML = event.target.value;
 51 |             }
 52 |         });
 53 | 
 54 |         selectRevealer.addEventListener('change', function (event) {
 55 |             if (event.target.value === "no") return;
 56 |             revealSecret(false);
 57 |         });
 58 | 
 59 |         function isContentScrolledToBottom(element) {
 60 |             const rest = element.scrollHeight - element.scrollTop;
 61 |             return Math.abs(element.clientHeight - rest) < 1;
 62 |         }
 63 | 
 64 |         function revealHint() {
 65 |             const p = document.createElement("p");
 66 |             p.id = "hint";
 67 |             p.innerHTML = "terces";
 68 |             divContainer.appendChild(p);
 69 |         }
 70 | 
 71 |         function revealSecret(check) {
 72 |             if (inputRevealer.value !== "secret" && check) return;
 73 |             if (document.getElementById("secret")) return;
 74 |             const secretCookie = getCookie("secretCookie");
 75 |             if (secretCookie && secretCookie !== "secret" && check) return;
 76 |             const p = document.createElement("p");
 77 |             p.id = "secret";
 78 |             p.innerHTML = "This is a secret message!";
 79 |             divContainer.appendChild(p);
 80 |             const a = document.createElement("a");
 81 |             a.id = "concealer";
 82 |             a.href = "#";
 83 |             a.innerHTML = "Conceal!";
 84 |             a.onclick = concealSecret;
 85 |             divContainer.appendChild(a);
 86 |         }
 87 | 
 88 |         function concealSecret() {
 89 |             document.getElementById("secret").remove();
 90 |             document.getElementById("concealer").remove();
 91 |         }
 92 | 
 93 |         function getCookie(name) {
 94 |             var nameEQ = name + "=";
 95 |             var ca = document.cookie.split(';');
 96 |             for (var i = 0; i < ca.length; i++) {
 97 |                 var c = ca[i];
 98 |                 while (c.charAt(0) == ' ') c = c.substring(1, c.length);
 99 |                 if (c.indexOf(nameEQ) == 0) return c.substring(nameEQ.length, c.length);
100 |             }
101 |             return null;
102 |         }
103 |     </script>
104 | </body>
105 | 
106 | </html>


--------------------------------------------------------------------------------
/tests/templates/pricing.css:
--------------------------------------------------------------------------------
  1 | body{
  2 | 	padding:0;
  3 | 	margin: 0;
  4 | 	background:#1E1E21;
  5 | 	color:#fff;
  6 | }
  7 | h2{
  8 | 	font-size: 30px;
  9 | 	color:#fff;
 10 | }
 11 | p{
 12 | 	color:rgb(238, 231, 231);
 13 | }
 14 | .has-tooltip:not(.tooltip-disabled) {
 15 | 	cursor: help;
 16 | 	position:relative;
 17 | }
 18 | 
 19 | .has-tooltip:not(.tooltip-disabled):hover
 20 | 	.tooltip-container {
 21 | 	display: block;
 22 | }
 23 | 
 24 | .has-tooltip .tooltip-container {
 25 | 	display: none;
 26 | 	position: absolute;
 27 | 	background-color: #BE3760;
 28 | 	color:#fff;
 29 | 	z-index: 20;
 30 | 	bottom: calc(100% + 13px);
 31 | 	left: -10px;
 32 | 	right: -10px;
 33 | 	padding: 16px 20px;
 34 | 	border-radius: 8px;
 35 | 
 36 | }
 37 | .has-tooltip .tooltip-container:after {
 38 | 	width: 0;
 39 | 	height: 0;
 40 | 	border-left: 10px solid transparent;
 41 | 	border-right: 10px solid transparent;
 42 | 	border-top: 10px solid #BE3760;
 43 | 	font-size: 0;
 44 | 	line-height: 0;
 45 | 	content: "";
 46 | 	position: absolute;
 47 | 	bottom: -10px;
 48 | }
 49 | 
 50 | .has-tooltip .tooltip-container h6 {
 51 | 	font-weight: 600;
 52 | 	font-size: 16px;
 53 | 	margin: 0px;
 54 | }
 55 | 
 56 | .has-tooltip .tooltip-container  {
 57 | 	color: #555;
 58 | 	margin-top:4px;
 59 | }
 60 | /* - - - - - - - End Tooltips - - - */
 61 | 
 62 | 
 63 | 
 64 | 
 65 | 
 66 | #pricing-container * {
 67 | 	box-sizing:border-box;
 68 | }
 69 | 
 70 | #pricing-container {
 71 | 	font-family: "Open Sans", Helvetica, Arial, Lucida, sans-serif;
 72 | 	-webkit-font-smoothing: antialiased;
 73 | 	max-width: 1080px;
 74 | 	margin: 0 auto 50px;
 75 | 	justify-content: center;
 76 | 	line-height: 1;
 77 | 	color: #000;
 78 | }
 79 | 
 80 | 
 81 | 
 82 | /* Cards */
 83 | 
 84 | #pricing-cards {
 85 | 	display: flex;
 86 | 	font-size: 14px;
 87 | }
 88 | 
 89 | #pricing-container .price-card {
 90 | 	background-color: #33333a;
 91 | 	color:#fff;
 92 | 	font-family: "Open Sans", Helvetica, Arial, Lucida, sans-serif;
 93 | 	display: block;
 94 | 	box-shadow: 0 3px 1px -2px rgba(0, 0, 0, 0.12), 0 1px 5px 0 rgba(0, 0, 0, 0.2);
 95 | 	transition: all 0.25s;
 96 | 	position: relative;
 97 | 	margin: 0 6px;
 98 | 	flex-grow: 1;
 99 | 	flex-shrink:1;
100 | }
101 | 
102 | #pricing-container .price-card:hover {
103 | 	transform: translateY(-5px);
104 | 		box-shadow: 0 6px 20px 0 rgba(0, 0, 0, 0.14),
105 | 		0 3px 1px -2px rgba(0, 0, 0, 0.12), 0 1px 5px 0 rgba(0, 0, 0, 0.2);
106 | }
107 | 
108 | 
109 | /* - - - - - - - - - Header Section - - - */
110 | 
111 | #pricing-container .price-card--header {
112 | 	
113 | 	margin: 0;
114 | 	padding: 20px 0;
115 | 	text-align: center;
116 | }
117 | 
118 | #pricing-container .price-card--header h4 {
119 | 	margin: 0;
120 | 	padding: 0;
121 | 	font-size: 28px;
122 | 	color: #fff;
123 | 	font-family: "Raleway", Helvetica, Arial, Lucida, sans-serif;
124 | 	font-weight: 700;
125 | }
126 | 
127 | 
128 | /* - - - - - - - - - Price Section - - - */
129 | 
130 | #pricing-container .price-card--price {
131 | 	text-align: center;
132 | 	padding: 28px 0 6px 0;
133 | }
134 | 
135 | #pricing-container .price-card--price-text {
136 | 	font-size: 48px;
137 | }
138 | 
139 | #pricing-container .price-card--price-number {
140 | 	font-weight: 500;
141 | 	opacity: 0.89;
142 | }
143 | 
144 | .odometer div {
145 | 	display: inline-block;
146 | }
147 | 
148 | #pricing-container .price-card--price-number:before {
149 | 	content: "$";
150 | 	font-size: 24px;
151 | 	top: -16px;
152 | 	display: inline-block;
153 | 	position: relative;
154 | }
155 | 
156 | #pricing-container .price-card--price-number:after {
157 | 	content: "/month";
158 | 	font-size: 12px;
159 | 	display: inline-block;
160 | 	color: grey;
161 | }
162 | 
163 | #pricing-container .price-card--price-conditions {
164 | 	padding: 14px 0;
165 | 	color: #888;
166 | 	line-height: 1.5;
167 | }
168 | 
169 | 
170 | 
171 | 
172 | /* - - - - - - - - - CTA Button Section - - - */
173 | 
174 | #pricing-container .price-card--cta {
175 | 	padding: 0 20px 24px;
176 | 	text-align: center;
177 | }
178 | 
179 | #pricing-container .price-card--cta--button.btn{
180 | 	min-width:20px;
181 | 	display:block;
182 | 	max-width:183px;
183 | 	margin:0 auto;
184 | }
185 | 
186 | 
187 | 
188 | 
189 | 
190 | /* - - - - - - - - - Features Section - - - */
191 | 
192 | #pricing-container .price-card--features {
193 | 
194 | 	padding: 16px 0 20px;
195 | }
196 | 
197 | #pricing-container ul.price-card--features--list {
198 | 	padding: 0 32px;
199 | 	list-style: none;
200 | 	margin: 0;
201 | }
202 | 
203 | #pricing-container li.price-card--features--item {
204 | 	margin: 8px 0;
205 | 	padding-left: 8px;
206 | 	line-height: 1.5;
207 | 	position: relative;
208 | }
209 | 
210 | #pricing-container
211 | 	li.price-card--features--item:not(.features-disabled):before {
212 | 	content: "✓";
213 | 	color: #BE3760;
214 | 	display: block;
215 | 	position: absolute;
216 | 	left: -8px;
217 | }
218 | 
219 | #pricing-container li.price-card--features--item.features-highlight {
220 | 	/* 	font-weight: 600; */
221 | }
222 | 
223 | #pricing-container li.price-card--features--item.features-disabled {
224 | 	opacity: 0.1;
225 | 	/* 	text-decoration: line-through; */
226 | }
227 | 
228 | 
229 | 
230 | 
231 | /* - - - - - - - - - Mobile Features Toggle - - - */
232 | 
233 | #pricing-container .price-card--mobile-features-toggle {
234 | 	text-align: center;
235 | 	margin: 24px 0 0;
236 | 	padding: 16px 0;
237 | 	cursor: pointer;
238 | 	display: none;
239 | 	color: green;
240 | }
241 | 
242 | #pricing-container .price-card--mobile-features-toggle:after {
243 | 	content: "Show All Features ▾";
244 | }
245 | #pricing-container .price-card--mobile-features-toggle.hideall:after {
246 | 	content: "Hide Features ▴";
247 | }
248 | 
249 | /* - - - - - - - - - Hero Card Styles - - - */
250 | 
251 | #pricing-container .price-card--hero {
252 | 	margin: -38px 6px 0;
253 | 	/* 	width:31%; */
254 | 	z-index: 10;
255 | }
256 | 
257 | #pricing-container .price-card--hero-text {
258 | 	background-color: #BE3760;
259 | 	height: 38px;
260 | 	color: white;
261 | 	line-height: 38px;
262 | 	text-align: center;
263 | 	font-weight: 600;
264 | }
265 | 
266 | /* - - - - - - - - - Only Yearly Basic Styles - - - */
267 | 
268 | #pricing-container .only-yearly {
269 | 	position: relative;
270 | }
271 | 
272 | #pricing-container .only-yearly .price-card--price-number {
273 | 	transition: opacity 0.2s;
274 | }
275 | 
276 | #pricing-container .only-yearly .only-yearly--text {
277 | 	position: absolute;
278 | 	top: -0.2em;
279 | 	left: 0;
280 | 	right: 0;
281 | 	display: none;
282 | }
283 | #pricing-container .only-yearly .only-yearly--text span {
284 | 	font-size: 14px;
285 | }
286 | 
287 | #pricing-container .only-yearly.if-monthly .price-card--price-number {
288 | 	opacity: 0;
289 | }
290 | 
291 | #pricing-container .only-yearly.if-monthly .only-yearly--text {
292 | 	display: block;
293 | }
294 | 
295 | /* - - - - - - - - - Switch Section - - - */
296 | 
297 | #pricing-switch {
298 | 	margin: 80px auto 100px;
299 | 	text-align: center;
300 | 	line-height: 1.4;
301 | 	position: relative;
302 | 	max-width: 1080px;
303 | }
304 | 
305 | #pricing-switch .switch-label {
306 | 	display: inline-block;
307 | 	width: 200px;
308 | 	text-align: center;
309 | 	opacity: 0.4;
310 | 	font-size: 16px;
311 | 	cursor: pointer;
312 | 	padding: 0 20px;
313 | }
314 | 
315 | #pricing-switch .switch-label .save-money {
316 | 	color: #4caf50;
317 | 	font-style: italic;
318 | 	padding-left: 8px;
319 | }
320 | #pricing-switch .save-money--mobile {
321 | 	color: #4caf50;
322 | 	font-style: italic;
323 | 	padding-top: 22px;
324 | 	display: none;
325 | }
326 | 
327 | #pricing-switch .switch-label.active {
328 | 	font-size: 18px;
329 | 	opacity: 1;
330 | }
331 | #pricing-switch .switch-label-monthly {
332 | 	text-align: right;
333 | }
334 | #pricing-switch .switch-label-yearly {
335 | 	text-align: left;
336 | }
337 | 
338 | #pricing-switch .switch {
339 | 	position: relative;
340 | 	display: inline-block;
341 | 	width: 60px;
342 | 	height: 34px;
343 | 	vertical-align: -50%;
344 | 	margin: 0;
345 | }
346 | 
347 | #pricing-switch .switch input {
348 | 	display: none;
349 | }
350 | 
351 | #pricing-switch .slider {
352 | 	position: absolute;
353 | 	cursor: pointer;
354 | 	top: 0;
355 | 	left: 0;
356 | 	right: 0;
357 | 	bottom: 0;
358 | 	background-color: #BE3760;
359 | 	border-radius: 34px;
360 | 	-webkit-transition: 0.1s;
361 | 	transition: 0.1s;
362 | }
363 | 
364 | #pricing-switch .slider:before {
365 | 	position: absolute;
366 | 	content: "";
367 | 	height: 26px;
368 | 	width: 26px;
369 | 	left: 4px;
370 | 	bottom: 4px;
371 | 	background-color: white;
372 | 	border-radius: 50%;
373 | 	-webkit-transition: 0.1s;
374 | 	transition: 0.1s;
375 | }
376 | 
377 | .btn{
378 | 	width: 200px;
379 | 	height:45px;
380 | 	background: transparent;
381 | 	border:1px solid #BE3760;
382 | 	color:#fff;
383 | 	border-radius: 5px;
384 | 	cursor: pointer;
385 | margin:2em 5em;
386 | }
387 | .btn-pro{
388 | 	width: 200px;
389 | 	height:45px;
390 | 	background: #BE3760;
391 | 	border:none;
392 | 	color:#fff;
393 | 	border-radius: 5px;
394 | 	cursor: pointer;
395 | margin:2em 5em;
396 | }
397 | .switch-label{
398 | 	color:#fff;
399 | }
400 | .active{
401 | 	color: #BE3760;
402 | }
403 | strong{
404 | 	color:#fff;
405 | }


--------------------------------------------------------------------------------
/tests/templates/pricing.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html lang="en">
  3 | 
  4 | <head>
  5 |     <meta charset="UTF-8">
  6 |     <meta http-equiv="X-UA-Compatible" content="IE=edge">
  7 |     <meta name="viewport" content="width=device-width, initial-scale=1.0">
  8 |     <link rel="stylesheet" href="pricing.css">
  9 |     <title>Test Page</title>
 10 | </head>
 11 | 
 12 | <body>
 13 |     <section id="pricing-container">
 14 | 
 15 |         <div id="pricing-switch">
 16 |             <h2>Choose a plan</h2>
 17 |             <span class="switch-label switch-label-monthly active">Monthly</span>
 18 |             <label class="switch" id="">
 19 |                 <input type="checkbox" checked="checked">
 20 |                 <span class="slider"></span>
 21 |             </label>
 22 |             <span class="switch-label switch-label-yearly ">Yearly </span>
 23 | 
 24 |             <div class="save-money--mobile">Save 10% on Yearly Plans</div>
 25 | 
 26 |         </div>
 27 |         <!-- end of pricing-switch -->
 28 | 
 29 |         <div id="pricing-cards">
 30 | 
 31 |             <!-- ============================= -->
 32 |             <!-- = Premium Subscription Card = -->
 33 |             <!-- ============================= -->
 34 |             <div class="price-card">
 35 | 
 36 |                 <div class="price-card--header">
 37 |                     <h4>Premium</h4>
 38 |                     <!-- <p>for growing your business</p> -->
 39 |                 </div>
 40 | 
 41 |                 <div class="price-card--price">
 42 |                     <div class="price-card--price-text">
 43 |                         <div class="price-card--price-number toggle-price-content odometer" data-price-monthly="2,499"
 44 |                             data-price-yearly="2,250">2,250</div>
 45 |                     </div>
 46 |                     <div class="price-card--price-conditions">
 47 |                         <div class="toggle-price-content" data-price-monthly="Billed Monthly"
 48 |                             data-price-yearly="Billed Annually">Billed Annually</div>
 49 | 
 50 |                     </div>
 51 |                 </div>
 52 | 
 53 | 
 54 | 
 55 |                 <div class="price-card--features">
 56 | 
 57 |                     <ul class="price-card--features--list">
 58 |                         <li class="price-card--features--item has-tooltip">Sales & Marketing Platform
 59 |                             <div class="tooltip-container"><strong>Sales & Marketing Platform</strong>
 60 |                                 <p>Vendasta's end-to-end sales solution for companies that serve local businesses, and
 61 |                                     want to grow digital revenue.</p>
 62 |                             </div>
 63 |                         </li>
 64 |                         <li class="price-card--features--item has-tooltip">Snapshot & Campaigns
 65 |                             <div class="tooltip-container"><strong>Snapshot & Campaigns</strong>
 66 |                                 <p>Access to award-winning automated needs-assessment and email marketing automation.
 67 |                                     Unlimited emails sent to unlimited accounts.</p>
 68 |                             </div>
 69 |                         </li>
 70 |                         <li class="price-card--features--item has-tooltip">Marketplace
 71 |                             <div class="tooltip-container"><strong>Marketplace</strong>
 72 |                                 <p>Access to world-class resellable products & services, aimed to fulfilling the digital
 73 |                                     marketing needs of your small & medium sized business clients.</p>
 74 |                             </div>
 75 |                         </li>
 76 |                         <li class="price-card--features--item has-tooltip">Business Center & Store
 77 |                             <div class="tooltip-container"><strong>Business Center & Store</strong>
 78 |                                 <p>Portal for your local business clients, featuring your agency’s branding. They can
 79 |                                     access the products purchased from your Marketplace.</p>
 80 |                             </div>
 81 |                         </li>
 82 | 
 83 |                         <li class="price-card--features--item has-tooltip">Tech support
 84 |                             <div class="tooltip-container"><strong>Tech support</strong>
 85 |                                 <p>Phone, Email and Web Chat support, <br />9am-9pm EST Mon-Fri, <br />2pm-7pm EST
 86 |                                     Sat-Sun</p>
 87 |                             </div>
 88 |                         </li>
 89 |                         <li class="price-card--features--item has-tooltip">Partner Success Manager
 90 |                             <div class="tooltip-container"><strong>Partner Success Manager</strong>
 91 |                                 <p>One-on-one with a Vendasta Success Manager, who will provide sales training so you
 92 |                                     reach your goals quickly.</p>
 93 |                             </div>
 94 |                         </li>
 95 | 
 96 | 
 97 |                     </ul>
 98 |                     <button class="btn">Get started</button>
 99 |                 </div>
100 | 
101 |                 <div class="price-card--mobile-features-toggle"></div>
102 | 
103 |             </div>
104 |             <!-- End of Card -->
105 | 
106 |             <!-- ================================ -->
107 |             <!-- = Enterprise Subscription Card = -->
108 |             <!-- ================================ -->
109 |             <div class="price-card price-card--hero">
110 | 
111 |                 <div class="price-card--header">
112 |                     <h4>Enterprise</h4>
113 | 
114 |                 </div>
115 |                 <div class="price-card--hero-text">
116 |                     Most Popular Plan
117 |                 </div>
118 | 
119 |                 <div class="price-card--price">
120 |                     <div class="price-card--price-text">
121 |                         <div class="price-card--price-number toggle-price-content odometer" data-price-monthly="999"
122 |                             data-price-yearly="900">900</div>
123 |                     </div>
124 |                     <div class="price-card--price-conditions">
125 |                         <div class="toggle-price-content" data-price-monthly="Billed Monthly"
126 |                             data-price-yearly="Billed Annually">Billed Annually</div>
127 | 
128 |                     </div>
129 |                 </div>
130 | 
131 |                 <div class="price-card--features">
132 |                     <ul class="price-card--features--list">
133 | 
134 |                         <li class="price-card--features--item has-tooltip">Sales & Marketing Platform
135 |                             <div class="tooltip-container"><strong>Sales & Marketing Platform</strong>
136 |                                 <p>Vendasta's end-to-end sales solution for companies that serve local businesses, and
137 |                                     want to grow digital revenue.</p>
138 |                             </div>
139 |                         </li>
140 |                         <li class="price-card--features--item has-tooltip">Snapshot & Campaigns
141 |                             <div class="tooltip-container"><strong>Snapshot & Campaigns</strong>
142 |                                 <p>Access to award-winning automated needs-assessment and email marketing automation.
143 |                                     Unlimited emails sent to unlimited accounts.</p>
144 |                             </div>
145 |                         </li>
146 |                         <li class="price-card--features--item has-tooltip">Marketplace
147 |                             <div class="tooltip-container"><strong>Marketplace</strong>
148 |                                 <p>Access to world-class resellable products & services, aimed to fulfilling the digital
149 |                                     marketing needs of your small & medium sized business clients.</p>
150 |                             </div>
151 |                         </li>
152 |                         <li class="price-card--features--item has-tooltip">Business Center & Store
153 |                             <div class="tooltip-container"><strong>Business Center & Store</strong>
154 |                                 <p>Portal for your local business clients, featuring your agency’s branding. They can
155 |                                     access the products purchased from your Marketplace.</p>
156 |                             </div>
157 |                         </li>
158 | 
159 |                         <li class="price-card--features--item has-tooltip">Tech support
160 |                             <div class="tooltip-container"><strong>Tech support</strong>
161 |                                 <p>Phone, Email and Web Chat support, <br />9am-9pm EST Mon-Fri, <br />2pm-7pm EST
162 |                                     Sat-Sun</p>
163 |                             </div>
164 |                         </li>
165 |                         <li class="price-card--features--item has-tooltip">Partner Success Manager
166 |                             <div class="tooltip-container"><strong>Partner Success Manager</strong>
167 |                                 <p>One-on-one with a Vendasta Success Manager, who will provide sales training so you
168 |                                     reach your goals quickly.</p>
169 |                             </div>
170 |                         </li>
171 | 
172 | 
173 |                     </ul>
174 |                     <button class="btn-pro">Get started</button>
175 |                 </div>
176 | 
177 |                 <div class="price-card--mobile-features-toggle"></div>
178 | 
179 |             </div>
180 | 
181 |             <div class="price-card">
182 | 
183 |                 <div class="price-card--header">
184 |                     <h4>Basic</h4>
185 | 
186 |                 </div>
187 | 
188 |                 <div class="price-card--price">
189 |                     <div class="price-card--price-text only-yearly">
190 |                         <div class="price-card--price-number odometer">249</div>
191 |                         <div class="only-yearly--text"><span>Only Available Yearly</span></div>
192 |                     </div>
193 |                     <div class="price-card--price-conditions">
194 |                         <div>Billed Annually</div>
195 | 
196 |                     </div>
197 |                 </div>
198 | 
199 | 
200 | 
201 |                 <div class="price-card--features">
202 |                     <ul class="price-card--features--list">
203 | 
204 |                         <li class="price-card--features--item has-tooltip">Sales & Marketing Platform
205 |                             <div class="tooltip-container"><strong>Sales & Marketing Platform</strong>
206 |                                 <p>Vendasta's end-to-end sales solution for companies that serve local businesses, and
207 |                                     want to grow digital revenue.</p>
208 |                             </div>
209 |                         </li>
210 |                         <li class="price-card--features--item has-tooltip">Snapshot & Campaigns
211 |                             <div class="tooltip-container"><strong>Snapshot & Campaigns</strong>
212 |                                 <p>Access to award-winning automated needs-assessment and email marketing automation.
213 |                                     Unlimited emails sent to unlimited accounts.</p>
214 |                             </div>
215 |                         </li>
216 |                         <li class="price-card--features--item has-tooltip">Marketplace
217 |                             <div class="tooltip-container"><strong>Marketplace</strong>
218 |                                 <p>Access to world-class resellable products & services, aimed to fulfilling the digital
219 |                                     marketing needs of your small & medium sized business clients.</p>
220 |                             </div>
221 |                         </li>
222 |                         <li class="price-card--features--item has-tooltip">Business Center & Store
223 |                             <div class="tooltip-container"><strong>Business Center & Store</strong>
224 |                                 <p>Portal for your local business clients, featuring your agency’s branding. They can
225 |                                     access the products purchased from your Marketplace.</p>
226 |                             </div>
227 |                         </li>
228 |                         <li class="price-card--features--item has-tooltip">Tech support (limited)
229 |                             <div class="tooltip-container"><strong>Tech support (limited)</strong>
230 |                                 <p>Email and Web Chat support, <br />9am-9pm EST Mon-Fri, <br />2pm-7pm EST Sat-Sun</p>
231 |                             </div>
232 |                         </li>
233 | 
234 |                         <li class="price-card--features--item has-tooltip tooltip-disabled features-disabled">Partner
235 |                             Success Manager
236 |                             <div class="tooltip-container"><strong>Partner Success Manager</strong>
237 |                                 <p>One-on-one with a Vendasta Success Manager, who will provide sales training so you
238 |                                     reach your goals quickly.</p>
239 |                             </div>
240 |                         </li>
241 | 
242 |                     </ul>
243 |                     <button class="btn">Get started</button>
244 |                 </div>
245 | 
246 |                 <div class="price-card--mobile-features-toggle"></div>
247 | 
248 |             </div>
249 |             <!-- End of Card -->
250 | 
251 |         </div>
252 |         <!--    end of pricing-cards -->
253 | 
254 | 
255 |     </section>
256 | </body>
257 | 
258 | </html>


--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
  1 | {
  2 |   "compilerOptions": {
  3 |     /* Visit https://aka.ms/tsconfig to read more about this file */
  4 | 
  5 |     /* Projects */
  6 |     // "incremental": true,                              /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
  7 |     // "composite": true,                                /* Enable constraints that allow a TypeScript project to be used with project references. */
  8 |     // "tsBuildInfoFile": "./.tsbuildinfo",              /* Specify the path to .tsbuildinfo incremental compilation file. */
  9 |     // "disableSourceOfProjectReferenceRedirect": true,  /* Disable preferring source files instead of declaration files when referencing composite projects. */
 10 |     // "disableSolutionSearching": true,                 /* Opt a project out of multi-project reference checking when editing. */
 11 |     // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
 12 | 
 13 |     /* Language and Environment */
 14 |     "target": "es2016" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
 15 |     // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
 16 |     // "jsx": "preserve",                                /* Specify what JSX code is generated. */
 17 |     // "experimentalDecorators": true,                   /* Enable experimental support for TC39 stage 2 draft decorators. */
 18 |     // "emitDecoratorMetadata": true,                    /* Emit design-type metadata for decorated declarations in source files. */
 19 |     // "jsxFactory": "",                                 /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
 20 |     // "jsxFragmentFactory": "",                         /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
 21 |     // "jsxImportSource": "",                            /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
 22 |     // "reactNamespace": "",                             /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
 23 |     // "noLib": true,                                    /* Disable including any library files, including the default lib.d.ts. */
 24 |     // "useDefineForClassFields": true,                  /* Emit ECMAScript-standard-compliant class fields. */
 25 |     // "moduleDetection": "auto",                        /* Control what method is used to detect module-format JS files. */
 26 | 
 27 |     /* Modules */
 28 |     "module": "commonjs" /* Specify what module code is generated. */,
 29 |     // "rootDir": "./",                                  /* Specify the root folder within your source files. */
 30 |     // "moduleResolution": "node",                       /* Specify how TypeScript looks up a file from a given module specifier. */
 31 |     // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */
 32 |     // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
 33 |     // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */
 34 |     // "typeRoots": [],                                  /* Specify multiple folders that act like './node_modules/@types'. */
 35 |     // "types": [],                                      /* Specify type package names to be included without being referenced in a source file. */
 36 |     // "allowUmdGlobalAccess": true,                     /* Allow accessing UMD globals from modules. */
 37 |     // "moduleSuffixes": [],                             /* List of file name suffixes to search when resolving a module. */
 38 |     // "resolveJsonModule": true,                        /* Enable importing .json files. */
 39 |     // "noResolve": true,                                /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
 40 | 
 41 |     /* JavaScript Support */
 42 |     // "allowJs": true,                                  /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
 43 |     // "checkJs": true,                                  /* Enable error reporting in type-checked JavaScript files. */
 44 |     // "maxNodeModuleJsDepth": 1,                        /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
 45 | 
 46 |     /* Emit */
 47 |     "declaration": true /* Generate .d.ts files from TypeScript and JavaScript files in your project. */,
 48 |     // "declarationMap": true,                           /* Create sourcemaps for d.ts files. */
 49 |     // "emitDeclarationOnly": true,                      /* Only output d.ts files and not JavaScript files. */
 50 |     "sourceMap": true /* Create source map files for emitted JavaScript files. */,
 51 |     // "outFile": "./",                                  /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
 52 |     "outDir": "./dist" /* Specify an output folder for all emitted files. */,
 53 |     // "removeComments": true,                           /* Disable emitting comments. */
 54 |     // "noEmit": true,                                   /* Disable emitting files from a compilation. */
 55 |     // "importHelpers": true,                            /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
 56 |     // "importsNotUsedAsValues": "remove",               /* Specify emit/checking behavior for imports that are only used for types. */
 57 |     // "downlevelIteration": true,                       /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
 58 |     // "sourceRoot": "",                                 /* Specify the root path for debuggers to find the reference source code. */
 59 |     // "mapRoot": "",                                    /* Specify the location where debugger should locate map files instead of generated locations. */
 60 |     // "inlineSourceMap": true,                          /* Include sourcemap files inside the emitted JavaScript. */
 61 |     // "inlineSources": true,                            /* Include source code in the sourcemaps inside the emitted JavaScript. */
 62 |     // "emitBOM": true,                                  /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
 63 |     // "newLine": "crlf",                                /* Set the newline character for emitting files. */
 64 |     // "stripInternal": true,                            /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
 65 |     // "noEmitHelpers": true,                            /* Disable generating custom helper functions like '__extends' in compiled output. */
 66 |     // "noEmitOnError": true,                            /* Disable emitting files if any type checking errors are reported. */
 67 |     // "preserveConstEnums": true,                       /* Disable erasing 'const enum' declarations in generated code. */
 68 |     // "declarationDir": "./",                           /* Specify the output directory for generated declaration files. */
 69 |     // "preserveValueImports": true,                     /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
 70 | 
 71 |     /* Interop Constraints */
 72 |     // "isolatedModules": true,                          /* Ensure that each file can be safely transpiled without relying on other imports. */
 73 |     // "allowSyntheticDefaultImports": true,             /* Allow 'import x from y' when a module doesn't have a default export. */
 74 |     "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
 75 |     // "preserveSymlinks": true,                         /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
 76 |     "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
 77 | 
 78 |     /* Type Checking */
 79 |     "strict": true /* Enable all strict type-checking options. */,
 80 |     // "noImplicitAny": true,                            /* Enable error reporting for expressions and declarations with an implied 'any' type. */
 81 |     // "strictNullChecks": true,                         /* When type checking, take into account 'null' and 'undefined'. */
 82 |     // "strictFunctionTypes": true,                      /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
 83 |     // "strictBindCallApply": true,                      /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
 84 |     // "strictPropertyInitialization": true,             /* Check for class properties that are declared but not set in the constructor. */
 85 |     // "noImplicitThis": true,                           /* Enable error reporting when 'this' is given the type 'any'. */
 86 |     // "useUnknownInCatchVariables": true,               /* Default catch clause variables as 'unknown' instead of 'any'. */
 87 |     // "alwaysStrict": true,                             /* Ensure 'use strict' is always emitted. */
 88 |     // "noUnusedLocals": true,                           /* Enable error reporting when local variables aren't read. */
 89 |     // "noUnusedParameters": true,                       /* Raise an error when a function parameter isn't read. */
 90 |     // "exactOptionalPropertyTypes": true,               /* Interpret optional property types as written, rather than adding 'undefined'. */
 91 |     // "noImplicitReturns": true,                        /* Enable error reporting for codepaths that do not explicitly return in a function. */
 92 |     // "noFallthroughCasesInSwitch": true,               /* Enable error reporting for fallthrough cases in switch statements. */
 93 |     // "noUncheckedIndexedAccess": true,                 /* Add 'undefined' to a type when accessed using an index. */
 94 |     // "noImplicitOverride": true,                       /* Ensure overriding members in derived classes are marked with an override modifier. */
 95 |     // "noPropertyAccessFromIndexSignature": true,       /* Enforces using indexed accessors for keys declared using an indexed type. */
 96 |     // "allowUnusedLabels": true,                        /* Disable error reporting for unused labels. */
 97 |     // "allowUnreachableCode": true,                     /* Disable error reporting for unreachable code. */
 98 | 
 99 |     /* Completeness */
100 |     // "skipDefaultLibCheck": true,                      /* Skip type checking .d.ts files that are included with TypeScript. */
101 |     "skipLibCheck": true /* Skip type checking all .d.ts files. */
102 |   },
103 |   "include": ["src/**/*.ts"]
104 | }
105 | 


--------------------------------------------------------------------------------