├── .env
├── .eslintrc.js
├── .gitignore
├── .prettierrc
├── README.md
├── nest-cli.json
├── package.json
├── pnpm-lock.yaml
├── src
├── amazon
│ ├── amazon.controller.spec.ts
│ ├── amazon.controller.ts
│ ├── amazon.module.ts
│ ├── amazon.service.spec.ts
│ └── amazon.service.ts
├── app.module.ts
└── main.ts
├── test
├── app.e2e-spec.ts
└── jest-e2e.json
├── tsconfig.build.json
└── tsconfig.json
/.env:
--------------------------------------------------------------------------------
1 | SBR_WS_ENDPOINT=wss://brd-customer-hl_652c1e33-zone-scraping_browser1:gpfgnn2qjm98@brd.superproxy.io:9222
--------------------------------------------------------------------------------
/.eslintrc.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | parser: '@typescript-eslint/parser',
3 | parserOptions: {
4 | project: 'tsconfig.json',
5 | tsconfigRootDir: __dirname,
6 | sourceType: 'module',
7 | },
8 | plugins: ['@typescript-eslint/eslint-plugin'],
9 | extends: [
10 | 'plugin:@typescript-eslint/recommended',
11 | 'plugin:prettier/recommended',
12 | ],
13 | root: true,
14 | env: {
15 | node: true,
16 | jest: true,
17 | },
18 | ignorePatterns: ['.eslintrc.js'],
19 | rules: {
20 | '@typescript-eslint/interface-name-prefix': 'off',
21 | '@typescript-eslint/explicit-function-return-type': 'off',
22 | '@typescript-eslint/explicit-module-boundary-types': 'off',
23 | '@typescript-eslint/no-explicit-any': 'off',
24 | },
25 | };
26 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # compiled output
2 | /dist
3 | /node_modules
4 |
5 | # Logs
6 | logs
7 | *.log
8 | npm-debug.log*
9 | pnpm-debug.log*
10 | yarn-debug.log*
11 | yarn-error.log*
12 | lerna-debug.log*
13 |
14 | # OS
15 | .DS_Store
16 |
17 | # Tests
18 | /coverage
19 | /.nyc_output
20 |
21 | # IDEs and editors
22 | /.idea
23 | .project
24 | .classpath
25 | .c9/
26 | *.launch
27 | .settings/
28 | *.sublime-workspace
29 |
30 | # IDE - VSCode
31 | .vscode/*
32 | !.vscode/settings.json
33 | !.vscode/tasks.json
34 | !.vscode/launch.json
35 | !.vscode/extensions.json
--------------------------------------------------------------------------------
/.prettierrc:
--------------------------------------------------------------------------------
1 | {
2 | "singleQuote": true,
3 | "trailingComma": "all"
4 | }
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | [circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456
6 | [circleci-url]: https://circleci.com/gh/nestjs/nest
7 |
8 | A progressive Node.js framework for building efficient and scalable server-side applications.
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
24 |
25 | ## Description
26 |
27 | [Nest](https://github.com/nestjs/nest) framework TypeScript starter repository.
28 |
29 | ## Installation
30 |
31 | ```bash
32 | $ pnpm install
33 | ```
34 |
35 | ## Running the app
36 |
37 | ```bash
38 | # development
39 | $ pnpm run start
40 |
41 | # watch mode
42 | $ pnpm run start:dev
43 |
44 | # production mode
45 | $ pnpm run start:prod
46 | ```
47 |
48 | ## Test
49 |
50 | ```bash
51 | # unit tests
52 | $ pnpm run test
53 |
54 | # e2e tests
55 | $ pnpm run test:e2e
56 |
57 | # test coverage
58 | $ pnpm run test:cov
59 | ```
60 |
61 | ## Support
62 |
63 | Nest is an MIT-licensed open source project. It can grow thanks to the sponsors and support by the amazing backers. If you'd like to join them, please [read more here](https://docs.nestjs.com/support).
64 |
65 | ## Stay in touch
66 |
67 | - Author - [Kamil Myśliwiec](https://kamilmysliwiec.com)
68 | - Website - [https://nestjs.com](https://nestjs.com/)
69 | - Twitter - [@nestframework](https://twitter.com/nestframework)
70 |
71 | ## License
72 |
73 | Nest is [MIT licensed](LICENSE).
74 |
--------------------------------------------------------------------------------
/nest-cli.json:
--------------------------------------------------------------------------------
1 | {
2 | "$schema": "https://json.schemastore.org/nest-cli",
3 | "collection": "@nestjs/schematics",
4 | "sourceRoot": "src",
5 | "compilerOptions": {
6 | "deleteOutDir": true
7 | }
8 | }
9 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "nestjs-scraping",
3 | "version": "0.0.1",
4 | "description": "",
5 | "author": "",
6 | "private": true,
7 | "license": "UNLICENSED",
8 | "scripts": {
9 | "build": "nest build",
10 | "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
11 | "start": "nest start",
12 | "start:dev": "nest start --watch",
13 | "start:debug": "nest start --debug --watch",
14 | "start:prod": "node dist/main",
15 | "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
16 | "test": "jest",
17 | "test:watch": "jest --watch",
18 | "test:cov": "jest --coverage",
19 | "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
20 | "test:e2e": "jest --config ./test/jest-e2e.json"
21 | },
22 | "dependencies": {
23 | "@nestjs/common": "^10.0.0",
24 | "@nestjs/config": "^3.0.0",
25 | "@nestjs/core": "^10.0.0",
26 | "@nestjs/platform-express": "^10.0.0",
27 | "puppeteer-core": "^21.0.3",
28 | "reflect-metadata": "^0.1.13",
29 | "rxjs": "^7.8.1"
30 | },
31 | "devDependencies": {
32 | "@nestjs/cli": "^10.0.0",
33 | "@nestjs/schematics": "^10.0.0",
34 | "@nestjs/testing": "^10.0.0",
35 | "@types/express": "^4.17.17",
36 | "@types/jest": "^29.5.2",
37 | "@types/node": "^20.3.1",
38 | "@types/supertest": "^2.0.12",
39 | "@typescript-eslint/eslint-plugin": "^6.0.0",
40 | "@typescript-eslint/parser": "^6.0.0",
41 | "eslint": "^8.42.0",
42 | "eslint-config-prettier": "^9.0.0",
43 | "eslint-plugin-prettier": "^5.0.0",
44 | "jest": "^29.5.0",
45 | "prettier": "^3.0.0",
46 | "source-map-support": "^0.5.21",
47 | "supertest": "^6.3.3",
48 | "ts-jest": "^29.1.0",
49 | "ts-loader": "^9.4.3",
50 | "ts-node": "^10.9.1",
51 | "tsconfig-paths": "^4.2.0",
52 | "typescript": "^5.1.3"
53 | },
54 | "jest": {
55 | "moduleFileExtensions": [
56 | "js",
57 | "json",
58 | "ts"
59 | ],
60 | "rootDir": "src",
61 | "testRegex": ".*\\.spec\\.ts$",
62 | "transform": {
63 | "^.+\\.(t|j)s$": "ts-jest"
64 | },
65 | "collectCoverageFrom": [
66 | "**/*.(t|j)s"
67 | ],
68 | "coverageDirectory": "../coverage",
69 | "testEnvironment": "node"
70 | }
71 | }
72 |
--------------------------------------------------------------------------------
/src/amazon/amazon.controller.spec.ts:
--------------------------------------------------------------------------------
1 | import { Test, TestingModule } from '@nestjs/testing';
2 | import { AmazonController } from './amazon.controller';
3 |
4 | describe('AmazonController', () => {
5 | let controller: AmazonController;
6 |
7 | beforeEach(async () => {
8 | const module: TestingModule = await Test.createTestingModule({
9 | controllers: [AmazonController],
10 | }).compile();
11 |
12 | controller = module.get(AmazonController);
13 | });
14 |
15 | it('should be defined', () => {
16 | expect(controller).toBeDefined();
17 | });
18 | });
19 |
--------------------------------------------------------------------------------
/src/amazon/amazon.controller.ts:
--------------------------------------------------------------------------------
1 | import { Controller, Get, Query } from '@nestjs/common';
2 | import { AmazonService } from './amazon.service';
3 |
4 | @Controller('amazon')
5 | export class AmazonController {
6 | constructor(private readonly amazonService: AmazonService) {}
7 |
8 | @Get('products')
9 | getProducts(@Query('product') product: string) {
10 | return this.amazonService.getProducts(product);
11 | }
12 | }
13 |
--------------------------------------------------------------------------------
/src/amazon/amazon.module.ts:
--------------------------------------------------------------------------------
1 | import { Module } from '@nestjs/common';
2 | import { AmazonController } from './amazon.controller';
3 | import { AmazonService } from './amazon.service';
4 |
5 | @Module({
6 | controllers: [AmazonController],
7 | providers: [AmazonService],
8 | })
9 | export class AmazonModule {}
10 |
--------------------------------------------------------------------------------
/src/amazon/amazon.service.spec.ts:
--------------------------------------------------------------------------------
1 | import { Test, TestingModule } from '@nestjs/testing';
2 | import { AmazonService } from './amazon.service';
3 |
4 | describe('AmazonService', () => {
5 | let service: AmazonService;
6 |
7 | beforeEach(async () => {
8 | const module: TestingModule = await Test.createTestingModule({
9 | providers: [AmazonService],
10 | }).compile();
11 |
12 | service = module.get(AmazonService);
13 | });
14 |
15 | it('should be defined', () => {
16 | expect(service).toBeDefined();
17 | });
18 | });
19 |
--------------------------------------------------------------------------------
/src/amazon/amazon.service.ts:
--------------------------------------------------------------------------------
1 | import { Injectable } from '@nestjs/common';
2 | import { ConfigService } from '@nestjs/config';
3 | import puppeteer from 'puppeteer-core';
4 |
5 | @Injectable()
6 | export class AmazonService {
7 | constructor(private readonly configService: ConfigService) {}
8 |
9 | async getProducts(products: string) {
10 | const browser = await puppeteer.connect({
11 | browserWSEndpoint: this.configService.getOrThrow('SBR_WS_ENDPOINT'),
12 | });
13 | try {
14 | const page = await browser.newPage();
15 | page.setDefaultNavigationTimeout(2 * 60 * 1000);
16 | await Promise.all([
17 | page.waitForNavigation(),
18 | page.goto('https://amazon.com'),
19 | ]);
20 | await page.type('#twotabsearchtextbox', products);
21 | await Promise.all([
22 | page.waitForNavigation(),
23 | page.click('#nav-search-submit-button'),
24 | ]);
25 | return await page.$$eval(
26 | '.s-search-results .s-card-container',
27 | (resultItems) => {
28 | return resultItems.map((resultItem) => {
29 | const url = resultItem.querySelector('a').href;
30 | const title = resultItem.querySelector(
31 | '.s-title-instructions-style span',
32 | )?.textContent;
33 | const price = resultItem.querySelector(
34 | '.a-price .a-offscreen',
35 | ).textContent;
36 | return {
37 | url,
38 | title,
39 | price,
40 | };
41 | });
42 | },
43 | );
44 | } finally {
45 | await browser.close();
46 | }
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/src/app.module.ts:
--------------------------------------------------------------------------------
1 | import { Module } from '@nestjs/common';
2 | import { AmazonModule } from './amazon/amazon.module';
3 | import { ConfigModule } from '@nestjs/config';
4 |
5 | @Module({
6 | imports: [ConfigModule.forRoot({ isGlobal: true }), AmazonModule],
7 | controllers: [],
8 | providers: [],
9 | })
10 | export class AppModule {}
11 |
--------------------------------------------------------------------------------
/src/main.ts:
--------------------------------------------------------------------------------
1 | import { NestFactory } from '@nestjs/core';
2 | import { AppModule } from './app.module';
3 |
4 | async function bootstrap() {
5 | const app = await NestFactory.create(AppModule);
6 | await app.listen(3000);
7 | }
8 | bootstrap();
9 |
--------------------------------------------------------------------------------
/test/app.e2e-spec.ts:
--------------------------------------------------------------------------------
1 | import { Test, TestingModule } from '@nestjs/testing';
2 | import { INestApplication } from '@nestjs/common';
3 | import * as request from 'supertest';
4 | import { AppModule } from './../src/app.module';
5 |
6 | describe('AppController (e2e)', () => {
7 | let app: INestApplication;
8 |
9 | beforeEach(async () => {
10 | const moduleFixture: TestingModule = await Test.createTestingModule({
11 | imports: [AppModule],
12 | }).compile();
13 |
14 | app = moduleFixture.createNestApplication();
15 | await app.init();
16 | });
17 |
18 | it('/ (GET)', () => {
19 | return request(app.getHttpServer())
20 | .get('/')
21 | .expect(200)
22 | .expect('Hello World!');
23 | });
24 | });
25 |
--------------------------------------------------------------------------------
/test/jest-e2e.json:
--------------------------------------------------------------------------------
1 | {
2 | "moduleFileExtensions": ["js", "json", "ts"],
3 | "rootDir": ".",
4 | "testEnvironment": "node",
5 | "testRegex": ".e2e-spec.ts$",
6 | "transform": {
7 | "^.+\\.(t|j)s$": "ts-jest"
8 | }
9 | }
10 |
--------------------------------------------------------------------------------
/tsconfig.build.json:
--------------------------------------------------------------------------------
1 | {
2 | "extends": "./tsconfig.json",
3 | "exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
4 | }
5 |
--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "module": "commonjs",
4 | "declaration": true,
5 | "removeComments": true,
6 | "emitDecoratorMetadata": true,
7 | "experimentalDecorators": true,
8 | "allowSyntheticDefaultImports": true,
9 | "target": "ES2021",
10 | "sourceMap": true,
11 | "outDir": "./dist",
12 | "baseUrl": "./",
13 | "incremental": true,
14 | "skipLibCheck": true,
15 | "strictNullChecks": false,
16 | "noImplicitAny": false,
17 | "strictBindCallApply": false,
18 | "forceConsistentCasingInFileNames": false,
19 | "noFallthroughCasesInSwitch": false
20 | }
21 | }
22 |
--------------------------------------------------------------------------------