├── .env ├── .eslintrc.js ├── .gitignore ├── .prettierrc ├── README.md ├── nest-cli.json ├── package.json ├── pnpm-lock.yaml ├── src ├── amazon │ ├── amazon.controller.spec.ts │ ├── amazon.controller.ts │ ├── amazon.module.ts │ ├── amazon.service.spec.ts │ └── amazon.service.ts ├── app.module.ts └── main.ts ├── test ├── app.e2e-spec.ts └── jest-e2e.json ├── tsconfig.build.json └── tsconfig.json /.env: -------------------------------------------------------------------------------- 1 | SBR_WS_ENDPOINT=wss://brd-customer-hl_652c1e33-zone-scraping_browser1:gpfgnn2qjm98@brd.superproxy.io:9222 -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | parser: '@typescript-eslint/parser', 3 | parserOptions: { 4 | project: 'tsconfig.json', 5 | tsconfigRootDir: __dirname, 6 | sourceType: 'module', 7 | }, 8 | plugins: ['@typescript-eslint/eslint-plugin'], 9 | extends: [ 10 | 'plugin:@typescript-eslint/recommended', 11 | 'plugin:prettier/recommended', 12 | ], 13 | root: true, 14 | env: { 15 | node: true, 16 | jest: true, 17 | }, 18 | ignorePatterns: ['.eslintrc.js'], 19 | rules: { 20 | '@typescript-eslint/interface-name-prefix': 'off', 21 | '@typescript-eslint/explicit-function-return-type': 'off', 22 | '@typescript-eslint/explicit-module-boundary-types': 'off', 23 | '@typescript-eslint/no-explicit-any': 'off', 24 | }, 25 | }; 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # compiled output 2 | /dist 3 | /node_modules 4 | 5 | # Logs 6 | logs 7 | *.log 8 | npm-debug.log* 9 | pnpm-debug.log* 10 | yarn-debug.log* 11 | yarn-error.log* 12 | lerna-debug.log* 13 | 14 | # OS 15 | .DS_Store 16 | 17 | # Tests 18 | /coverage 19 | /.nyc_output 20 | 21 | # IDEs and editors 22 | /.idea 23 | .project 24 | .classpath 25 | .c9/ 26 | *.launch 27 | .settings/ 28 | *.sublime-workspace 29 | 30 | # IDE - VSCode 31 | .vscode/* 32 | !.vscode/settings.json 33 | !.vscode/tasks.json 34 | !.vscode/launch.json 35 | !.vscode/extensions.json -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true, 3 | "trailingComma": "all" 4 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | Nest Logo 3 |

4 | 5 | [circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456 6 | [circleci-url]: https://circleci.com/gh/nestjs/nest 7 | 8 |

A progressive Node.js framework for building efficient and scalable server-side applications.

9 |

10 | NPM Version 11 | Package License 12 | NPM Downloads 13 | CircleCI 14 | Coverage 15 | Discord 16 | Backers on Open Collective 17 | Sponsors on Open Collective 18 | 19 | Support us 20 | 21 |

22 | 24 | 25 | ## Description 26 | 27 | [Nest](https://github.com/nestjs/nest) framework TypeScript starter repository. 28 | 29 | ## Installation 30 | 31 | ```bash 32 | $ pnpm install 33 | ``` 34 | 35 | ## Running the app 36 | 37 | ```bash 38 | # development 39 | $ pnpm run start 40 | 41 | # watch mode 42 | $ pnpm run start:dev 43 | 44 | # production mode 45 | $ pnpm run start:prod 46 | ``` 47 | 48 | ## Test 49 | 50 | ```bash 51 | # unit tests 52 | $ pnpm run test 53 | 54 | # e2e tests 55 | $ pnpm run test:e2e 56 | 57 | # test coverage 58 | $ pnpm run test:cov 59 | ``` 60 | 61 | ## Support 62 | 63 | Nest is an MIT-licensed open source project. It can grow thanks to the sponsors and support by the amazing backers. If you'd like to join them, please [read more here](https://docs.nestjs.com/support). 64 | 65 | ## Stay in touch 66 | 67 | - Author - [Kamil Myśliwiec](https://kamilmysliwiec.com) 68 | - Website - [https://nestjs.com](https://nestjs.com/) 69 | - Twitter - [@nestframework](https://twitter.com/nestframework) 70 | 71 | ## License 72 | 73 | Nest is [MIT licensed](LICENSE). 74 | -------------------------------------------------------------------------------- /nest-cli.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/nest-cli", 3 | "collection": "@nestjs/schematics", 4 | "sourceRoot": "src", 5 | "compilerOptions": { 6 | "deleteOutDir": true 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nestjs-scraping", 3 | "version": "0.0.1", 4 | "description": "", 5 | "author": "", 6 | "private": true, 7 | "license": "UNLICENSED", 8 | "scripts": { 9 | "build": "nest build", 10 | "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", 11 | "start": "nest start", 12 | "start:dev": "nest start --watch", 13 | "start:debug": "nest start --debug --watch", 14 | "start:prod": "node dist/main", 15 | "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", 16 | "test": "jest", 17 | "test:watch": "jest --watch", 18 | "test:cov": "jest --coverage", 19 | "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", 20 | "test:e2e": "jest --config ./test/jest-e2e.json" 21 | }, 22 | "dependencies": { 23 | "@nestjs/common": "^10.0.0", 24 | "@nestjs/config": "^3.0.0", 25 | "@nestjs/core": "^10.0.0", 26 | "@nestjs/platform-express": "^10.0.0", 27 | "puppeteer-core": "^21.0.3", 28 | "reflect-metadata": "^0.1.13", 29 | "rxjs": "^7.8.1" 30 | }, 31 | "devDependencies": { 32 | "@nestjs/cli": "^10.0.0", 33 | "@nestjs/schematics": "^10.0.0", 34 | "@nestjs/testing": "^10.0.0", 35 | "@types/express": "^4.17.17", 36 | "@types/jest": "^29.5.2", 37 | "@types/node": "^20.3.1", 38 | "@types/supertest": "^2.0.12", 39 | "@typescript-eslint/eslint-plugin": "^6.0.0", 40 | "@typescript-eslint/parser": "^6.0.0", 41 | "eslint": "^8.42.0", 42 | "eslint-config-prettier": "^9.0.0", 43 | "eslint-plugin-prettier": "^5.0.0", 44 | "jest": "^29.5.0", 45 | "prettier": "^3.0.0", 46 | "source-map-support": "^0.5.21", 47 | "supertest": "^6.3.3", 48 | "ts-jest": "^29.1.0", 49 | "ts-loader": "^9.4.3", 50 | "ts-node": "^10.9.1", 51 | "tsconfig-paths": "^4.2.0", 52 | "typescript": "^5.1.3" 53 | }, 54 | "jest": { 55 | "moduleFileExtensions": [ 56 | "js", 57 | "json", 58 | "ts" 59 | ], 60 | "rootDir": "src", 61 | "testRegex": ".*\\.spec\\.ts$", 62 | "transform": { 63 | "^.+\\.(t|j)s$": "ts-jest" 64 | }, 65 | "collectCoverageFrom": [ 66 | "**/*.(t|j)s" 67 | ], 68 | "coverageDirectory": "../coverage", 69 | "testEnvironment": "node" 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/amazon/amazon.controller.spec.ts: -------------------------------------------------------------------------------- 1 | import { Test, TestingModule } from '@nestjs/testing'; 2 | import { AmazonController } from './amazon.controller'; 3 | 4 | describe('AmazonController', () => { 5 | let controller: AmazonController; 6 | 7 | beforeEach(async () => { 8 | const module: TestingModule = await Test.createTestingModule({ 9 | controllers: [AmazonController], 10 | }).compile(); 11 | 12 | controller = module.get(AmazonController); 13 | }); 14 | 15 | it('should be defined', () => { 16 | expect(controller).toBeDefined(); 17 | }); 18 | }); 19 | -------------------------------------------------------------------------------- /src/amazon/amazon.controller.ts: -------------------------------------------------------------------------------- 1 | import { Controller, Get, Query } from '@nestjs/common'; 2 | import { AmazonService } from './amazon.service'; 3 | 4 | @Controller('amazon') 5 | export class AmazonController { 6 | constructor(private readonly amazonService: AmazonService) {} 7 | 8 | @Get('products') 9 | getProducts(@Query('product') product: string) { 10 | return this.amazonService.getProducts(product); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/amazon/amazon.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { AmazonController } from './amazon.controller'; 3 | import { AmazonService } from './amazon.service'; 4 | 5 | @Module({ 6 | controllers: [AmazonController], 7 | providers: [AmazonService], 8 | }) 9 | export class AmazonModule {} 10 | -------------------------------------------------------------------------------- /src/amazon/amazon.service.spec.ts: -------------------------------------------------------------------------------- 1 | import { Test, TestingModule } from '@nestjs/testing'; 2 | import { AmazonService } from './amazon.service'; 3 | 4 | describe('AmazonService', () => { 5 | let service: AmazonService; 6 | 7 | beforeEach(async () => { 8 | const module: TestingModule = await Test.createTestingModule({ 9 | providers: [AmazonService], 10 | }).compile(); 11 | 12 | service = module.get(AmazonService); 13 | }); 14 | 15 | it('should be defined', () => { 16 | expect(service).toBeDefined(); 17 | }); 18 | }); 19 | -------------------------------------------------------------------------------- /src/amazon/amazon.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@nestjs/common'; 2 | import { ConfigService } from '@nestjs/config'; 3 | import puppeteer from 'puppeteer-core'; 4 | 5 | @Injectable() 6 | export class AmazonService { 7 | constructor(private readonly configService: ConfigService) {} 8 | 9 | async getProducts(products: string) { 10 | const browser = await puppeteer.connect({ 11 | browserWSEndpoint: this.configService.getOrThrow('SBR_WS_ENDPOINT'), 12 | }); 13 | try { 14 | const page = await browser.newPage(); 15 | page.setDefaultNavigationTimeout(2 * 60 * 1000); 16 | await Promise.all([ 17 | page.waitForNavigation(), 18 | page.goto('https://amazon.com'), 19 | ]); 20 | await page.type('#twotabsearchtextbox', products); 21 | await Promise.all([ 22 | page.waitForNavigation(), 23 | page.click('#nav-search-submit-button'), 24 | ]); 25 | return await page.$$eval( 26 | '.s-search-results .s-card-container', 27 | (resultItems) => { 28 | return resultItems.map((resultItem) => { 29 | const url = resultItem.querySelector('a').href; 30 | const title = resultItem.querySelector( 31 | '.s-title-instructions-style span', 32 | )?.textContent; 33 | const price = resultItem.querySelector( 34 | '.a-price .a-offscreen', 35 | ).textContent; 36 | return { 37 | url, 38 | title, 39 | price, 40 | }; 41 | }); 42 | }, 43 | ); 44 | } finally { 45 | await browser.close(); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/app.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { AmazonModule } from './amazon/amazon.module'; 3 | import { ConfigModule } from '@nestjs/config'; 4 | 5 | @Module({ 6 | imports: [ConfigModule.forRoot({ isGlobal: true }), AmazonModule], 7 | controllers: [], 8 | providers: [], 9 | }) 10 | export class AppModule {} 11 | -------------------------------------------------------------------------------- /src/main.ts: -------------------------------------------------------------------------------- 1 | import { NestFactory } from '@nestjs/core'; 2 | import { AppModule } from './app.module'; 3 | 4 | async function bootstrap() { 5 | const app = await NestFactory.create(AppModule); 6 | await app.listen(3000); 7 | } 8 | bootstrap(); 9 | -------------------------------------------------------------------------------- /test/app.e2e-spec.ts: -------------------------------------------------------------------------------- 1 | import { Test, TestingModule } from '@nestjs/testing'; 2 | import { INestApplication } from '@nestjs/common'; 3 | import * as request from 'supertest'; 4 | import { AppModule } from './../src/app.module'; 5 | 6 | describe('AppController (e2e)', () => { 7 | let app: INestApplication; 8 | 9 | beforeEach(async () => { 10 | const moduleFixture: TestingModule = await Test.createTestingModule({ 11 | imports: [AppModule], 12 | }).compile(); 13 | 14 | app = moduleFixture.createNestApplication(); 15 | await app.init(); 16 | }); 17 | 18 | it('/ (GET)', () => { 19 | return request(app.getHttpServer()) 20 | .get('/') 21 | .expect(200) 22 | .expect('Hello World!'); 23 | }); 24 | }); 25 | -------------------------------------------------------------------------------- /test/jest-e2e.json: -------------------------------------------------------------------------------- 1 | { 2 | "moduleFileExtensions": ["js", "json", "ts"], 3 | "rootDir": ".", 4 | "testEnvironment": "node", 5 | "testRegex": ".e2e-spec.ts$", 6 | "transform": { 7 | "^.+\\.(t|j)s$": "ts-jest" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["node_modules", "test", "dist", "**/*spec.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "declaration": true, 5 | "removeComments": true, 6 | "emitDecoratorMetadata": true, 7 | "experimentalDecorators": true, 8 | "allowSyntheticDefaultImports": true, 9 | "target": "ES2021", 10 | "sourceMap": true, 11 | "outDir": "./dist", 12 | "baseUrl": "./", 13 | "incremental": true, 14 | "skipLibCheck": true, 15 | "strictNullChecks": false, 16 | "noImplicitAny": false, 17 | "strictBindCallApply": false, 18 | "forceConsistentCasingInFileNames": false, 19 | "noFallthroughCasesInSwitch": false 20 | } 21 | } 22 | --------------------------------------------------------------------------------