├── .gitignore
├── README.md
├── client
├── .eslintrc.cjs
├── .gitignore
├── index.html
├── package-lock.json
├── package.json
├── postcss.config.js
├── public
│ └── vite.svg
├── src
│ ├── App.css
│ ├── App.jsx
│ ├── assets
│ │ └── react.svg
│ ├── index.css
│ └── main.jsx
├── tailwind.config.js
└── vite.config.js
└── server
├── app.js
├── nodemon.json
├── package-lock.json
├── package.json
└── scrapers
├── dynamincSiteScraper.js
└── staticSiteScraper.js
/.gitignore:
--------------------------------------------------------------------------------
1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
2 |
3 | # dependencies
4 | */data
5 | */node_modules
6 | /.pnp
7 | .pnp.js
8 |
9 | # testing
10 | /coverage
11 |
12 | # production
13 | /build
14 |
15 | # misc
16 | .DS_Store
17 | .env.local
18 | .env.development.local
19 | .env.test.local
20 | .env.production.local
21 |
22 | npm-debug.log*
23 | yarn-debug.log*
24 | yarn-error.log*
25 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # React Node Web Scraper
2 |
3 | ## Overview
4 |
5 | This project is a web scraper that consists of a client and a server. The client is a [React](https://react.dev/) app bootstrapped with [vite](https://vitejs.dev/) and the server uses Express.js, Axios, Cheerio, and Puppeteer for web scraping.
6 |
7 | ## Project Structure
8 |
9 | The project has two main folders:
10 |
11 | - `client`: This is where the front-end code resides.
12 | - `server`: This contains the back-end code for the web scraper.
13 |
14 | ## Technologies Used
15 |
16 | - **Frontend**: React, Vite
17 | - **Backend**: Node.js, Express, Axios, Cheerio, Puppeteer
18 |
19 | ## Getting Started
20 |
21 | ### Prerequisites
22 |
23 | - Node.js (v14 or later)
24 | - npm or yarn
25 |
26 | ### Installation
27 |
28 | 1. Clone the repository:
29 |
30 | ```bash
31 | git clone https://github.com/yourusername/react-node-web-scraper.git
32 | cd react-node-web-scraper
33 | ```
34 |
35 | 2. Install dependencies for both server and client applications using:
36 | ```bash
37 | npm install
38 | # or
39 | yarn
40 | ```
41 |
42 | ### Running the Application
43 |
44 | ## Server
45 |
46 | ### Available Scripts
47 |
48 | In the server directory, you can run:
49 |
50 | #### `npm start`
51 |
52 | Runs the server in development mode with nodemon. Open http://localhost:8000 to view the response from `/` endpoint in the browser or any other api client.
53 |
54 | ## Client
55 |
56 | ### Available Scripts:
57 |
58 | In the client directory, you can run:
59 |
60 | ```bash
61 | npm run dev
62 | # or
63 | yarn dev
64 | ```
65 |
66 | Runs the app in development mode. Open http://localhost:5173 to view it in the browser.
67 |
68 | ```bash
69 | npm run build
70 | # or
71 | yarn build
72 | ```
73 |
74 | Builds the app for production to the `dist` folder.
75 |
76 | ```bash
77 | npm run preview
78 | # or
79 | yarn preview
80 | ```
81 |
82 | Preview the production build locally.
83 |
--------------------------------------------------------------------------------
/client/.eslintrc.cjs:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | env: {
3 | browser: true,
4 | es2021: true,
5 | },
6 | extends: 'airbnb',
7 | overrides: [
8 | {
9 | env: {
10 | node: true,
11 | },
12 | files: [
13 | '.eslintrc.{js,cjs}',
14 | ],
15 | parserOptions: {
16 | sourceType: 'script',
17 | },
18 | },
19 | ],
20 | parserOptions: {
21 | ecmaVersion: 'latest',
22 | sourceType: 'module',
23 | },
24 | rules: {
25 | },
26 | };
27 |
--------------------------------------------------------------------------------
/client/.gitignore:
--------------------------------------------------------------------------------
1 | # Logs
2 | logs
3 | *.log
4 | npm-debug.log*
5 | yarn-debug.log*
6 | yarn-error.log*
7 | pnpm-debug.log*
8 | lerna-debug.log*
9 |
10 | node_modules
11 | dist
12 | dist-ssr
13 | *.local
14 |
15 | # Editor directories and files
16 | .vscode/*
17 | !.vscode/extensions.json
18 | .idea
19 | .DS_Store
20 | *.suo
21 | *.ntvs*
22 | *.njsproj
23 | *.sln
24 | *.sw?
25 |
--------------------------------------------------------------------------------
/client/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 | Vite + React
8 |
9 |
10 |
11 |
12 |
13 |
14 |
--------------------------------------------------------------------------------
/client/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "client",
3 | "private": true,
4 | "version": "0.0.0",
5 | "type": "module",
6 | "scripts": {
7 | "dev": "vite",
8 | "build": "vite build",
9 | "lint": "eslint . --ext js,jsx --report-unused-disable-directives --max-warnings 0",
10 | "preview": "vite preview"
11 | },
12 | "dependencies": {
13 | "@ant-design/icons": "^5.4.0",
14 | "@tanstack/react-query": "^5.56.2",
15 | "antd": "^5.20.6",
16 | "react": "^18.2.0",
17 | "react-dom": "^18.2.0"
18 | },
19 | "devDependencies": {
20 | "@tanstack/eslint-plugin-query": "^5.56.1",
21 | "@types/react": "^18.2.15",
22 | "@types/react-dom": "^18.2.7",
23 | "@vitejs/plugin-react-swc": "^3.3.2",
24 | "autoprefixer": "^10.4.20",
25 | "eslint": "^8.57.1",
26 | "eslint-config-airbnb": "^19.0.4",
27 | "eslint-plugin-import": "^2.30.0",
28 | "eslint-plugin-jsx-a11y": "^6.10.0",
29 | "eslint-plugin-react": "^7.36.1",
30 | "eslint-plugin-react-hooks": "^4.6.2",
31 | "eslint-plugin-react-refresh": "^0.4.3",
32 | "postcss": "^8.4.47",
33 | "tailwindcss": "^3.4.12",
34 | "vite": "^4.4.5"
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/client/postcss.config.js:
--------------------------------------------------------------------------------
1 | export default {
2 | plugins: {
3 | tailwindcss: {},
4 | autoprefixer: {},
5 | },
6 | }
7 |
--------------------------------------------------------------------------------
/client/public/vite.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/client/src/App.css:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/codegratia/react-node-web-scraper/ff0c264659ce0c7af9296025545be33c7d49cf71/client/src/App.css
--------------------------------------------------------------------------------
/client/src/App.jsx:
--------------------------------------------------------------------------------
1 | import { useEffect, useState } from "react";
2 | import { Row, Col, Card, Typography, Divider } from "antd";
3 | import "./App.css";
4 |
5 | const { Title } = Typography;
6 | const { Meta } = Card;
7 |
8 | function App() {
9 | const [data, setData] = useState(null);
10 |
11 | useEffect(() => {
12 | fetch("http://localhost:8000/api/v1/items")
13 | .then((response) => response.json())
14 | .then((data) => setData(data))
15 | .catch((error) => console.error("Error:", error));
16 | }, []);
17 |
18 | return (
19 |
20 |
24 |
25 | {data &&
26 | data.map((item, index) => (
27 |
28 | }
31 | >
32 |
33 |
34 |
35 | ))}
36 |
37 |