├── .npmignore
├── test
    ├── public
    │   ├── index.js
    │   ├── index.css
    │   ├── nested
    │   │   ├── index.css
    │   │   └── index.html
    │   ├── nested.dot
    │   │   ├── index.dot.css
    │   │   └── index.html
    │   ├── iframe.html
    │   ├── index.html
    │   └── page.html
    ├── tsconfig.json
    ├── test-request.js
    ├── docker
    │   ├── nginx.Dockerfile
    │   ├── docker-compose.yml
    │   ├── ssr-proxy-js.config.json
    │   ├── ssr.Dockerfile
    │   └── nginx.conf
    ├── ssr-proxy-js.config.json
    ├── ssr-build-js.config.json
    ├── test-stream.js
    ├── proxy.ts
    ├── package.json
    ├── build.ts
    └── proxy.js
├── src
    ├── index.ts
    ├── plugins.ts
    ├── proxy-cache.ts
    ├── logger.ts
    ├── utils.ts
    ├── ssr-render.ts
    ├── ssr-build.ts
    ├── types.ts
    └── ssr-proxy.ts
├── .dockerignore
├── .gitignore
├── webpack.config.js
├── package.json
├── bin
    └── cli.js
├── tsconfig.json
└── README.md


/.npmignore:
--------------------------------------------------------------------------------
1 | test/


--------------------------------------------------------------------------------
/test/public/index.js:
--------------------------------------------------------------------------------
1 | document.body.innerHTML = `\n\t<p><strong>JS Imported!</strong></p>${document.body.innerHTML}`;


--------------------------------------------------------------------------------
/test/public/index.css:
--------------------------------------------------------------------------------
1 | body {
2 |     background: #222222;
3 | }
4 | 
5 | p {
6 |     margin: 50px;
7 |     color: #f5f5f5;
8 | }


--------------------------------------------------------------------------------
/test/public/nested/index.css:
--------------------------------------------------------------------------------
1 | body {
2 |     background: #777777 !important;
3 | }
4 | 
5 | p {
6 |     margin: 50px;
7 |     color: #f5f5f5;
8 | }


--------------------------------------------------------------------------------
/test/public/nested.dot/index.dot.css:
--------------------------------------------------------------------------------
1 | body {
2 |     background: #777777 !important;
3 | }
4 | 
5 | p {
6 |     margin: 50px;
7 |     color: #f5f5f5;
8 | }


--------------------------------------------------------------------------------
/test/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 |     "compilerOptions": {
3 |         "module": "commonjs",
4 |         "target": "esnext"
5 |     },
6 |     "exclude": [ "node_modules" ]
7 | }


--------------------------------------------------------------------------------
/test/test-request.js:
--------------------------------------------------------------------------------
1 | const axios = require('axios');
2 | 
3 | axios.get('http://localhost:3000/login')
4 |     .then(r => r.data)
5 |     .catch(err => err.response ? err.response.data : err.toString())
6 |     .then(console.log);


--------------------------------------------------------------------------------
/src/index.ts:
--------------------------------------------------------------------------------
1 | export * from './logger';
2 | export * from './plugins';
3 | export * from './proxy-cache';
4 | export * from './ssr-build';
5 | export * from './ssr-proxy';
6 | export * from './ssr-render';
7 | export * from './types';
8 | export * from './utils';
9 | 


--------------------------------------------------------------------------------
/.dockerignore:
--------------------------------------------------------------------------------
 1 | # Node.js .dockerignore
 2 | 
 3 | **/dist
 4 | **/logs
 5 | **/*.log
 6 | **/node_modules/
 7 | **/npm-debug.log
 8 | **/.git
 9 | **/.vscode
10 | **/.gitignore
11 | **/.dockerignore
12 | **/README.md
13 | **/LICENSE
14 | **/.editorconfig
15 | **/Dockerfile
16 | **/*.Dockerfile
17 | **/docs
18 | **/.github


--------------------------------------------------------------------------------
/test/docker/nginx.Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM nginx:1.27.0-bookworm
 2 | 
 3 | WORKDIR /app
 4 | 
 5 | RUN apt-get update
 6 | RUN apt-get install -y net-tools nano
 7 | RUN rm -rf /var/lib/apt/lists/*
 8 | 
 9 | COPY ./public/ ./public/
10 | COPY ./docker/nginx.conf /etc/nginx/
11 | 
12 | EXPOSE 8080
13 | 
14 | CMD nginx -g 'daemon off;'
15 | 


--------------------------------------------------------------------------------
/test/public/iframe.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="en">
 3 | <head>
 4 |     <meta charset="UTF-8">
 5 |     <meta http-equiv="X-UA-Compatible" content="IE=edge">
 6 |     <meta name="viewport" content="width=device-width, initial-scale=1.0">
 7 |     <title>Document</title>
 8 | </head>
 9 | <body>
10 |     <iframe src="/"></iframe>
11 |     <p>
12 |         Ipsum tempor non amet amet tempor fugiat consequat voluptate Lorem veniam culpa. Nostrud et voluptate ea excepteur veniam exercitation. Reprehenderit nulla irure tempor laborum ut velit.
13 |     </p>
14 | </body>
15 | </html>


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # Node.js .gitignore
 2 | 
 3 | **/dist
 4 | **/node_modules
 5 | output.html
 6 | 
 7 | # Logs
 8 | logs
 9 | *.log
10 | npm-debug.log*
11 | pnpm-debug.log*
12 | yarn-debug.log*
13 | yarn-error.log*
14 | lerna-debug.log*
15 | 
16 | # OS
17 | .DS_Store
18 | 
19 | # Tests
20 | /coverage
21 | /.nyc_output
22 | 
23 | # IDEs and editors
24 | /.idea
25 | .project
26 | .classpath
27 | .c9/
28 | *.launch
29 | .settings/
30 | *.sublime-workspace
31 | 
32 | # IDE - VSCode
33 | .vscode/*
34 | !.vscode/settings.json
35 | !.vscode/tasks.json
36 | !.vscode/launch.json
37 | !.vscode/extensions.json


--------------------------------------------------------------------------------
/test/docker/docker-compose.yml:
--------------------------------------------------------------------------------
 1 | services:
 2 |     web-server:
 3 |         container_name: web-server
 4 |         image: web-server
 5 |         restart: always
 6 |         build:
 7 |             context: ../
 8 |             dockerfile: ./docker/nginx.Dockerfile
 9 |         ports:
10 |             - 8080:8080
11 |         networks:
12 |             - default
13 |         depends_on:
14 |             - ssr-proxy
15 | 
16 |     ssr-proxy:
17 |         container_name: ssr-proxy
18 |         image: ssr-proxy
19 |         restart: always
20 |         build:
21 |             context: ../../
22 |             dockerfile: ./test/docker/ssr.Dockerfile
23 |         ports:
24 |             - 8081:8081
25 |         networks:
26 |             - default
27 | 
28 | networks:
29 |   default:
30 |     driver: bridge


--------------------------------------------------------------------------------
/test/ssr-proxy-js.config.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "httpPort": 8081,
 3 |   "hostname": "0.0.0.0",
 4 |   "targetRoute": "https://react.dev",
 5 |   "proxyOrder": ["SsrProxy","HttpProxy"],
 6 |   "isBot": true,
 7 |   "ssr": {
 8 |     "browserConfig": {
 9 |       "headless": true,
10 |       "args": ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"]
11 |     },
12 |     "allowedResources": ["document", "script", "xhr", "fetch"],
13 |     "waitUntil": "networkidle0"
14 |   },
15 |   "httpProxy": {
16 |     "shouldUse": true,
17 |     "unsafeHttps": true
18 |   },
19 |   "static": {
20 |     "shouldUse": false
21 |   },
22 |   "log": {
23 |     "level": 2,
24 |     "console": {
25 |       "enabled": true
26 |     },
27 |     "file": {
28 |       "enabled": false
29 |     }
30 |   },
31 |   "cache": {
32 |     "shouldUse": false
33 |   }
34 | }


--------------------------------------------------------------------------------
/test/ssr-build-js.config.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "httpPort": 8080,
 3 |   "hostname": "localhost",
 4 |   "src": "public",
 5 |   "dist": "dist",
 6 |   "ssr": {
 7 |     "browserConfig": { "headless": true, "args": ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"], "timeout": 60000 },
 8 |     "sharedBrowser": false,
 9 |     "queryParams": [{ "key": "headless", "value": "true" }],
10 |     "allowedResources": ["document", "script", "xhr", "fetch"],
11 |     "waitUntil": "networkidle0",
12 |     "timeout": 60000
13 |   },
14 |   "log": {
15 |     "level": 2,
16 |     "console": {
17 |       "enabled": true
18 |     },
19 |     "file": {
20 |       "enabled": false
21 |     }
22 |   },
23 |   "job": {
24 |     "retries": 3,
25 |     "parallelism": 5,
26 |     "routes": [
27 |       { "method": "GET", "url": "/" },
28 |       { "method": "GET", "url": "/nested" },
29 |       { "method": "GET", "url": "/page.html" },
30 |       { "method": "GET", "url": "/iframe.html" }
31 |     ]
32 |   }
33 | }


--------------------------------------------------------------------------------
/webpack.config.js:
--------------------------------------------------------------------------------
 1 | const path = require('path');
 2 | const pkg = require('./package.json');
 3 | const nodeExternals = require('webpack-node-externals');
 4 | 
 5 | module.exports = {
 6 |     // mode: 'production',
 7 |     target: 'node', // in order to ignore built-in modules like path, fs, etc.
 8 |     entry: path.resolve(__dirname, 'src/index.ts'),
 9 |     module: {
10 |         rules: [{
11 |             test: /\.ts$/,
12 |             include: /src/,
13 |             use: [{
14 |                 loader: 'ts-loader' ,
15 |                 options: { configFile: 'tsconfig.json' },
16 |             }],
17 |         }],
18 |     },
19 |     resolve: {
20 |         extensions: ['.ts', '.js'],
21 |     },
22 |     output: {
23 |         library: pkg.name,
24 |         libraryTarget: 'umd',
25 |         filename: 'index.js',
26 |         path: path.resolve(__dirname, 'dist'),
27 |     },
28 |     externals: [
29 |         nodeExternals(), // in order to ignore all modules in node_modules folder
30 |     ],
31 | };


--------------------------------------------------------------------------------
/test/test-stream.js:
--------------------------------------------------------------------------------
 1 | const fs = require('fs');
 2 | const Transform = require('stream').Transform;
 3 | 
 4 | const parser = new Transform();
 5 | parser._transform = (chunk, encoding, callback) => {
 6 |     const str = chunk.toString();
 7 |     const error = null; // new Error('test');
 8 | 
 9 |     console.log('\n--- CHUNK ---\n', str, '\n', error);
10 | 
11 |     callback(error, str);
12 | };
13 | 
14 | console.log('\n--- BEGIN STREAM ---');
15 | 
16 | // Create and Transform Stream
17 | let stream = fs.createReadStream('../../../build/index.html'); // Create
18 | stream = stream.pipe(parser); // Transform
19 | stream = stream.on('end', () => console.log('\n--- END STREAM ---')); // Runs after all data is read
20 | 
21 | // Read Stream
22 | streamToString(stream).then(str => console.log('\n--- FULL DATA ---\n', str)).catch(e => e);
23 | 
24 | 
25 | 
26 | function streamToString(stream) {
27 |     const chunks = [];
28 |     return new Promise((res, rej) => {
29 |         if (!stream?.on) return res(stream);
30 |         stream.on('data', chunk => chunks.push(Buffer.from(chunk)));
31 |         stream.on('error', err => rej(err));
32 |         stream.on('end', () => res(Buffer.concat(chunks).toString('utf8')));
33 |     });
34 | }


--------------------------------------------------------------------------------
/src/plugins.ts:
--------------------------------------------------------------------------------
 1 | // https://rollupjs.org/plugin-development/#writebundle
 2 | // https://vite.dev/guide/api-plugin
 3 | 
 4 | import { SsrBuild } from './ssr-build';
 5 | import { SsrBuildConfig } from './types';
 6 | 
 7 | type Apply = 'serve' | 'build';
 8 | type Enforce = 'pre' | 'post' | undefined;
 9 | type Order = 'pre' | 'post' | undefined;
10 | type Event = 'writeBundle' | 'buildEnd' | 'closeBundle' | (string & {});
11 | 
12 | export const ssrBuildVitePlugin = (config: SsrBuildConfig, pluginOverride?: { apply?: Apply, enforce?: Enforce, [key: string]: any; }) => {
13 |   return {
14 |     name: 'ssr-build-js',
15 |     apply: 'build' as Apply,
16 |     // enforce: 'pre' as Enforce,
17 |     writeBundle: {
18 |       sequential: true,
19 |       // order: 'pre' as Order,
20 |       async handler(outputOptions: any, bundle: any) {
21 |         const ssrBuild = new SsrBuild(config);
22 |         const result = await ssrBuild.start();
23 |         result.forEach(e => {
24 |           const fileName = e.urlPath.replace(/^\/+/, '');
25 |           const duplicate = Object.keys(bundle).find(e => e === fileName);
26 |           if (duplicate) delete bundle[duplicate];
27 |           (this as any).emitFile({ type: 'asset', fileName, source: e.text });
28 |         });
29 |       },
30 |     },
31 |     ...(pluginOverride || {}),
32 |   };
33 | };


--------------------------------------------------------------------------------
/test/proxy.ts:
--------------------------------------------------------------------------------
 1 | // Run "npm run serve" in parallel
 2 | 
 3 | import { LogLevel, SsrProxy, SsrProxyConfig } from 'ssr-proxy-js-local'; // ssr-proxy-js or ssr-proxy-js-local
 4 | 
 5 | const BASE_PROXY_PORT = '8080';
 6 | const BASE_PROXY_ROUTE = `http://localhost:${BASE_PROXY_PORT}`;
 7 | 
 8 | // Proxy
 9 | 
10 | const config: SsrProxyConfig = {
11 |     httpPort: 8081,
12 |     hostname: '0.0.0.0',
13 |     targetRoute: BASE_PROXY_ROUTE,
14 |     isBot: true,
15 |     reqMiddleware: async (params) => {
16 | 		params.targetUrl.search = '';
17 | 		params.targetUrl.pathname = params.targetUrl.pathname.replace(/\/+$/, '') || '/';
18 |         return params;
19 |     },
20 |     resMiddleware: async (params, result) => {
21 |         if (result.text == null) return result;
22 |         result.text = result.text.replace('</html>', '\n\t<div>MIDDLEWARE</div>\n</html>');
23 |         result.text = result.text.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '');
24 |         return result;
25 |     },
26 |     log: { level: LogLevel.Debug },
27 | };
28 | 
29 | const ssrProxy = new SsrProxy(config);
30 | 
31 | ssrProxy.start();
32 | 
33 | // Server
34 | 
35 | import * as express from 'express';
36 | const app = express();
37 | 
38 | // Serve Static Files
39 | app.use(express.static('public'));
40 | 
41 | app.listen(BASE_PROXY_PORT, () => {
42 |     console.log(`Express listening at ${BASE_PROXY_ROUTE}`);
43 | });


--------------------------------------------------------------------------------
/test/docker/ssr-proxy-js.config.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "httpPort": 8081,
 3 |     "hostname": "0.0.0.0",
 4 |     "targetRoute": "http://web-server:8080",
 5 |     "proxyOrder": ["SsrProxy","HttpProxy"],
 6 |     "isBot": true,
 7 |     "ssr": {
 8 |       "browserConfig": {
 9 |         "headless": true,
10 |         "args": ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"]
11 |       },
12 |       "queryParams": [{ "key": "headless", "value": "true" }],
13 |       "allowedResources": ["document", "script", "xhr", "fetch"],
14 |       "waitUntil": "networkidle0"
15 |     },
16 |     "httpProxy": {
17 |       "shouldUse": true
18 |     },
19 |     "static": {
20 |       "shouldUse": false
21 |     },
22 |     "log": {
23 |       "level": 3,
24 |       "console": {
25 |         "enabled": true
26 |       },
27 |       "file": {
28 |         "enabled": true,
29 |         "dirPath": "/tmp/ssr-proxy/logs"
30 |       }
31 |     },
32 |     "cache": {
33 |       "shouldUse": true,
34 |       "maxEntries": 50,
35 |       "maxByteSize": 52428800,
36 |       "expirationMs": 14400000,
37 |       "autoRefresh": {
38 |         "enabled": true,
39 |         "shouldUse": true,
40 |         "proxyOrder": ["SsrProxy","HttpProxy"],
41 |         "initTimeoutMs": 5000,
42 |         "intervalCron": "0 0 3 * * *",
43 |         "intervalTz": "Etc/UTC",
44 |         "parallelism": 2,
45 |         "isBot": true,
46 |         "routes": [
47 |           { "method": "GET", "url": "/" },
48 |           { "method": "GET", "url": "/nested" },
49 |           { "method": "GET", "url": "/nested.dot/index.html" }
50 |         ]
51 |       }
52 |     }
53 |   }


--------------------------------------------------------------------------------
/test/docker/ssr.Dockerfile:
--------------------------------------------------------------------------------
 1 | # curl -fsSL -A 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' http://localhost/ > output-bot.html
 2 | # curl -fsSL -A 'bingbot/2.0' https://investtester.com/ > output-bot.html
 3 | # Chrome DevTools -> Network conditions -> User agent -> bingbot/2.0 / Console -> Ctrl + Shift + P -> Disable JavaScript
 4 | 
 5 | # BUILD
 6 | 
 7 | FROM node:20.12.2-bookworm AS build
 8 | 
 9 | WORKDIR /app
10 | 
11 | COPY package*.json ./
12 | RUN npm install
13 | 
14 | COPY . .
15 | RUN npm run build
16 | 
17 | # RUN
18 | 
19 | FROM node:20.12.2-bookworm AS run
20 | 
21 | WORKDIR /app
22 | 
23 | RUN apt-get update
24 | # RUN apt search ^chromium$ && exit 1
25 | RUN apt-get install -y chromium
26 | RUN apt-get install -y gettext-base moreutils
27 | RUN rm -rf /var/lib/apt/lists/*
28 | ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
29 | ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
30 | 
31 | # # https://stackoverflow.com/a/71128432
32 | # RUN apt-get update
33 | # RUN apt-get install -y ca-certificates fonts-liberation \
34 | #     libappindicator3-1 libasound2 libatk-bridge2.0-0 libatk1.0-0 \
35 | #     libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \
36 | #     libgbm1 libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 \
37 | #     libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 \
38 | #     libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 \
39 | #     libxi6 libxrandr2 libxrender1 libxss1 libxtst6 lsb-release wget xdg-utils
40 | # RUN rm -rf /var/lib/apt/lists/*
41 | 
42 | COPY --from=build /app/dist/ ./dist/
43 | COPY --from=build /app/node_modules/ ./node_modules/
44 | COPY --from=build /app/bin/cli.js ./bin/
45 | COPY ./test/docker/ssr-proxy-js.config.json .
46 | 
47 | EXPOSE 8081
48 | 
49 | CMD node ./bin/cli.js -c ./ssr-proxy-js.config.json
50 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "ssr-proxy-js",
 3 |   "version": "2.2.0",
 4 |   "description": "Server-Side Rendering Proxy",
 5 |   "keywords": [
 6 |     "ssr",
 7 |     "proxy",
 8 |     "spa"
 9 |   ],
10 |   "repository": {
11 |     "type": "git",
12 |     "url": "https://github.com/Tpessia/ssr-proxy-js"
13 |   },
14 |   "author": "Thiago Pessia",
15 |   "license": "MIT",
16 |   "main": "./dist/index.js",
17 |   "types": "./dist/index.d.ts",
18 |   "bin": "./bin/cli.js",
19 |   "files": [
20 |     "bin/**/*",
21 |     "dist/**/*",
22 |     "src/**/*"
23 |   ],
24 |   "scripts": {
25 |     "init": "npm i -g np && npm i",
26 |     "build:dev": "rimraf dist && cross-env NODE_ENV=development webpack --config webpack.config.js --mode development --watch",
27 |     "build": "rimraf dist && cross-env NODE_ENV=production webpack --config webpack.config.js --mode production",
28 |     "publish:pack": "npm pack",
29 |     "publish:dry": "npm publish --dry-run",
30 |     "publish:np": "npm run build && np --no-yarn --no-tests --branch=main --no-2fa"
31 |   },
32 |   "dependencies": {
33 |     "axios": "^1.7.2",
34 |     "clone-deep": "^4.0.1",
35 |     "deepmerge": "^4.3.1",
36 |     "express": "^4.19.2",
37 |     "isbot": "^5.1.13",
38 |     "mime-types": "^2.1.35",
39 |     "minimist": "^1.2.8",
40 |     "node-schedule": "^2.1.1",
41 |     "puppeteer": "^22.13.1",
42 |     "winston": "^3.13.1",
43 |     "winston-daily-rotate-file": "^4.7.1"
44 |   },
45 |   "devDependencies": {
46 |     "@types/clone-deep": "^4.0.4",
47 |     "@types/express": "^4.17.21",
48 |     "@types/mime-types": "^2.1.4",
49 |     "@types/node-schedule": "^2.1.7",
50 |     "cross-env": "^7.0.3",
51 |     "rimraf": "^6.0.1",
52 |     "ts-loader": "^9.5.1",
53 |     "typescript": "^4.9.5",
54 |     "webpack": "^5.93.0",
55 |     "webpack-cli": "^4.10.0",
56 |     "webpack-node-externals": "^3.0.0"
57 |   }
58 | }
59 | 


--------------------------------------------------------------------------------
/test/public/nested/index.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="en">
 3 | <head>
 4 |     <meta charset="UTF-8">
 5 |     <meta http-equiv="X-UA-Compatible" content="IE=edge">
 6 |     <meta name="viewport" content="width=device-width, initial-scale=1.0">
 7 |     <title>Document</title>
 8 | 
 9 |     <link rel="stylesheet" href="./index.css">
10 |     <link rel="stylesheet" href="../index.css">
11 | </head>
12 | <body>
13 |     <p>
14 |         Ipsum tempor non amet amet tempor fugiat consequat voluptate Lorem veniam culpa. Nostrud et voluptate ea excepteur veniam exercitation. Reprehenderit nulla irure tempor laborum ut velit.
15 |     </p>
16 |     <p>
17 |         Qui nulla proident aliquip dolore adipisicing sunt Lorem laborum. Culpa eiusmod minim nostrud in est labore reprehenderit mollit eiusmod do. Velit incididunt pariatur sunt qui aliqua anim. Qui laboris incididunt pariatur aliqua occaecat pariatur excepteur consectetur irure et nisi nostrud elit.
18 |     </p>
19 |     <p>
20 |         Mollit tempor amet amet deserunt ullamco tempor eu ipsum dolor commodo. Pariatur ut qui magna aliquip qui occaecat mollit voluptate reprehenderit excepteur id mollit. In cillum esse voluptate est cillum. Enim esse qui sint magna non Lorem proident proident ex. Duis deserunt esse nisi voluptate. Culpa consequat incididunt Lorem eiusmod dolor ullamco sit laboris voluptate elit occaecat et.
21 |     </p>
22 |     <p>
23 |         Commodo aliqua non occaecat nisi voluptate ea ipsum. Fugiat eiusmod fugiat adipisicing commodo id dolore aliqua laboris id aute. Mollit dolore minim magna ullamco officia eu elit dolore aliquip labore. In deserunt proident fugiat Lorem incididunt eu excepteur. Eiusmod anim anim enim esse.
24 |     </p>
25 |     <p>
26 |         Reprehenderit anim amet nostrud esse non nisi dolor veniam velit incididunt. Nulla et id laborum officia ut dolor sint ea elit cupidatat. Nulla officia dolore amet laborum anim.
27 |     </p>
28 | 
29 |     <script src="../index.js"></script>
30 | </body>
31 | </html>


--------------------------------------------------------------------------------
/test/public/nested.dot/index.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="en">
 3 | <head>
 4 |     <meta charset="UTF-8">
 5 |     <meta http-equiv="X-UA-Compatible" content="IE=edge">
 6 |     <meta name="viewport" content="width=device-width, initial-scale=1.0">
 7 |     <title>Document</title>
 8 | 
 9 |     <link rel="stylesheet" href="./index.dot.css">
10 |     <link rel="stylesheet" href="../index.css">
11 | </head>
12 | <body>
13 |     <p>
14 |         Ipsum tempor non amet amet tempor fugiat consequat voluptate Lorem veniam culpa. Nostrud et voluptate ea excepteur veniam exercitation. Reprehenderit nulla irure tempor laborum ut velit.
15 |     </p>
16 |     <p>
17 |         Qui nulla proident aliquip dolore adipisicing sunt Lorem laborum. Culpa eiusmod minim nostrud in est labore reprehenderit mollit eiusmod do. Velit incididunt pariatur sunt qui aliqua anim. Qui laboris incididunt pariatur aliqua occaecat pariatur excepteur consectetur irure et nisi nostrud elit.
18 |     </p>
19 |     <p>
20 |         Mollit tempor amet amet deserunt ullamco tempor eu ipsum dolor commodo. Pariatur ut qui magna aliquip qui occaecat mollit voluptate reprehenderit excepteur id mollit. In cillum esse voluptate est cillum. Enim esse qui sint magna non Lorem proident proident ex. Duis deserunt esse nisi voluptate. Culpa consequat incididunt Lorem eiusmod dolor ullamco sit laboris voluptate elit occaecat et.
21 |     </p>
22 |     <p>
23 |         Commodo aliqua non occaecat nisi voluptate ea ipsum. Fugiat eiusmod fugiat adipisicing commodo id dolore aliqua laboris id aute. Mollit dolore minim magna ullamco officia eu elit dolore aliquip labore. In deserunt proident fugiat Lorem incididunt eu excepteur. Eiusmod anim anim enim esse.
24 |     </p>
25 |     <p>
26 |         Reprehenderit anim amet nostrud esse non nisi dolor veniam velit incididunt. Nulla et id laborum officia ut dolor sint ea elit cupidatat. Nulla officia dolore amet laborum anim.
27 |     </p>
28 | 
29 |     <script src="../index.js"></script>
30 | </body>
31 | </html>


--------------------------------------------------------------------------------
/test/public/index.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="en">
 3 | <head>
 4 |     <meta charset="UTF-8">
 5 |     <meta http-equiv="X-UA-Compatible" content="IE=edge">
 6 |     <meta name="viewport" content="width=device-width, initial-scale=1.0">
 7 |     <title>Document</title>
 8 | 
 9 |     <link rel="stylesheet" href="./index.css">
10 | 
11 |     <style>
12 |         p {
13 |             background-color: black;
14 |         }
15 |     </style>
16 | </head>
17 | <body>
18 |     <p>
19 |         Ipsum tempor non amet amet tempor fugiat consequat voluptate Lorem veniam culpa. Nostrud et voluptate ea excepteur veniam exercitation. Reprehenderit nulla irure tempor laborum ut velit.
20 |     </p>
21 |     <p>
22 |         Qui nulla proident aliquip dolore adipisicing sunt Lorem laborum. Culpa eiusmod minim nostrud in est labore reprehenderit mollit eiusmod do. Velit incididunt pariatur sunt qui aliqua anim. Qui laboris incididunt pariatur aliqua occaecat pariatur excepteur consectetur irure et nisi nostrud elit.
23 |     </p>
24 |     <p>
25 |         Mollit tempor amet amet deserunt ullamco tempor eu ipsum dolor commodo. Pariatur ut qui magna aliquip qui occaecat mollit voluptate reprehenderit excepteur id mollit. In cillum esse voluptate est cillum. Enim esse qui sint magna non Lorem proident proident ex. Duis deserunt esse nisi voluptate. Culpa consequat incididunt Lorem eiusmod dolor ullamco sit laboris voluptate elit occaecat et.
26 |     </p>
27 |     <p>
28 |         Commodo aliqua non occaecat nisi voluptate ea ipsum. Fugiat eiusmod fugiat adipisicing commodo id dolore aliqua laboris id aute. Mollit dolore minim magna ullamco officia eu elit dolore aliquip labore. In deserunt proident fugiat Lorem incididunt eu excepteur. Eiusmod anim anim enim esse.
29 |     </p>
30 |     <p>
31 |         Reprehenderit anim amet nostrud esse non nisi dolor veniam velit incididunt. Nulla et id laborum officia ut dolor sint ea elit cupidatat. Nulla officia dolore amet laborum anim.
32 |     </p>
33 | 
34 |     <script src="./index.js"></script>
35 | </body>
36 | </html>


--------------------------------------------------------------------------------
/test/public/page.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="en">
 3 | <head>
 4 |     <meta charset="UTF-8">
 5 |     <meta http-equiv="X-UA-Compatible" content="IE=edge">
 6 |     <meta name="viewport" content="width=device-width, initial-scale=1.0">
 7 |     <title>Document</title>
 8 | 
 9 |     <link rel="stylesheet" href="./index.css">
10 | 
11 |     <style>
12 |         p {
13 |             background-color: black;
14 |         }
15 |     </style>
16 | </head>
17 | <body>
18 |     <p>
19 |         Ipsum tempor non amet amet tempor fugiat consequat voluptate Lorem veniam culpa. Nostrud et voluptate ea excepteur veniam exercitation. Reprehenderit nulla irure tempor laborum ut velit.
20 |     </p>
21 |     <p>
22 |         Qui nulla proident aliquip dolore adipisicing sunt Lorem laborum. Culpa eiusmod minim nostrud in est labore reprehenderit mollit eiusmod do. Velit incididunt pariatur sunt qui aliqua anim. Qui laboris incididunt pariatur aliqua occaecat pariatur excepteur consectetur irure et nisi nostrud elit.
23 |     </p>
24 |     <p>
25 |         Mollit tempor amet amet deserunt ullamco tempor eu ipsum dolor commodo. Pariatur ut qui magna aliquip qui occaecat mollit voluptate reprehenderit excepteur id mollit. In cillum esse voluptate est cillum. Enim esse qui sint magna non Lorem proident proident ex. Duis deserunt esse nisi voluptate. Culpa consequat incididunt Lorem eiusmod dolor ullamco sit laboris voluptate elit occaecat et.
26 |     </p>
27 |     <p>
28 |         Commodo aliqua non occaecat nisi voluptate ea ipsum. Fugiat eiusmod fugiat adipisicing commodo id dolore aliqua laboris id aute. Mollit dolore minim magna ullamco officia eu elit dolore aliquip labore. In deserunt proident fugiat Lorem incididunt eu excepteur. Eiusmod anim anim enim esse.
29 |     </p>
30 |     <p>
31 |         Reprehenderit anim amet nostrud esse non nisi dolor veniam velit incididunt. Nulla et id laborum officia ut dolor sint ea elit cupidatat. Nulla officia dolore amet laborum anim.
32 |     </p>
33 | 
34 |     <script src="./index.js"></script>
35 | </body>
36 | </html>


--------------------------------------------------------------------------------
/test/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "ssr-proxy-js-test",
 3 |   "version": "1.0.0",
 4 |   "scripts": {
 5 |     "init": "npm install",
 6 |     "build": "cd .. && npm run build",
 7 |     "start": "npm run build && nodemon --watch ./ --watch ../ proxy.js",
 8 |     "start:ts": "npm run build && ts-node-dev --project tsconfig.json proxy.ts",
 9 |     "start:build": "rimraf dist && cpx 'public/**/*' dist/ && npm run build && ts-node-dev --project tsconfig.json build.ts",
10 |     "start:docker": "sudo docker compose -f ./docker/docker-compose.yml up --build",
11 |     "start:cli": "npm run build && node ./node_modules/ssr-proxy-js-local/bin/cli.js -c ./ssr-proxy-js.config.json",
12 |     "start:cli-build": "npm run build && node ./node_modules/ssr-proxy-js-local/bin/cli.js --mode build -c ./ssr-build-js.config.json",
13 |     "start:cli-build-args": "npm run build && node ./node_modules/ssr-proxy-js-local/bin/cli.js --mode=build --httpPort=8080 --src=./public --dist=./dist --job.routes='[{\"url\":\"/\"},{\"url\":\"/nested\"}]'",
14 |     "start:npx": "npm_config_yes=true npx --yes ssr-proxy-js -c ./ssr-proxy-js.config.json",
15 |     "start:npx-build": "npm_config_yes=true npx --yes ssr-proxy-js --mode build -c ./ssr-build-js.config.json",
16 |     "test:request": "node test-request.js",
17 |     "test:stream": "node test-stream.js",
18 |     "test:curl-bot": "curl -fsSL -A 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' http://localhost:8080/ > output.html && xdg-open output.html",
19 |     "serve": "npm_config_yes=true npx --yes http-server ./public -p 8080 -a 0.0.0.0",
20 |     "serve:dist": "npm_config_yes=true npx --yes http-server ./dist -p 8080 -a 0.0.0.0",
21 |     "docker": "sudo docker build -f Dockerfile -t ssr-proxy-js . && sudo docker run -it --rm -p 8080:8080 ssr-proxy-js"
22 |   },
23 |   "dependencies": {
24 |     "axios": "^0.24.0",
25 |     "express": "^4.17.1",
26 |     "ssr-proxy-js": "^1.0.1",
27 |     "ssr-proxy-js-local": "file:../"
28 |   },
29 |   "devDependencies": {
30 |     "cpx": "^1.5.0",
31 |     "nodemon": "^2.0.15",
32 |     "rimraf": "^6.0.1",
33 |     "ts-node-dev": "^1.1.8",
34 |     "typescript": "^4.4.4"
35 |   }
36 | }
37 | 


--------------------------------------------------------------------------------
/test/build.ts:
--------------------------------------------------------------------------------
 1 | import * as os from 'os';
 2 | import * as path from 'path';
 3 | import { LogLevel, SsrBuild, SsrBuildConfig } from 'ssr-proxy-js-local'; // ssr-proxy-js or ssr-proxy-js-local
 4 | 
 5 | const config: SsrBuildConfig = {
 6 |     httpPort: 8080,
 7 |     hostname: 'localhost',
 8 |     src: 'public',
 9 |     dist: 'dist',
10 |     // stopOnError: true,
11 |     serverMiddleware: async (req, res, next) => {
12 |         // res.sendFile(path.join(__dirname, 'public/index.html'));
13 |         // res.sendFile(path.join(__dirname, 'public', req.path));
14 |         next();
15 |     },
16 |     reqMiddleware: async (params) => {
17 |         params.headers['Referer'] = 'http://google.com';
18 |         return params;
19 |     },
20 |     resMiddleware: async (params, result) => {
21 |         if (result.text == null) return result;
22 |         result.text = result.text.replace('</html>', '\n\t<div>MIDDLEWARE</div>\n</html>');
23 |         result.text = result.text.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '');
24 |         return result;
25 |     },
26 |     ssr: {
27 |         browserConfig: { headless: false, slowMo: 1000, args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'], timeout: 60000 },
28 |         sharedBrowser: true,
29 |         queryParams: [{ key: 'headless', value: 'true' }],
30 |         allowedResources: ['document', 'script', 'xhr', 'fetch'],
31 |         waitUntil: 'networkidle0',
32 |         timeout: 60000,
33 |         sleep: 1000,
34 |     },
35 |     log: {
36 |         level: LogLevel.Info,
37 |         console: {
38 |             enabled: true,
39 |         },
40 |         file: {
41 |             enabled: true,
42 |             dirPath: path.join(os.tmpdir(), 'ssr-proxy-js/logs'),
43 |         },
44 |     },
45 |     job: {
46 |         retries: 3,
47 |         parallelism: 3,
48 |         routes: [
49 |             { method: 'GET', url: '/' },
50 |             { method: 'GET', url: '/nested' },
51 |             { method: 'GET', url: '/page.html' },
52 |             { method: 'GET', url: '/iframe.html' },
53 |             { method: 'GET', url: '/fail' },
54 |         ],
55 |     },
56 | };
57 | 
58 | const ssrBuild = new SsrBuild(config);
59 | 
60 | ssrBuild.start();


--------------------------------------------------------------------------------
/bin/cli.js:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env node
 2 | 
 3 | // cd test && npx ssr-proxy-js-local
 4 | 
 5 | // npx ssr-proxy-js
 6 | // npx ssr-proxy-js -c ./ssr-proxy-js.config.json
 7 | // npx ssr-proxy-js --httpPort=8080 --targetRoute=http://localhost:3000 --static.dirPath=./public --proxyOrder=SsrProxy --proxyOrder=StaticProxy --log.level=3
 8 | 
 9 | const fs = require('fs');
10 | const path = require('path');
11 | const minimist = require('minimist');
12 | const deepmerge = require('deepmerge');
13 | const { SsrProxy, SsrBuild } = require('../dist/index');
14 | 
15 | const argv = minimist(process.argv.slice(2));
16 | 
17 | const { _: argv_, mode: argv_mode, c: argv_c, config: argv_config, ...argv_rest } = argv;
18 | const explicitConfig = !!(argv_c || argv_config);
19 | 
20 | if (!!argv_mode && argv_mode !== 'proxy' && argv_mode !== 'build') {
21 |     logWarn('Invalid mode, must be either "proxy" or "build"');
22 |     process.exit(1);
23 | }
24 | 
25 | const mode = argv_mode || 'proxy';
26 | 
27 | const options = { };
28 | options.configPath = argv_c || argv_config || (mode === 'proxy' ? './ssr-proxy-js.config.json' : './ssr-build-js.config.json');
29 | options.configPath = path.resolve(process.cwd(), options.configPath);
30 | 
31 | try {
32 |     if (options.configPath)
33 |         options.configJson = fs.readFileSync(options.configPath, { encoding: 'utf8' });
34 | } catch (err) {
35 |     if (explicitConfig)
36 |         logWarn(`Unable to find the config, looking for: ${options.configPath}`, err);
37 | }
38 | 
39 | try {
40 |     if (options.configJson) options.config = JSON.parse(options.configJson);
41 |     else options.config = {};
42 | } catch (err) {
43 |     logWarn('Unable to parse the config', err);
44 | }
45 | 
46 | if (typeof argv_rest?.cache?.autoRefresh?.routes === 'string') argv_rest.cache.autoRefresh.routes = JSON.parse(argv_rest.cache.autoRefresh.routes);
47 | if (typeof argv_rest?.job?.routes === 'string') argv_rest.job.routes = JSON.parse(argv_rest.job.routes);
48 | 
49 | options.config = deepmerge(options.config, argv_rest, {
50 |     arrayMerge: (destArray, srcArray, opts) => srcArray,
51 | });
52 | 
53 | if (isEmpty(options.config)) {
54 |     logWarn('No config file or cli arguments found!');
55 | }
56 | 
57 | if (mode === 'proxy') {
58 |     const ssrProxy = new SsrProxy(options.config);
59 |     ssrProxy.start();
60 | } else if (mode === 'build') {
61 |     const ssrBuild = new SsrBuild(options.config);
62 |     ssrBuild.start();
63 | }
64 | 
65 | // Utils
66 | 
67 | function logWarn(...msg) {
68 |     if (!msg || !msg.length) return;
69 |     msg[0] = '\x1b[33m' + msg[0];
70 |     msg[msg.length - 1] += '\x1b[0m';
71 |     console.log(...msg);
72 | }
73 | 
74 | function isEmpty(obj) {
75 |     return obj && Object.keys(obj).length === 0 && Object.getPrototypeOf(obj) === Object.prototype;
76 | }


--------------------------------------------------------------------------------
/test/docker/nginx.conf:
--------------------------------------------------------------------------------
  1 | # https://stackoverflow.com/a/59846239
  2 | 
  3 | user nginx;
  4 | worker_processes auto;
  5 | 
  6 | error_log /var/log/nginx/error.log notice;
  7 | pid /var/run/nginx.pid;
  8 | 
  9 | events {
 10 |   worker_connections 1024;
 11 | }
 12 | 
 13 | http {
 14 |   include /etc/nginx/mime.types;
 15 |   default_type application/octet-stream;
 16 | 
 17 |   # Security headers
 18 | 
 19 |   add_header X-Frame-Options "SAMEORIGIN" always;
 20 |   add_header X-Content-Type-Options "nosniff" always;
 21 |   add_header Referrer-Policy "no-referrer-when-downgrade" always;
 22 | 
 23 |   # Logging
 24 | 
 25 |   log_format main '$remote_addr - $remote_user [$time_local] [$request_time] "$request" '
 26 |                   '$status $body_bytes_sent "$http_referer" '
 27 |                   '"$http_user_agent" "$http_x_forwarded_for"';
 28 |   access_log /var/log/nginx/access.log main;
 29 | 
 30 |   # Basic settings
 31 | 
 32 |   sendfile on;
 33 |   tcp_nopush on;
 34 |   tcp_nodelay on;
 35 |   keepalive_timeout 65;
 36 |   types_hash_max_size 2048;
 37 |   server_tokens off;
 38 |   client_max_body_size 1M;
 39 | 
 40 |   # Check Bots
 41 | 
 42 |   map $http_user_agent $is_bot_agent {
 43 |     default 0;
 44 |     "~*(googlebot|Google-InspectionTool|bingbot|yandex|baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterest|slackbot|vkShare|W3C_Validator|mj12bot|ahrefsbot|semrushbot|dotbot|applebot|duckduckbot|sogou|exabot|petalbot|ia_archiver|alexabot|msnbot|archive.org_bot|screaming frog|proximic|yahoo! slurp)" 1;
 45 |   }
 46 | 
 47 |   map $request_uri $is_bot_query {
 48 |     default 0;
 49 |     "~*[\?&]isbot(?:=|&|$)" 1;
 50 |   }
 51 | 
 52 |   map "$is_bot_agent$is_bot_query" $should_proxy {
 53 |     default 0;
 54 |     ~*1 1;
 55 |   }
 56 | 
 57 |   # Server
 58 | 
 59 |   server {
 60 |     listen 8080;
 61 |     server_name _;
 62 |     http2  on;
 63 | 
 64 |     root /app/public;
 65 |     index index.html;
 66 | 
 67 |     # Static Files
 68 | 
 69 |     error_page 404 /_not-found.html;
 70 |     location = /_not-found.html {
 71 |       internal;
 72 |       return 200 "404 Not Found";
 73 |     }
 74 | 
 75 |     location /404 {
 76 |       if ($should_proxy) {
 77 |         proxy_pass http://ssr-proxy:8081;
 78 |         break;
 79 |       }
 80 | 
 81 |       return 404;
 82 |     }
 83 | 
 84 |     location /301 {
 85 |       if ($should_proxy) {
 86 |         proxy_pass http://ssr-proxy:8081;
 87 |         break;
 88 |       }
 89 | 
 90 |       return 301 $scheme://$http_host/;
 91 |     }
 92 | 
 93 |     location / {
 94 |       if ($should_proxy) {
 95 |         proxy_pass http://ssr-proxy:8081;
 96 |         break;
 97 |       }
 98 | 
 99 |       try_files $uri /index.html;
100 |     }
101 |   }
102 | }


--------------------------------------------------------------------------------
/src/proxy-cache.ts:
--------------------------------------------------------------------------------
 1 | import { Stream } from 'stream';
 2 | import { CacheDeletion, CacheItem, InternalCacheItem } from './types';
 3 | import { streamToString } from './utils';
 4 | 
 5 | export class ProxyCache {
 6 |     private cache: Map<string, InternalCacheItem> = new Map<string, InternalCacheItem>();
 7 | 
 8 |     maxEntries: number;
 9 |     maxSize: number;
10 |     expirationMs: number;
11 | 
12 |     constructor(maxEntries: number = 10, maxSize: number = 10 * 1024 * 1024, expirationMs: number = 5 * 60 * 1000) {
13 |         this.maxEntries = maxEntries;
14 |         this.maxSize = maxSize;
15 |         this.expirationMs = expirationMs;
16 |     }
17 | 
18 |     has(urlStr: string) {
19 |         return this.cache.has(urlStr);
20 |     }
21 | 
22 |     keys() {
23 |         return this.cache.keys();
24 |     }
25 | 
26 |     get(urlStr: string): CacheItem | null {
27 |         const entry = this.cache.get(urlStr);
28 |         if (!entry) return null;
29 |         entry.hits++;
30 |         return { text: entry.text, contentType: entry.contentType };
31 |     }
32 | 
33 |     set(urlStr: string, text: string, status: number, contentType: string) {
34 |         return this.cache.set(urlStr, { text, status, contentType, hits: 0, date: new Date() });
35 |     }
36 | 
37 |     delete(urlStr: string) {
38 |         return this.cache.delete(urlStr);
39 |     }
40 | 
41 |     async pipe(urlStr: string, stream: Stream, status: number, contentType: string) {
42 |         return await streamToString(stream).then(str => this.cache.set(urlStr, { text: str, status, contentType, hits: 0, date: new Date() }));
43 |     }
44 | 
45 |     tryClear() {
46 |         const $this = this;
47 |         let cacheSize = 0;
48 |         const entries = [...this.cache.entries()].sort((a, b) => b[1].hits - a[1].hits);
49 | 
50 |         const deleted: CacheDeletion[] = [];
51 | 
52 |         for (const i in entries) {
53 |             const key = entries[i][0];
54 |             const entry = entries[i][1];
55 | 
56 |             if (this.cache.has(key)) {
57 |                 cacheSize += cacheSize > this.maxSize ? cacheSize : Buffer.from(entry.text).length;
58 | 
59 |                 const deleteBySize = cacheSize > this.maxSize;
60 |                 if (deleteBySize) deleteEntry(key, 'size');
61 | 
62 |                 // delete by length
63 |                 const deleteByLength = this.cache.size > this.maxEntries;
64 |                 if (deleteByLength) deleteEntry(entries[this.maxEntries - +i][0], 'length');
65 | 
66 |                 // delete by date
67 |                 const deleteByDate = new Date().getTime() - entry.date.getTime() > this.expirationMs;
68 |                 if (deleteByDate) deleteEntry(key, 'expired');
69 |             }
70 |         }
71 | 
72 |         return deleted;
73 | 
74 |         function deleteEntry(key: string, reason: string) {
75 |             if ($this.delete(key)) deleted.push({ key, reason });
76 |         }
77 |     }
78 | }


--------------------------------------------------------------------------------
/test/proxy.js:
--------------------------------------------------------------------------------
 1 | const os = require('os');
 2 | const path = require('path');
 3 | const { SsrProxy } = require('ssr-proxy-js-local'); // ssr-proxy-js or ssr-proxy-js-local
 4 | 
 5 | const BASE_PROXY_PORT = '8080';
 6 | const BASE_PROXY_ROUTE = `http://localhost:${BASE_PROXY_PORT}`;
 7 | const STATIC_FILES_PATH = path.join(process.cwd(), 'public');
 8 | const LOGGING_PATH = path.join(os.tmpdir(), 'ssr-proxy/logs');
 9 | 
10 | console.log(`\nLogging at: ${LOGGING_PATH}`);
11 | 
12 | // Proxy
13 | 
14 | const ssrProxy = new SsrProxy({
15 |     httpPort: 8081,
16 |     hostname: '0.0.0.0',
17 |     targetRoute: BASE_PROXY_ROUTE,
18 |     proxyOrder: ['SsrProxy', 'HttpProxy', 'StaticProxy'],
19 |     isBot: (method, url, headers) => true,
20 |     failStatus: params => 404,
21 |     customError: err => err.toString(),
22 |     skipOnError: false,
23 |     ssr: {
24 |         shouldUse: params => params.isBot && (/\.html$/.test(params.targetUrl.pathname) || !/\./.test(params.targetUrl.pathname)),
25 |         browserConfig: { headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'], timeout: 60000 },
26 |         queryParams: [{ key: 'headless', value: 'true' }],
27 |         allowedResources: ['document', 'script', 'xhr', 'fetch'],
28 |         waitUntil: 'networkidle0',
29 |         timeout: 60000,
30 |     },
31 |     httpProxy: {
32 |         shouldUse: params => true,
33 |         timeout: 60000,
34 |     },
35 |     static: {
36 |         shouldUse: params => true,
37 |         dirPath: STATIC_FILES_PATH,
38 |         useIndexFile: path => path.endsWith('/'),
39 |         indexFile: 'index.html',
40 |     },
41 |     log: {
42 |         level: 3,
43 |         console: {
44 |             enabled: true,
45 |         },
46 |         file: {
47 |             enabled: true,
48 |             dirPath: LOGGING_PATH,
49 |         },
50 |     },
51 |     cache: {
52 |         shouldUse: params => params.proxyType === 'SsrProxy',
53 |         maxEntries: 50,
54 |         maxByteSize: 50 * 1024 * 1024, // 50MB
55 |         expirationMs: 25 * 60 * 60 * 1000, // 25h
56 |         autoRefresh: {
57 |             enabled: true,
58 |             shouldUse: () => true,
59 |             proxyOrder: ['SsrProxy', 'HttpProxy'],
60 |             initTimeoutMs: 5 * 1000, // 5s
61 |             intervalCron: '0 0 3 * * *', // every day at 3am
62 |             intervalTz: 'Etc/UTC',
63 |             retries: 3,
64 |             parallelism: 5,
65 |             closeBrowser: true,
66 |             isBot: true,
67 |             routes: [
68 |                 { method: 'GET', url: '/' },
69 |                 { method: 'GET', url: '/login' },
70 |             ],
71 |         },
72 |     },
73 | });
74 | 
75 | ssrProxy.start();
76 | 
77 | // Server
78 | 
79 | const express = require('express');
80 | const app = express();
81 | 
82 | app.get('/', (req, res) => {
83 |   res.send('Hello World!');
84 | });
85 | 
86 | app.get('/301', (req, res) => {
87 |     res.redirect(301, '/');
88 | });
89 | 
90 | app.get('/302', (req, res) => {
91 |     res.redirect(302, '/');
92 | });
93 | 
94 | app.listen(BASE_PROXY_PORT, () => {
95 |   console.log(`Express listening at ${BASE_PROXY_ROUTE}`);
96 | });


--------------------------------------------------------------------------------
/src/logger.ts:
--------------------------------------------------------------------------------
  1 | import path from 'path';
  2 | import winston from 'winston';
  3 | import 'winston-daily-rotate-file';
  4 | import { LogLevel } from './types';
  5 | 
  6 | export class Logger {
  7 |     private static logLevel: LogLevel = LogLevel.None;
  8 |     private static enableConsole?: boolean;
  9 |     private static fileLogger?: winston.Logger;
 10 | 
 11 |     loggerId?: number;
 12 |     loggerIdStr: string = '';
 13 | 
 14 |     constructor(useId = true) {
 15 |         if (useId) {
 16 |             this.loggerId = Math.round(Math.random() * 99999);
 17 |             this.loggerIdStr = `[${this.loggerId}] `;
 18 |         }
 19 |     }
 20 | 
 21 |     error(errName: string, err: any, withStack: boolean = false) {
 22 |         Logger.error(errName, err, withStack, this.loggerIdStr);
 23 |     }
 24 | 
 25 |     warn(errName: string, err: any, withStack: boolean = false) {
 26 |         Logger.warn(errName, err, withStack, this.loggerIdStr);
 27 |     }
 28 | 
 29 |     info(msg: string) {
 30 |         Logger.info(msg, this.loggerIdStr);
 31 |     }
 32 | 
 33 |     debug(msg: string) {
 34 |         Logger.debug(msg, this.loggerIdStr);
 35 |     }
 36 | 
 37 |     static error(errName: string, err: any, withStack: boolean = false, prefix: string = '') {
 38 |         const logMsg = `${this.logPrefix()}${prefix}${errName}: ${this.errorStr(err)}${(withStack && ('\n' + err.stack)) || ''}`;
 39 |         if (this.logLevel >= 1) {
 40 |             if (this.enableConsole) console.log(`\x1b[31m${logMsg}\x1b[0m`);
 41 |             if (this.fileLogger) this.fileLogger.error(logMsg);
 42 |         }
 43 |     }
 44 | 
 45 |     static warn(errName: string, err: any, withStack: boolean = false, prefix: string = '') {
 46 |         const logMsg = `${this.logPrefix()}${prefix}${errName}: ${this.errorStr(err)}${(withStack && ('\n' + err.stack)) || ''}`;
 47 |         if (this.logLevel >= 2) {
 48 |             if (this.enableConsole) console.log(`\x1b[33m${logMsg}\x1b[0m`);
 49 |             if (this.fileLogger) this.fileLogger.warn(logMsg);
 50 |         }
 51 |     }
 52 | 
 53 |     static info(msg: string, prefix: string = '') {
 54 |         const logMsg = `${this.logPrefix()}${prefix}${msg}`;
 55 |         if (this.logLevel >= 2) {
 56 |             if (this.enableConsole) console.log(`\x1b[37m${logMsg}\x1b[0m`);
 57 |             if (this.fileLogger) this.fileLogger.info(logMsg);
 58 |         }
 59 |     }
 60 | 
 61 |     static debug(msg: string, prefix: string = '') {
 62 |         const logMsg = `${this.logPrefix()}${prefix}${msg}`;
 63 |         if (this.logLevel >= 3) {
 64 |             if (this.enableConsole) console.log(`\x1b[34m${logMsg}\x1b[0m`);
 65 |             if (this.fileLogger) this.fileLogger.debug(logMsg);
 66 |         }
 67 |     }
 68 | 
 69 |     static errorStr(err: any) {
 70 |         return err && (err.message || err.toString());
 71 |     }
 72 | 
 73 |     static setLevel(level: LogLevel) {
 74 |         this.logLevel = level;
 75 |     }
 76 | 
 77 |     static configConsole(enable: boolean) {
 78 |         this.enableConsole = enable;
 79 |     }
 80 | 
 81 |     static configFile(enable: boolean, dirPath: string) {
 82 |         if (!enable || !dirPath) {
 83 |             this.fileLogger = undefined;
 84 |         }
 85 | 
 86 |         const transport = new winston.transports.DailyRotateFile({
 87 |             filename: path.join(dirPath, 'log-%DATE%.log'),
 88 |             datePattern: 'YYYY-MM-DD',
 89 |             maxSize: '20m',
 90 |             maxFiles: '5d'
 91 |         });
 92 | 
 93 |         const logger = winston.createLogger({
 94 |             exitOnError: false,
 95 |             level: 'info',
 96 |             format: winston.format.combine(
 97 |                 winston.format.timestamp(),
 98 |                 winston.format.printf(info => `${info.timestamp} ${info.level}: ${info.message}`+(info.splat!==undefined?`${info.splat}`:" "))
 99 |             ),
100 |             transports: [transport]
101 |         });
102 | 
103 |         this.fileLogger = logger;
104 |     }
105 | 
106 |     private static logPrefix() {
107 |         return `[${new Date().toISOString()}] `;
108 |     }
109 | }


--------------------------------------------------------------------------------
/src/utils.ts:
--------------------------------------------------------------------------------
  1 | import { Stream } from 'stream';
  2 | 
  3 | export function getOrCall<T>(obj: T | ((...args: any[]) => T), ...args: any[]): T;
  4 | export function getOrCall<T>(obj?: T | ((...args: any[]) => T), ...args: any[]): T | undefined {
  5 |     return typeof obj === 'function' ? (obj as (...args: any[]) => T)?.(...args) : obj;
  6 | }
  7 | 
  8 | export function streamToString(stream: Stream): Promise<string> {
  9 |     const chunks: Buffer[] = [];
 10 |     return new Promise((res, rej) => {
 11 |         if (!stream?.on) return res(stream as any);
 12 |         stream.on('data', chunk => chunks.push(Buffer.from(chunk)));
 13 |         stream.on('error', err => rej(err));
 14 |         stream.on('end', () => res(Buffer.concat(chunks as any).toString('utf8')));
 15 |     });
 16 | }
 17 | 
 18 | export function promiseParallel<T, TRej = T>(tasks: (() => Promise<T>)[], concurrencyLimit: number, noReject: boolean = false): Promise<(T | TRej)[]> {
 19 |     return new Promise<(T | TRej)[]>((res, rej) => {
 20 |         if (tasks.length === 0) res([]);
 21 | 
 22 |         const results: (T | TRej)[] = [];
 23 |         const pool: Promise<T | TRej>[] = [];
 24 |         let canceled: boolean = false;
 25 | 
 26 |         tasks.slice(0, concurrencyLimit).map(async (e) => await runPromise(e));
 27 | 
 28 |         function runPromise(task: () => Promise<T>): Promise<T | TRej> {
 29 |             let promise: Promise<T | TRej> = task();
 30 | 
 31 |             pool.push(promise);
 32 | 
 33 |             if (noReject) promise = promise.catch((e: TRej) => e);
 34 | 
 35 |             promise = promise.then(async r => {
 36 |                 if (canceled) return r;
 37 | 
 38 |                 results.push(r);
 39 | 
 40 |                 const poolIndex = pool.indexOf(promise);
 41 |                 pool.splice(poolIndex, 1);
 42 | 
 43 |                 if (tasks.length === results.length)
 44 |                     res(results);
 45 | 
 46 |                 const nextIndex = concurrencyLimit + results.length - 1;
 47 |                 const nextTask = tasks[nextIndex];
 48 | 
 49 |                 if (!nextTask) return r;
 50 | 
 51 |                 return await runPromise(nextTask);
 52 |             });
 53 | 
 54 |             if (!noReject) promise = promise.catch(err => { canceled = true; rej(err); return err; });
 55 | 
 56 |             return promise;
 57 |         }
 58 |     });
 59 | }
 60 | 
 61 | export async function promiseRetry<T>(func: () => Promise<T>, maxRetries: number, onError?: (err: any) => void): Promise<T> {
 62 |     try {
 63 |         return await func();
 64 |     } catch (err) {
 65 |         onError?.(err);
 66 |         const funcAny = (func as any);
 67 |         funcAny._retries = (funcAny._retries as number ?? 0) + 1;
 68 |         if (funcAny._retries >= maxRetries) throw err;
 69 |         else return await promiseRetry(func, maxRetries, onError);
 70 |     }
 71 | }
 72 | 
 73 | export function promiseDeferred<T>(): { promise: Promise<T>, resolve: (value: T) => void, reject: (reason?: any) => void } {
 74 |     let resolve: (value: T) => void;
 75 |     let reject: (reason?: any) => void;
 76 |     const promise = new Promise<T>((res, rej) => { resolve = res; reject = rej; });
 77 |     return { promise, resolve: resolve!, reject: reject! };
 78 | }
 79 | 
 80 | export const createLock = () => {
 81 |     const queue: (() => Promise<void>)[] = [];
 82 | 
 83 |     let active = false;
 84 | 
 85 |     return (fn: () => Promise<any>) => {
 86 |         const { promise, resolve, reject } = promiseDeferred();
 87 | 
 88 |         // call function then next on queue
 89 |         const exec = async () => {
 90 |             await fn().then(resolve, reject);
 91 |             if (queue.length > 0) {
 92 |                 queue.shift()!(); // call next function
 93 |             } else {
 94 |                 active = false;
 95 |             }
 96 |         };
 97 | 
 98 |         // call current or add to queue
 99 |         if (active) {
100 |             queue.push(exec);
101 |         } else {
102 |             active = true;
103 |             exec();
104 |         }
105 | 
106 |         return promise;
107 |     };
108 | };
109 | 
110 | export function timeoutAsync(callback: () => void | Promise<void>, timeout: number) {
111 |     return new Promise<void>((res, rej) => {
112 |         setTimeout(async () => {
113 |             await callback();
114 |             res();
115 |         }, timeout);
116 |     });
117 | }
118 | 
119 | export function intervalAsync(callback: () => boolean | Promise<boolean>, timeout: number, eager: boolean = false) {
120 |     return new Promise<void>(async (res, rej) => {
121 |         if (eager && await callback()) return res();
122 |         const interval = setInterval(async () => {
123 |             if (await callback()) {
124 |                 clearInterval(interval);
125 |                 return res();
126 |             }
127 |         }, timeout);
128 |     });
129 | }
130 | 
131 | export function sleep(timeout: number) {
132 |     return timeoutAsync(async () => {}, timeout);
133 | }


--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "compilerOptions": {
 3 |         /* Basic Options */
 4 |         "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
 5 |         "target": "esnext",                       /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', or 'ESNext'. */
 6 |         // "lib": [ "es2019", "dom" ],            /* Specify library files to be included in the compilation */
 7 |         "allowJs": true,                          /* Allow javascript files to be compiled. */
 8 |         // "checkJs": true,                       /* Report errors in .js files. */
 9 |         // "jsx": "preserve",                     /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
10 |         "declaration": true,                      /* Generates corresponding '.d.ts' file. */
11 |         "declarationMap": true,                   /* Generates corresponding '.d.ts.map' file. */
12 |         "declarationDir": "./dist",               /* Specifies the declaration files destination */
13 |         "sourceMap": true,                        /* Generates corresponding '.map' file. */
14 |         // "outFile": "./",                       /* Concatenate and emit output to single file. */
15 |         // "outDir": "./dist",                    /* Redirect output structure to the directory. */
16 |         "removeComments": false,                  /* Do not emit comments to output. */
17 |         // "noEmit": true,                        /* Do not emit outputs. */
18 |         // "importHelpers": true,                 /* Import emit helpers from 'tslib'. */
19 |         // "downlevelIteration": true,            /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
20 |         // "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
21 |     
22 |         /* Strict Type-Checking Options */
23 |         "strict": true,                           /* Enable all strict type-checking options. */
24 |         "noImplicitAny": true,                    /* Raise error on expressions and declarations with an implied 'any' type. */
25 |         // "strictNullChecks": true,              /* Enable strict null checks. */
26 |         // "strictFunctionTypes": true,           /* Enable strict checking of function types. */
27 |         // "noImplicitThis": true,                /* Raise error on 'this' expressions with an implied 'any' type. */
28 |         // "alwaysStrict": true,                  /* Parse in strict mode and emit "use strict" for each source file. */
29 |         // "skipLibCheck": true,                  /* Skip type checking of all declaration files (*.d.ts). */
30 |     
31 |         /* Additional Checks */
32 |         // "noUnusedLocals": true,                /* Report errors on unused locals. */
33 |         // "noUnusedParameters": true,            /* Report errors on unused parameters. */
34 |         // "noImplicitReturns": true,             /* Report error when not all code paths in function return a value. */
35 |         // "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */
36 |     
37 |         /* Module Resolution Options */
38 |         "moduleResolution": "node",               /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
39 |         // "baseUrl": "./",                       /* Base directory to resolve non-absolute module names. */
40 |         // "paths": {},                           /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
41 |         // "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
42 |         "typeRoots": [                            /* List of folders to include type definitions from. */
43 |             "./src/@types",
44 |             "./node_modules/@types"
45 |         ],
46 |         // "types": [],                           /* Type declaration files to be included in compilation. */
47 |         "allowSyntheticDefaultImports": true,     /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
48 |         "esModuleInterop": true,                  /* Allows modules compatibility (commonjs, es6, ...) */
49 |         // "preserveSymlinks": true,              /* Do not resolve the real path of symlinks. */
50 |     
51 |         /* Source Map Options */
52 |         // "sourceRoot": "./",                    /* Specify the location where debugger should locate TypeScript files instead of source locations. */
53 |         // "mapRoot": "./",                       /* Specify the location where debugger should locate map files instead of generated locations. */
54 |         // "inlineSourceMap": true,               /* Emit a single file with source maps instead of having a separate file. */
55 |         // "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
56 |     
57 |         /* Experimental Options */
58 |         // "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
59 |         // "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
60 |     },
61 |     "include": [ "src" ],
62 |     "exclude": [ "node_modules" ]
63 | }


--------------------------------------------------------------------------------
/src/ssr-render.ts:
--------------------------------------------------------------------------------
  1 | import puppeteer, { Browser, ContinueRequestOverrides, Page } from 'puppeteer';
  2 | import { Logger } from './logger';
  3 | import { HttpHeaders, SsrConfig, SsrRenderResult } from './types';
  4 | import { createLock, sleep } from './utils';
  5 | 
  6 | export abstract class SsrRender {
  7 |     constructor(protected configSsr: SsrConfig) { }
  8 | 
  9 |     // Reusable browser connection
 10 |     protected sharedBrowser?: {
 11 |         browser: Promise<Browser>;
 12 |         wsEndpoint: Promise<string>;
 13 |         close: () => Promise<void>;
 14 |     };
 15 | 
 16 |     protected tempBrowsers: Browser[] = [];
 17 | 
 18 |     private lock = createLock();
 19 | 
 20 |     protected async getBrowser(logger: Logger): Promise<Browser> {
 21 |         const cSsr = this.configSsr!;
 22 | 
 23 |         try {
 24 |             await this.lock(async () => {
 25 |                 if (cSsr.sharedBrowser && !this.sharedBrowser) {
 26 |                     logger.debug('SSR: Creating browser instance');
 27 |                     const browserMain = puppeteer.launch(cSsr.browserConfig!);
 28 |                     const wsEndpoint = browserMain.then(e => e.wsEndpoint());
 29 |                     this.sharedBrowser = {
 30 |                         browser: browserMain,
 31 |                         wsEndpoint: wsEndpoint,
 32 |                         close: async () => {
 33 |                             try {
 34 |                                 logger.debug('SSR: Closing browser instance');
 35 |                                 this.sharedBrowser = undefined;
 36 |                                 await (await browserMain).close();
 37 |                             } catch (err) {
 38 |                                 logger.error('BrowserCloseError', err, false);
 39 |                             }
 40 |                         },
 41 |                     };
 42 |                 }
 43 |             });
 44 |         } catch (err: any) {
 45 |             logger.error('BrowserError', err, false);
 46 |         }
 47 | 
 48 |         logger.debug('SSR: Connecting');
 49 |         const wsEndpoint = this.sharedBrowser?.wsEndpoint && await this.sharedBrowser.wsEndpoint;
 50 | 
 51 |         logger.debug(`SSR: WSEndpoint=${wsEndpoint}`);
 52 |         const browser = wsEndpoint ? await puppeteer.connect({ browserWSEndpoint: wsEndpoint }) : await puppeteer.launch(cSsr.browserConfig!);
 53 | 
 54 |         return browser;
 55 |     }
 56 | 
 57 |     protected async tryRender(urlStr: string, headers: HttpHeaders, logger: Logger, method?: string): Promise<SsrRenderResult> {
 58 |         const cSsr = this.configSsr!;
 59 |         const start = Date.now();
 60 | 
 61 |         let browser: Browser | undefined;
 62 |         let page: Page | undefined;
 63 | 
 64 |         try {
 65 |             browser = await this.getBrowser(logger);
 66 |             if (!cSsr.sharedBrowser) this.tempBrowsers.push(browser);
 67 | 
 68 |             // await sleep(10_000); // test sigterm shutdown
 69 | 
 70 |             const url = new URL(urlStr);
 71 | 
 72 |             // Indicate headless render to client
 73 |             // e.g. use to disable some features if ssr
 74 |             for (let param of cSsr.queryParams!)
 75 |                 url.searchParams.set(param.key, param.value);
 76 | 
 77 |             logger.debug('SSR: New Page');
 78 |             page = await browser.newPage();
 79 | 
 80 |             // Page middleware
 81 |             this.configSsr.pageMiddleware && await this.configSsr.pageMiddleware(page, url);
 82 | 
 83 |             // Intercept network requests
 84 |             let interceptCount = 0;
 85 |             await page.setRequestInterception(true);
 86 |             page.on('request', req => {
 87 |                 // console.log('Request:', req.url());
 88 | 
 89 |                 interceptCount++;
 90 | 
 91 |                 // Ignore requests for resources that don't produce DOM (e.g. images, stylesheets, media)
 92 |                 const reqType = req.resourceType();
 93 |                 if (!cSsr.allowedResources!.includes(reqType)) return req.abort();
 94 | 
 95 |                 // Custom headers and method
 96 |                 let override: ContinueRequestOverrides = { method: 'GET', headers: req.headers() };
 97 |                 if (interceptCount === 1) {
 98 |                     if (method) override.method = method;
 99 |                     override.headers = this.fixReqHeaders({ ...(headers || {}), ...(override.headers || {}) });
100 |                     logger.debug(`SSR: Intercepted - ${JSON.stringify(override.headers)}`);
101 |                 }
102 | 
103 |                 // Pass through all other requests
104 |                 req.continue(override);
105 |             });
106 | 
107 |             // Render
108 | 
109 |             logger.debug('SSR: Accessing');
110 |             const response = await page.goto(url.toString(), { waitUntil: cSsr.waitUntil, timeout: cSsr.timeout });
111 |             // await page.waitForNetworkIdle({ idleTime: 1000, timeout: cSsr.timeout });
112 | 
113 |             const ssrStatus = response?.status();
114 |             const ssrHeaders = response?.headers();
115 |             const resHeaders = this.fixResHeaders(ssrHeaders);
116 | 
117 |             logger.debug(`SSR: Rendered - ${JSON.stringify(resHeaders)}`);
118 | 
119 |             if (cSsr.sleep) await sleep(cSsr.sleep);
120 | 
121 |             // Serialize text from DOM
122 | 
123 |             const text = await page.content();
124 |             logger.debug(`SSR: DOM Serialized - ${text.length} size`);
125 | 
126 |             const ttRenderMs = Date.now() - start;
127 | 
128 |             return { status: ssrStatus, text, headers: resHeaders, ttRenderMs };
129 |         } catch (err: any) {
130 |             let error = ((err && (err.message || err.toString())) || 'Proxy Error');
131 |             const ttRenderMs = Date.now() - start;
132 |             return { ttRenderMs, error };
133 |         } finally {
134 |             logger.debug('SSR: Closing');
135 |             if (page && !page.isClosed()) await page.close();
136 |             if (browser) {
137 |                 if (cSsr.sharedBrowser) {
138 |                     await browser.disconnect();
139 |                 } else {
140 |                     await browser.close();
141 |                     this.tempBrowsers = this.tempBrowsers.filter(e => e !== browser);
142 |                 }
143 |             }
144 |             logger.debug('SSR: Closed');
145 |         }
146 |     }
147 | 
148 |     protected async browserShutDown() {
149 |         if (this.configSsr!.sharedBrowser) {
150 |             Logger.debug('Closing the shared browser...');
151 |             await this.sharedBrowser?.close();
152 |         } else {
153 |             this.tempBrowsers.forEach(async (browser,i,arr) => {
154 |                 Logger.debug(`Closing temp browser ${browser?.process()?.pid} (${i+1}/${arr.length})...`);
155 |                 if (browser) await browser.close();
156 |             });
157 |         }
158 |     }
159 | 
160 |     protected fixReqHeaders(headers: any) {
161 |         const proxyHeaders = this.fixHeaders(headers);
162 |         delete proxyHeaders['host'];
163 |         delete proxyHeaders['referer'];
164 |         delete proxyHeaders['user-agent'];
165 |         return proxyHeaders;
166 |     }
167 | 
168 |     protected fixResHeaders(headers: any) {
169 |         const proxyHeaders = this.fixHeaders({});
170 |         // TODO: fix response headers
171 |         // delete proxyHeaders['content-encoding'];
172 |         // delete proxyHeaders['transfer-encoding'];
173 |         return proxyHeaders;
174 |     }
175 | 
176 |     protected fixHeaders(headers: object) {
177 |         return Object.entries(headers).reduce((acc, [key, value]) => (value != null ? { ...acc, [key.toLowerCase()]: value?.toString() } : acc), {} as HttpHeaders);
178 |     }
179 | }
180 | 


--------------------------------------------------------------------------------
/src/ssr-build.ts:
--------------------------------------------------------------------------------
  1 | import deepmerge from 'deepmerge';
  2 | import express from 'express';
  3 | import fs from 'fs';
  4 | import os from 'os';
  5 | import path from 'path';
  6 | import { Logger } from './logger';
  7 | import { SsrRender } from './ssr-render';
  8 | import { BuildParams, BuildResult, LogLevel, SsrBuildConfig } from './types';
  9 | import { promiseParallel, promiseRetry } from './utils';
 10 | 
 11 | export class SsrBuild extends SsrRender {
 12 |     private config: SsrBuildConfig;
 13 | 
 14 |     constructor(customConfig: SsrBuildConfig) {
 15 |         const defaultConfig: SsrBuildConfig = {
 16 |             httpPort: 8080,
 17 |             hostname: 'localhost',
 18 |             src: 'src',
 19 |             dist: 'dist',
 20 |             stopOnError: false,
 21 |             forceExit: false,
 22 |             serverMiddleware: undefined,
 23 |             reqMiddleware: undefined,
 24 |             resMiddleware: undefined,
 25 |             ssr: {
 26 |                 browserConfig: { headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'], timeout: 60000 },
 27 |                 sharedBrowser: true,
 28 |                 queryParams: [{ key: 'headless', value: 'true' }],
 29 |                 allowedResources: ['document', 'script', 'xhr', 'fetch'],
 30 |                 waitUntil: 'networkidle0',
 31 |                 timeout: 60000,
 32 |                 sleep: undefined,
 33 |             },
 34 |             log: {
 35 |                 level: LogLevel.Info,
 36 |                 console: {
 37 |                     enabled: true,
 38 |                 },
 39 |                 file: {
 40 |                     enabled: true,
 41 |                     dirPath: path.join(os.tmpdir(), 'ssr-proxy-js/logs'),
 42 |                 },
 43 |             },
 44 |             job: {
 45 |                 retries: 3,
 46 |                 parallelism: 5,
 47 |                 routes: [{ method: 'GET', url: '/' }],
 48 |             },
 49 |         };
 50 | 
 51 |         let config: SsrBuildConfig;
 52 | 
 53 |         if (customConfig) {
 54 |             config = deepmerge<SsrBuildConfig>(defaultConfig, customConfig, {
 55 |                 arrayMerge: (destArray, srcArray, opts) => srcArray,
 56 |             });
 57 |         } else {
 58 |             console.warn('No configuration found for ssr-proxy-js, using default config!');
 59 |             config = defaultConfig;
 60 |         }
 61 | 
 62 |         config.src = path.isAbsolute(config.src!) ? config.src! : path.join(process.cwd(), config.src!);
 63 |         config.dist = path.isAbsolute(config.dist!) ? config.dist! : path.join(process.cwd(), config.dist!);
 64 |         if (config.job!.parallelism! < 1) throw new Error(`Parallelism should be greater than 0 (${config.job!.parallelism})`);
 65 | 
 66 |         super(config.ssr!);
 67 |         this.config = config;
 68 | 
 69 |         const cLog = this.config.log;
 70 |         Logger.setLevel(cLog!.level!);
 71 |         Logger.configConsole(cLog!.console!.enabled!);
 72 |         Logger.configFile(cLog!.file!.enabled!, cLog!.file!.dirPath!);
 73 |     }
 74 | 
 75 |     async start(): Promise<BuildResult[]> {
 76 |         Logger.info(`SrcPath: ${this.config.src!}`);
 77 |         Logger.info(`DistPath: ${this.config.dist!}`);
 78 | 
 79 |         const { server } = await this.serve();
 80 | 
 81 |         const shutDown = async () => {
 82 |             Logger.debug('Shutting down...');
 83 | 
 84 |             await this.browserShutDown();
 85 | 
 86 |             Logger.debug('Closing the server...');
 87 |             server.close(() => {
 88 |                 Logger.debug('Shut down completed!');
 89 |                 if (this.config.forceExit) process.exit(0);
 90 |             });
 91 | 
 92 |             if (this.config.forceExit) {
 93 |                 setTimeout(() => {
 94 |                     Logger.error(`Shutdown`, 'Could not shut down in time, forcefully shutting down!');
 95 |                     process.exit(1);
 96 |                 }, 10000);
 97 |             }
 98 |         };
 99 |         process.on('SIGTERM', shutDown);
100 |         process.on('SIGINT', shutDown);
101 | 
102 |         try {
103 |             return await this.render();
104 |         } catch (err) {
105 |             throw err;
106 |         } finally {
107 |             await shutDown();
108 |         }
109 |     }
110 | 
111 |     async serve() {
112 |         const cfg = this.config;
113 | 
114 |         const app = express();
115 | 
116 |         // Serve Static Files
117 |         app.use(express.static(cfg.src!));
118 | 
119 |         // Catch-all: Serve index.html for any non-file request
120 |         app.use((req, res, next) => {
121 |             if (cfg.serverMiddleware) cfg.serverMiddleware(req, res, next);
122 |             else res.sendFile(path.join(cfg.src!, 'index.html')); // serve root index.html
123 |         });
124 | 
125 |         // Error Handler
126 |         app.use((err: any, req: any, res: any, next: any) => {
127 |             Logger.error('Error', err, true);
128 |             res.contentType('text/plain');
129 |             res.status(err.status || 500);
130 |             res.send(Logger.errorStr(err));
131 |             next();
132 |         });
133 | 
134 |         // HTTP Listen
135 |         const server = app.listen(this.config.httpPort!, this.config.hostname!, () => {
136 |             Logger.debug('----- Starting HTTP Server -----');
137 |             Logger.debug(`Listening on http://${this.config.hostname!}:${this.config.httpPort!}`);
138 |         });
139 | 
140 |         return { app, server };
141 |     }        
142 | 
143 |     async render(): Promise<BuildResult[]> {
144 |         const $this = this;
145 |         const cJob = this.config.job!;
146 | 
147 |         const routesStr = '> ' + cJob.routes!.map(e => `[${e.method ?? 'GET'}] ${e.url}`).join('\n> ');
148 |         Logger.info(`SSR Building (p:${cJob.parallelism},r:${cJob.retries}):\n${routesStr}`);
149 | 
150 |         const results = await promiseParallel<BuildResult>(cJob.routes!.map((route) => () => new Promise(async (res, rej) => {
151 |             const logger = new Logger(true);
152 | 
153 |             try {
154 |                 const result = await promiseRetry(runRender, cJob.retries!, e => logger.warn('SSR Build Retry', e, false));
155 |                 res(result);
156 |             } catch (err) {
157 |                 logger.error('SSR Build', err);
158 |                 rej(err);
159 |             }
160 | 
161 |             async function runRender(): Promise<BuildResult> {
162 |                 const targetUrl = new URL(route.url, `http://${$this.config.hostname!}:${$this.config.httpPort!}`);
163 | 
164 |                 const params: BuildParams = { method: route.method, targetUrl, headers: route.headers || {} };
165 |                 if ($this.config.reqMiddleware) await $this.config.reqMiddleware(params);
166 | 
167 |                 const { text, status, headers, ttRenderMs } = await $this.tryRender(params.targetUrl.toString(), params.headers || {}, logger, params.method);
168 | 
169 |                 const urlPath = path.join(params.targetUrl.pathname, params.targetUrl.pathname.endsWith('.html') ? '' : 'index.html');
170 |                 const filePath = path.join($this.config.dist!, urlPath);
171 |                 const result: BuildResult = { text, status, headers, urlPath, filePath, encoding: 'utf-8' };
172 |                 if ($this.config.resMiddleware) await $this.config.resMiddleware(params, result);
173 | 
174 |                 if (status !== 200) {
175 |                     const msg = `Render failed: ${params.targetUrl} - Status ${status} - ${ttRenderMs}ms\n${text}`;
176 |                     if ($this.config.stopOnError) throw new Error(msg);
177 |                     logger.warn('SSR Build', msg);
178 |                     return result;
179 |                 }
180 | 
181 |                 if (result.text == null) {
182 |                     logger.warn('SSR Build', `Empty content: ${params.targetUrl} - ${ttRenderMs}ms`);
183 |                     return result;
184 |                 }
185 | 
186 |                 logger.info(`Saving render: ${params.targetUrl} -> ${result.filePath}`);
187 | 
188 |                 const dirPath = path.dirname(result.filePath);
189 |                 if (!fs.existsSync(dirPath)) fs.mkdirSync(dirPath, { recursive: true });
190 |                 fs.writeFileSync(result.filePath, result.text, { encoding: result.encoding });
191 | 
192 |                 logger.debug(`SSR Built: ${params.targetUrl} - ${ttRenderMs}ms`);
193 | 
194 |                 return result;
195 |             }
196 |         })), cJob.parallelism!, false);
197 | 
198 |         Logger.info(`SSR build finished!`);
199 | 
200 |         return results;
201 |     }
202 | }
203 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | [![NuGet Status](https://img.shields.io/npm/v/ssr-proxy-js)](https://www.npmjs.com/package/ssr-proxy-js)
  2 | [![NuGet Status](https://img.shields.io/npm/dt/ssr-proxy-js)](https://www.npmjs.com/package/ssr-proxy-js)
  3 | 
  4 | # SSRProxy.js
  5 | 
  6 | Modes:\
  7 | [SSR Build - Static Site Generator](#ssr-build-static-site-generator-mode)\
  8 | [SSR Proxy - Server Rendering](#ssr-proxy-server-rendering-mode)
  9 | 
 10 | A Node.js tool for Server-Side Rendering (SSR) and Static Site Generation (SSG) using headless Chrome via Puppeteer.
 11 | 
 12 | Server-Side Rendering, or SSR for short, is a technique used to serve Single-Page Applications (SPAs, e.g. React.js, Vue.js and Angular based websites) with Web Crawlers in mind, such as Googlebot. Crawlers are used everywhere in the internet to a variety of objectives, with the most known being for indexing the web for search engines, which is done by companies such as Google (Googlebot), Bing (Bingbot) and DuckDuckGo (DuckDuckBot).
 13 | 
 14 | The main problem of serving SPAs "normally" (i.e. Client-Side Rendering) is that when your website is accessed by a Web Crawler, it's usually only able to read the source HTML code, which most probably does not represent your actual website. In case of a React App, for example, a Crawler might be only able to interpret your website like so:
 15 | 
 16 | ```html
 17 | <!DOCTYPE html>
 18 | <html lang="en">
 19 |   <head>
 20 |     <meta charset="utf-8" />
 21 |     <title>React App</title>
 22 |   </head>
 23 |   <body>
 24 |     <div id="root"></div>
 25 |     <script src="/app.js"></script>
 26 |   </body>
 27 | </html>
 28 | ```
 29 | 
 30 | For the contents of a SPA to be correct, the JavaScript files should be loaded and executed by the browser, and that's where Server-Side Rendering plays a big role. SSR will receive the HTTP request from the client, create a browser instance, load the page just like we do while surfing the web, and just then return the actual rendered HTML to the request, after the SPA is fully loaded.
 31 | 
 32 | The implemantation of this package is hugelly inspired by an article from Google, using Pupperteer as it's engine:
 33 | https://developers.google.com/web/tools/puppeteer/articles/ssr
 34 | 
 35 | The main problem regarding the workflow described above is that the process of rendering the web page through a browser takes some time, so if done incorrectly, it might have a big impact on the users experience. That's why this package also comes with two essencial feature: **Caching**, **Fallbacks** and **Static Site Generation**.
 36 | 
 37 | ---
 38 | 
 39 | ## SSR Build (Static Site Generator mode)
 40 | 
 41 | Build pre-rendered pages to serve to your users, without any added server complexity or extra response delay, using your server tool of choice (e g. nginx).
 42 | 
 43 | If all your content is static, meaning it won't change dependending on who or how your pages are accessed, you can pre-build all your routes using the `--mode=build` option, instead of building in real time with the default SSR Proxy mode. This will access all your pre-defined routes in build time, render the HTML, and save the resulting content back to a dist folder. You can then serve your dist folder instead of serving your original non pre-rendered bundle.
 44 | 
 45 | ### npx Example
 46 | 
 47 | **Commands**
 48 | ```bash
 49 | # With Args
 50 | npx ssr-proxy-js --mode=build --src=./public --dist=./dist --job.routes='[{"url":"/"},{"url":"/nested"}]'
 51 | 
 52 | # With Config File
 53 | npx ssr-proxy-js --mode=build -c ./ssr-build-js.config.json
 54 | ```
 55 | 
 56 | **Config File**
 57 | ```javascript
 58 | // ./ssr-build-js.config.json
 59 | {
 60 |     "src": "./public",
 61 |     "dist": "./src",
 62 |     "job": {
 63 |         "routes": [
 64 |             { "method": "GET", "url": "/" },
 65 |             { "method": "GET", "url": "/nested" }
 66 |         ]
 67 |     }
 68 | }
 69 | ```
 70 | 
 71 | ### Simple Example
 72 | 
 73 | ```javascript
 74 | const { SsrBuild } = require('ssr-proxy-js');
 75 | 
 76 | const ssrBuild = new SsrBuild({
 77 |     "src": "./public",
 78 |     "dist": "./src",
 79 |     "job": {
 80 |         "routes": [
 81 |             { "method": "GET", "url": "/" },
 82 |             { "method": "GET", "url": "/nested" }
 83 |         ]
 84 |     }
 85 | });
 86 | 
 87 | ssrBuild.start();
 88 | ```
 89 | 
 90 | ### Full Example
 91 | 
 92 | ```typescript
 93 | import * as os from 'os';
 94 | import * as path from 'path';
 95 | import { LogLevel, SsrBuild, SsrBuildConfig } from 'ssr-proxy-js';
 96 | 
 97 | const config: SsrBuildConfig = {
 98 |     httpPort: 8080,
 99 |     hostname: 'localhost',
100 |     src: 'public',
101 |     dist: 'dist',
102 |     stopOnError: false,
103 |     serverMiddleware: async (req, res, next) => {
104 |         res.sendFile(path.join(__dirname, 'public/index.html'));
105 |     },
106 |     reqMiddleware: async (params) => {
107 |         params.headers['Referer'] = 'http://google.com';
108 |         return params;
109 |     },
110 |     resMiddleware: async (params, result) => {
111 |         if (result.text == null) return result;
112 |         result.text = result.text.replace('</html>', '\n\t<div>MIDDLEWARE</div>\n</html>');
113 |         result.text = result.text.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '');
114 |         return result;
115 |     },
116 |     ssr: {
117 |         browserConfig: { headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'], timeout: 60000 },
118 |         sharedBrowser: true,
119 |         queryParams: [{ key: 'headless', value: 'true' }],
120 |         allowedResources: ['document', 'script', 'xhr', 'fetch'],
121 |         waitUntil: 'networkidle0',
122 |         timeout: 60000,
123 |     },
124 |     log: {
125 |         level: LogLevel.Info,
126 |         console: {
127 |             enabled: true,
128 |         },
129 |         file: {
130 |             enabled: true,
131 |             dirPath: path.join(os.tmpdir(), 'ssr-proxy-js/logs'),
132 |         },
133 |     },
134 |     job: {
135 |         retries: 3,
136 |         parallelism: 5,
137 |         routes: [{ method: 'GET', url: '/' },{ method: 'GET', url: '/nested' },{ method: 'GET', url: '/page.html' },{ method: 'GET', url: '/iframe.html' }],
138 |     },
139 | };
140 | 
141 | const ssrBuild = new SsrBuild(config);
142 | 
143 | ssrBuild.start();
144 | ```
145 | 
146 | ---
147 | 
148 | ## SSR Proxy (Server Rendering mode)
149 | 
150 | Proxy your requests via the SSR server to serve pre-rendered pages to your users.
151 | 
152 | > Note: to ensure the best security and performance, it's adivisable to use this proxy behind a reverse proxy, such as [Nginx](https://www.nginx.com/).
153 | 
154 | ### npx Example
155 | 
156 | **Commands**
157 | ```bash
158 | # With Args
159 | npx ssr-proxy-js --httpPort=8080 --targetRoute=http://localhost:3000 --static.dirPath=./public --proxyOrder=SsrProxy --proxyOrder=StaticProxy
160 | 
161 | # With Config File
162 | npx ssr-proxy-js -c ./ssr-proxy-js.config.json
163 | ```
164 | 
165 | **Config File**
166 | ```javascript
167 | // ./ssr-proxy-js.config.json
168 | {
169 |     "httpPort": 8080,
170 |     "targetRoute": "http://localhost:3000"
171 | }
172 | ```
173 | 
174 | ### Simple Example
175 | 
176 | ```javascript
177 | const { SsrProxy } = require('ssr-proxy-js');
178 | 
179 | const ssrProxy = new SsrProxy({
180 |     httpPort: 8080,
181 |     targetRoute: 'http://localhost:3000'
182 | });
183 | 
184 | ssrProxy.start();
185 | ```
186 | 
187 | ### Full Example
188 | 
189 | ```javascript
190 | const os = require('os');
191 | const path = require('path');
192 | const { SsrProxy } = require('ssr-proxy-js-local');
193 | 
194 | const BASE_PROXY_PORT = '8080';
195 | const BASE_PROXY_ROUTE = `http://localhost:${BASE_PROXY_PORT}`;
196 | const STATIC_FILES_PATH = path.join(process.cwd(), 'public');
197 | const LOGGING_PATH = path.join(os.tmpdir(), 'ssr-proxy/logs');
198 | 
199 | console.log(`\nLogging at: ${LOGGING_PATH}`);
200 | 
201 | const ssrProxy = new SsrProxy({
202 |     httpPort: 8081,
203 |     hostname: '0.0.0.0',
204 |     targetRoute: BASE_PROXY_ROUTE,
205 |     proxyOrder: ['SsrProxy', 'HttpProxy', 'StaticProxy'],
206 |     isBot: (method, url, headers) => true,
207 |     failStatus: params => 404,
208 |     customError: err => err.toString(),
209 |     ssr: {
210 |         shouldUse: params => params.isBot && (/\.html$/.test(params.targetUrl.pathname) || !/\./.test(params.targetUrl.pathname)),
211 |         browserConfig: { headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'], timeout: 60000 },
212 |         queryParams: [{ key: 'headless', value: 'true' }],
213 |         allowedResources: ['document', 'script', 'xhr', 'fetch'],
214 |         waitUntil: 'networkidle0',
215 |         timeout: 60000,
216 |     },
217 |     httpProxy: {
218 |         shouldUse: params => true,
219 |         timeout: 60000,
220 |     },
221 |     static: {
222 |         shouldUse: params => true,
223 |         dirPath: STATIC_FILES_PATH,
224 |         useIndexFile: path => path.endsWith('/'),
225 |         indexFile: 'index.html',
226 |     },
227 |     log: {
228 |         level: 3,
229 |         console: {
230 |             enabled: true,
231 |         },
232 |         file: {
233 |             enabled: true,
234 |             dirPath: LOGGING_PATH,
235 |         },
236 |     },
237 |     cache: {
238 |         shouldUse: params => params.proxyType === 'SsrProxy',
239 |         maxEntries: 50,
240 |         maxByteSize: 50 * 1024 * 1024, // 50MB
241 |         expirationMs: 25 * 60 * 60 * 1000, // 25h
242 |         autoRefresh: {
243 |             enabled: true,
244 |             shouldUse: () => true,
245 |             proxyOrder: ['SsrProxy', 'HttpProxy'],
246 |             initTimeoutMs: 5 * 1000, // 5s
247 |             intervalCron: '0 0 3 * * *', // every day at 3am
248 |             intervalTz: 'Etc/UTC',
249 |             retries: 3,
250 |             parallelism: 5,
251 |             closeBrowser: true,
252 |             isBot: true,
253 |             routes: [
254 |                 { method: 'GET', url: '/' },
255 |                 { method: 'GET', url: '/login' },
256 |             ],
257 |         },
258 |     },
259 | });
260 | 
261 | ssrProxy.start();
262 | ```
263 | 
264 | ### Caching
265 | 
266 | Caching allows us to increase the performance of the web serving by preventing excessive new renders for web pages that have been accessed recently. Caching is highly configurable to allow total control of the workflow, for example, it's possible to decide if cache should or shouldn't be used each time the website is accessed, with the "shouldUse" option. Also, it's possible to configure a automatic cache refresh, using the "cache.autoRefresh" configuration.
267 | 
268 | ### Fallbacks
269 | 
270 | In case of a human user access, we can serve the web site the "normal" way, without asking the SSR to pre-render the page. For that it's possible to use 3 types of proxies: SSR Proxy, HTTP Proxy or Static File Serving, in any order that you see fit. Firstly, the order of priority should be configured with the "proxyOrder" option, so for example, if configured as ['SsrProxy', 'HttpProxy', 'StaticProxy'], "ssr.shouldUse" will ask if SSR should be used, if it returns false, then "httpProxy.shouldUse" will ask if HTTP Proxy should be used, and finally, "static.shouldUse" will ask if Static File Serving should be used. If the return of all proxy options is false, or if one of then returns a exception (e.g. page not found), the web server will return a empty HTTP response with status equals to the return of the "failStatus" callback.
271 | 
272 | ## More options
273 | 
274 | For further options, check:
275 | 
276 | Example: https://github.com/Tpessia/ssr-proxy-js/tree/main/test
277 | 
278 | Types: https://github.com/Tpessia/ssr-proxy-js/blob/main/src/types.ts
279 | 
280 | <!-- ## Release
281 | 
282 | 1. Commit new code
283 | 2. npm run publish:np -->
284 | 
285 | <!-- Node.js modules
286 | https://zellwk.com/blog/publish-to-npm/
287 | https://www.sensedeep.com/blog/posts/2021/how-to-create-single-source-npm-module.html
288 | https://electerious.medium.com/from-commonjs-to-es-modules-how-to-modernize-your-node-js-app-ad8cdd4fb662
289 | 
290 | Puppeteer SSR
291 | https://developers.google.com/web/tools/puppeteer/articles/ssr
292 | 
293 | Bot User Agent
294 | https://www.keycdn.com/blog/web-crawlers
295 | https://github.com/omrilotan/isbot/blob/main/src/list.json
296 | https://github.com/monperrus/crawler-user-agents/blob/master/crawler-user-agents.json
297 | 
298 | Nginx Redirect by User Agent
299 | https://serverfault.com/questions/775463/nginx-redirect-based-on-user-agent
300 | https://serverfault.com/questions/865055/nginx-redirect-if-user-agent-contains-xyz
301 | 
302 | Other Proxies
303 | https://github.com/xiamx/ssr-proxy / https://www.npmjs.com/package/ssr-proxy
304 | https://cnpmjs.org/package/spa-ssr-proxy / https://www.npmjs.com/package/spa-ssr-proxy
305 | https://github.com/jamiekyle-eb/ssr-proxy
306 | https://github.com/postor/ssr-proxy-puppeteer / https://www.npmjs.com/package/ssr-proxy-puppeteer -->
307 | 


--------------------------------------------------------------------------------
/src/types.ts:
--------------------------------------------------------------------------------
  1 | import { NextFunction, Request, Response } from 'express';
  2 | import { BrowserConnectOptions, BrowserLaunchArgumentOptions, LaunchOptions, Page, Product, PuppeteerLifeCycleEvent, ResourceType } from 'puppeteer';
  3 | import { Stream } from 'stream';
  4 | 
  5 | // SSR
  6 | 
  7 | export type HttpHeaders = Record<string, string>;
  8 | 
  9 | export interface SsrRenderResult {
 10 |     status?: number;
 11 |     text?: string;
 12 |     error?: string;
 13 |     headers?: HttpHeaders;
 14 |     ttRenderMs: number;
 15 | }
 16 | 
 17 | /**
 18 |  * SSR config
 19 |  * @public
 20 |  */
 21 | export interface SsrConfig {
 22 |     /**
 23 |      * Browser configuration used by Puppeteer
 24 |      * @default
 25 |      * { headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'], timeout: 60000 }
 26 |      */
 27 |     browserConfig?: SsrBrowerConfig;
 28 |     /**
 29 |      * Use shared browser instance
 30 |      * @default true
 31 |      */
 32 |     sharedBrowser?: boolean;
 33 |     /**
 34 |      * Which query string params to include in the url before proxying
 35 |      * @default
 36 |      * [{ key: 'headless', value: 'true' }]
 37 |      */
 38 |     queryParams?: {
 39 |         key: string;
 40 |         value: string;
 41 |     }[];
 42 |     /**
 43 |      * Which resource types to load
 44 |      * @default
 45 |      * ['document', 'script', 'xhr', 'fetch']
 46 |      */
 47 |     allowedResources?: ResourceType[];
 48 |     /**
 49 |      * Which events to wait before returning the rendered HTML
 50 |      * @default 'networkidle0'
 51 |      */
 52 |     waitUntil?: PuppeteerLifeCycleEvent | PuppeteerLifeCycleEvent[];
 53 |     /**
 54 |      * Timeout
 55 |      * @default 60000
 56 |      */
 57 |     timeout?: number;
 58 |     /**
 59 |      * Sleep time for debugging
 60 |      * @default undefined
 61 |      */
 62 |     sleep?: number;
 63 |     /**
 64 |      * Function for configuring the page before rendering
 65 |      * @example async (page, url) => { await page.setViewport({ width: 1366, height: 768 }); }
 66 |      * @default undefined
 67 |      */
 68 |     pageMiddleware?: (page: Page, url: URL) => Promise<void>;
 69 | }
 70 | 
 71 | /**
 72 |  * SSR job config
 73 |  * @public
 74 |  */
 75 | export interface SsrJob {
 76 |     /**
 77 |      * Number of retries if fails
 78 |      * @default 3
 79 |      */
 80 |     retries?: number;
 81 |     /**
 82 |      * Maximum number of parallel refreshes
 83 |      * @default 5 * 60 * 1000 // 5 minutes
 84 |      */
 85 |     parallelism?: number;
 86 |     /**
 87 |      * Routes to auto refresh
 88 |      * @default
 89 |      * [{ method: 'GET', url: '/' }]
 90 |      */
 91 |     routes?: {
 92 |         /**
 93 |          * Route URL
 94 |          * @example '/example/'
 95 |          */
 96 |         url: string;
 97 |         /**
 98 |          * Route HTTP Method
 99 |          * @example 'GET'
100 |          */
101 |         method?: string;
102 |         /**
103 |          * Route Headers
104 |          * @example { 'X-Example': 'Test' }
105 |          */
106 |         headers?: HttpHeaders;
107 |     }[];
108 | }
109 | 
110 | /**
111 |  * Logging configuration
112 |  * @public
113 |  */
114 | export interface LogConfig {
115 |     /**
116 |      * Logging level
117 |      * @example
118 |      * ```text
119 |      * None = 0, Error = 1, Info = 2, Debug = 3
120 |      * ```
121 |      * @default 2
122 |      */
123 |     level?: LogLevel;
124 |     /**
125 |      * Console logging configuration
126 |      */
127 |     console?: {
128 |         /**
129 |          * Indicates whether to enable the console logging method
130 |          * @default true
131 |          */
132 |         enabled?: boolean;
133 |     };
134 |     /**
135 |      * File logging configuration
136 |      */
137 |     file?: {
138 |         /**
139 |          * Indicates whether to enable the file logging method
140 |          * @default true
141 |          */
142 |         enabled?: boolean;
143 |         /**
144 |          * Absolute path of the logging directory
145 |          * @default path.join(os.tmpdir(), 'ssr-proxy-js/logs')
146 |          */
147 |         dirPath?: string;
148 |     };
149 | };
150 | 
151 | // SSR Build
152 | 
153 | export interface BuildResult {
154 |     text?: string;
155 |     status?: number;
156 |     headers?: HttpHeaders;
157 |     urlPath: string;
158 |     filePath: string;
159 |     encoding: BufferEncoding;
160 | }
161 | 
162 | export interface BuildParams {
163 |     method?: string;
164 |     targetUrl: URL;
165 |     headers: HttpHeaders;
166 | }
167 | 
168 | /**
169 |  * Build config
170 |  * @public
171 |  */
172 | export interface SsrBuildConfig {
173 |     /**
174 |      * File server http port
175 |      * @default 8080
176 |      */
177 |     httpPort?: number;
178 |     /**
179 |      * Proxy server hostname
180 |      * @default 'localhost'
181 |      */
182 |     hostname?: string;
183 |     /**
184 |      * Source directory
185 |      * @default 'src'
186 |      */
187 |     src?: string;
188 |     /**
189 |      * Build output directory
190 |      * @default 'dist'
191 |      */
192 |     dist?: string;
193 |     /**
194 |      * Indicates whether to stop the build process on error (non-200 status code) 
195 |      * @default false
196 |      */
197 |     stopOnError?: boolean;
198 |     /**
199 |      * Indicates whether to force exit with process.exit on shutdown
200 |      * @default false
201 |      */
202 |     forceExit?: boolean;
203 |     /**
204 |      * Custom server middleware
205 |      * @default undefined
206 |      */
207 |     serverMiddleware?: (req: Request, res: Response, next: NextFunction) => Promise<void>;
208 |     /**
209 |      * Function for processing the original request before proxying
210 |      * @default undefined
211 |      */
212 |     reqMiddleware?: (params: BuildParams) => Promise<BuildParams>;
213 |     /**
214 |      * Function for processing the proxy result before serving
215 |      * @default undefined
216 |      */
217 |     resMiddleware?: (params: BuildParams, result: BuildResult) => Promise<BuildResult>;
218 |     ssr?: SsrConfig;
219 |     job?: SsrJob;
220 |     log?: LogConfig;
221 | }
222 | 
223 | // SSR Proxy
224 | 
225 | export enum ProxyType {
226 |     SsrProxy = 'SsrProxy',
227 |     HttpProxy = 'HttpProxy',
228 |     StaticProxy = 'StaticProxy',
229 |     // Redirect = 'Redirect',
230 | }
231 | 
232 | export interface ProxyResult {
233 |     text?: string;
234 |     status?: number;
235 |     stream?: Stream;
236 |     contentType?: string;
237 |     skipped?: boolean;
238 |     error?: any;
239 |     headers?: HttpHeaders;
240 | }
241 | 
242 | export interface ProxyParams {
243 |     sourceUrl: string;
244 |     method?: string;
245 |     headers: HttpHeaders;
246 |     targetUrl: URL;
247 |     isBot: boolean;
248 |     cacheBypass: boolean;
249 |     lastError?: any;
250 | }
251 | 
252 | export interface ProxyTypeParams extends ProxyParams {
253 |     proxyType: ProxyType;
254 | }
255 | 
256 | export type SsrBrowerConfig = LaunchOptions & BrowserLaunchArgumentOptions & BrowserConnectOptions & {
257 |     product?: Product;
258 |     extraPrefsFirefox?: Record<string, unknown>;
259 | };
260 | 
261 | /**
262 |  * Proxy config
263 |  * @public
264 |  */
265 | export interface SsrProxyConfig {
266 |     /**
267 |      * Proxy server http port
268 |      * @default 8080
269 |      */
270 |     httpPort?: number;
271 |     /**
272 |      * Proxy server https port
273 |      * @default 8443
274 |      */
275 |     httpsPort?: number;
276 |     /**
277 |      * Proxy server https key
278 |      * @default undefined
279 |      */
280 |     httpsKey?: string;
281 |     /**
282 |      * Proxy server https cert
283 |      * @default undefined
284 |      */
285 |     httpsCert?: string;
286 |     /**
287 |      * Proxy server hostname
288 |      * @default '0.0.0.0'
289 |      */
290 |     hostname?: string;
291 |     /**
292 |      * Target route for SSR and HTTP proxy
293 |      * 
294 |      * With the default configuration, http://0.0.0.0:8080 will proxy to http://localhost:80
295 |      * @default 'http://localhost:80'
296 |      */
297 |     targetRoute?: string;
298 |     /**
299 |      * Defines the order which the proxy service will follow in case of errors
300 |      * 
301 |      * For example, if defined as [ProxyType.SsrProxy, ProxyType.HttpProxy, ProxyType.StaticProxy],
302 |      * it will try to use Server-Side Rendering first, and in case of an error, will try to use a HTTP Proxy,
303 |      * and if that fails, it will fallback to Static File Serving
304 |      * 
305 |      * Note: "error" in this context can mean an actual exception, or "shouldUse" returning false
306 |      * @default [ProxyType.SsrProxy, ProxyType.HttpProxy, ProxyType.StaticProxy]
307 |      */
308 |     proxyOrder?: ProxyType[];
309 |     /**
310 |      * Custom implementation to define whether the client is a bot (e.g. Googlebot)
311 |      * 
312 |      * @default Defaults to 'https://www.npmjs.com/package/isbot'
313 |      */
314 |     isBot?: boolean | ((method: string, url: string, headers: HttpHeaders) => boolean);
315 |     /**
316 |      * Which HTTP response status code to return in case of an error
317 |      * @default 404
318 |      */
319 |     failStatus?: number | ((params: ProxyTypeParams) => number);
320 |     /**
321 |      * Custom error message handler
322 |      * @example err => err.toString()
323 |      * @default undefined
324 |      */
325 |     customError?: string | ((err: any) => string);
326 |     /**
327 |      * Skip to next proxy type on error
328 |      * @default true
329 |      */
330 |     skipOnError?: boolean;
331 |     /**
332 |      * Indicates whether to force exit with process.exit on shutdown
333 |      * @default true
334 |      */
335 |     forceExit?: boolean;
336 |     /**
337 |      * Function for processing the original request before proxying
338 |      * @default undefined
339 |      */
340 |     reqMiddleware?: (params: ProxyParams) => Promise<ProxyParams>;
341 |     /**
342 |      * Function for processing the proxy result before serving
343 |      * @default undefined
344 |      */
345 |     resMiddleware?: (params: ProxyParams, result: ProxyResult) => Promise<ProxyResult>;
346 |     /**
347 |      * Server-Side Rendering configuration
348 |      */
349 |     ssr?: SsrConfig & {
350 |         /**
351 |          * Indicates if the SSR Proxy should be used
352 |          * @default params => params.isBot && (/\.html$/.test(params.targetUrl.pathname) || !/\./.test(params.targetUrl.pathname))
353 |          */
354 |         shouldUse?: boolean | ((params: ProxyParams) => boolean);
355 |         /**
356 |          * Cron expression for closing the shared browser instance
357 |          * @default undefined
358 |          */
359 |         cleanUpCron?: string;
360 |         /**
361 |          * Tz for cleanUpCron
362 |          * @default 'Etc/UTC'
363 |          */
364 |         cleanUpTz?: string;
365 |     };
366 |     /**
367 |      * HTTP Proxy configuration
368 |      */
369 |     httpProxy?: {
370 |         /**
371 |          * Indicates if the HTTP Proxy should be used
372 |          * @default true
373 |          */
374 |         shouldUse?: boolean | ((params: ProxyParams) => boolean);
375 |         /**
376 |          * Which query string params to include in the url before proxying
377 |          * @example
378 |          * [{ key: 'headless', value: 'false' }]
379 |          * @default
380 |          * []
381 |          */
382 |         queryParams?: {
383 |             key: string;
384 |             value: string;
385 |         }[];
386 |         /**
387 |          * Ignore https errors via rejectUnauthorized=false
388 |          * @default false
389 |          */
390 |         unsafeHttps?: boolean | ((params: ProxyParams) => boolean);
391 |         /**
392 |          * Timeout
393 |          * @default 60000
394 |          */
395 |         timeout?: number;
396 |     };
397 |     /**
398 |      * Static File Serving configuration
399 |      */
400 |     static?: {
401 |         /**
402 |          * Indicates if the Static File Serving should be used
403 |          * @default false
404 |          */
405 |         shouldUse?: boolean | ((params: ProxyParams) => boolean);
406 |         /**
407 |          * Absolute path of the directory to serve
408 |          * @default 'public'
409 |          */
410 |         dirPath?: string;
411 |         /**
412 |          * Indicates whether to use the default index file
413 |          * @default path => path.endsWith('/')
414 |          */
415 |         useIndexFile?: (path: string) => boolean;
416 |         /**
417 |          * Default index file to use
418 |          * @default 'index.html'
419 |          */
420 |         indexFile?: string;
421 |     };
422 |     /**
423 |      * Logging configuration
424 |      */
425 |     log?: LogConfig;
426 |     /**
427 |      * Caching configuration
428 |      */
429 |     cache?: {
430 |         /**
431 |          * Indicates if the caching should be used
432 |          * @default params => params.proxyType === ProxyType.SsrProxy
433 |          */
434 |         shouldUse?: boolean | ((params: ProxyTypeParams) => boolean);
435 |         /**
436 |          * Defines the maximum number of pages to cache
437 |          * @default 50
438 |          */
439 |         maxEntries?: number;
440 |         /**
441 |          * Defines the maximum size of the cache in bytes
442 |          * @default 50 * 1000 * 1000 // 50MB
443 |          */
444 |         maxByteSize?: number;
445 |         /**
446 |          * Defines the expiration time for each cached page
447 |          * @default 25 * 60 * 60 * 1000 // 25h
448 |          */
449 |         expirationMs?: number;
450 |         /**
451 |          * Auto refreshing configuration
452 |          * 
453 |          * Auto refresh will access the configured pages periodically, and cache the result to be used on following access
454 |          */
455 |         autoRefresh?: SsrJob & {
456 |             /**
457 |              * Enable auto refreshing
458 |              * @default false
459 |              */
460 |             enabled?: boolean;
461 |             /**
462 |              * Indicates if the auto refresh should be used
463 |              * @default true
464 |              */
465 |             shouldUse?: boolean | (() => boolean);
466 |             /**
467 |              * Defines the order which the proxy service will follow in case of errors, similar to 'config.proxyOrder'
468 |              * @default [ProxyType.SsrProxy]
469 |              */
470 |             proxyOrder?: ProxyType[];
471 |             /**
472 |              * Whether to access routes as bot while auto refreshing
473 |              * @default true
474 |              */
475 |             isBot?: boolean;
476 |             /**
477 |              * Delay before first refresh
478 |              * @default 5 * 1000 // 5s
479 |              */
480 |             initTimeoutMs?: number;
481 |             /**
482 |              * Cron expression for interval between refreshes
483 |              * @default '0 0 3 * * *' // every day at 3am
484 |              */
485 |             intervalCron?: string;
486 |             /**
487 |              * Tz for intervalCron
488 |              * @default 'Etc/UTC'
489 |              */
490 |             intervalTz?: string;
491 |             /**
492 |              * Whether to close the shared browser instance after refreshing the cache
493 |              * @default true
494 |              */
495 |             closeBrowser?: boolean;
496 |         };
497 |     };
498 | }
499 | 
500 | // Proxy Cache
501 | 
502 | export interface CacheItem {
503 |     text: string;
504 |     contentType: string;
505 | }
506 | 
507 | export interface CacheDeletion {
508 |     key: string;
509 |     reason: string;
510 | }
511 | 
512 | export interface InternalCacheItem {
513 |     text: string;
514 |     status: number;
515 |     contentType: string;
516 |     hits: number;
517 |     date: Date;
518 | }
519 | 
520 | // Logger
521 | 
522 | export enum LogLevel {
523 |     None = 0,
524 |     Error = 1,
525 |     Info = 2,
526 |     Debug = 3
527 | }


--------------------------------------------------------------------------------
/src/ssr-proxy.ts:
--------------------------------------------------------------------------------
  1 | import axios from 'axios';
  2 | import cloneDeep from 'clone-deep';
  3 | import deepmerge from 'deepmerge';
  4 | import express from 'express';
  5 | import fs from 'fs';
  6 | import http from 'http';
  7 | import https from 'https';
  8 | import { isbot } from 'isbot';
  9 | import mime from 'mime-types';
 10 | import { scheduleJob } from 'node-schedule';
 11 | import os from 'os';
 12 | import path from 'path';
 13 | import { Stream } from 'stream';
 14 | import { Logger } from './logger';
 15 | import { ProxyCache } from './proxy-cache';
 16 | import { SsrRender } from './ssr-render';
 17 | import { CacheItem, LogLevel, HttpHeaders, ProxyParams, ProxyResult, ProxyType, ProxyTypeParams, SsrProxyConfig } from './types';
 18 | import { getOrCall, promiseParallel, promiseRetry, streamToString } from './utils';
 19 | 
 20 | export class SsrProxy extends SsrRender {
 21 |     private config: SsrProxyConfig;
 22 |     private proxyCache?: ProxyCache; // In-memory cache of rendered pages
 23 | 
 24 |     constructor(customConfig: SsrProxyConfig) {
 25 |         const defaultConfig: SsrProxyConfig = {
 26 |             // TODO: AllowRedirect: boolean, return without redirecting
 27 |             httpPort: 8080,
 28 |             httpsPort: 8443,
 29 |             httpsKey: undefined,
 30 |             httpsCert: undefined,
 31 |             hostname: '0.0.0.0',
 32 |             targetRoute: 'http://localhost:80',
 33 |             proxyOrder: [ProxyType.SsrProxy, ProxyType.HttpProxy, ProxyType.StaticProxy],
 34 |             isBot: (method, url, headers) => headers?.['user-agent'] ? isbot(headers['user-agent']) : false,
 35 |             failStatus: 404,
 36 |             customError: undefined,
 37 |             skipOnError: true,
 38 |             forceExit: true,
 39 |             reqMiddleware: undefined,
 40 |             resMiddleware: undefined,
 41 |             ssr: {
 42 |                 shouldUse: params => params.isBot && (/\.html$/.test(params.targetUrl.pathname) || !/\./.test(params.targetUrl.pathname)),
 43 |                 browserConfig: { headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'], timeout: 60000 },
 44 |                 sharedBrowser: true,
 45 |                 queryParams: [{ key: 'headless', value: 'true' }],
 46 |                 allowedResources: ['document', 'script', 'xhr', 'fetch'],
 47 |                 waitUntil: 'networkidle0',
 48 |                 timeout: 60000,
 49 |                 sleep: undefined,
 50 |                 cleanUpCron: undefined,
 51 |                 cleanUpTz: 'Etc/UTC',
 52 |                 pageMiddleware: undefined,
 53 |             },
 54 |             httpProxy: {
 55 |                 shouldUse: true,
 56 |                 queryParams: [],
 57 |                 unsafeHttps: false,
 58 |                 timeout: 60000,
 59 |             },
 60 |             static: {
 61 |                 shouldUse: false,
 62 |                 dirPath: 'public',
 63 |                 useIndexFile: path => path.endsWith('/'),
 64 |                 indexFile: 'index.html',
 65 |             },
 66 |             log: {
 67 |                 level: LogLevel.Info,
 68 |                 console: {
 69 |                     enabled: true,
 70 |                 },
 71 |                 file: {
 72 |                     enabled: true,
 73 |                     dirPath: path.join(os.tmpdir(), 'ssr-proxy-js/logs'),
 74 |                 },
 75 |             },
 76 |             cache: {
 77 |                 shouldUse: params => params.proxyType === ProxyType.SsrProxy,
 78 |                 maxEntries: 50,
 79 |                 maxByteSize: 50 * 1000 * 1000, // 50MB
 80 |                 expirationMs: 25 * 60 * 60 * 1000, // 25h
 81 |                 autoRefresh: {
 82 |                     enabled: false,
 83 |                     shouldUse: true,
 84 |                     proxyOrder: [ProxyType.SsrProxy],
 85 |                     initTimeoutMs: 5 * 1000, // 5s
 86 |                     intervalCron: '0 0 3 * * *', // every day at 3am
 87 |                     intervalTz: 'Etc/UTC',
 88 |                     retries: 3,
 89 |                     parallelism: 5,
 90 |                     closeBrowser: true,
 91 |                     isBot: true,
 92 |                     routes: [{ method: 'GET', url: '/' }],
 93 |                 },
 94 |             },
 95 |         };
 96 | 
 97 |         let config: SsrProxyConfig;
 98 | 
 99 |         if (customConfig) {
100 |             config = deepmerge<SsrProxyConfig>(defaultConfig, customConfig, {
101 |                 arrayMerge: (destArray, srcArray, opts) => srcArray,
102 |             });
103 |         } else {
104 |             console.warn('No configuration found for ssr-proxy-js, using default config!');
105 |             config = defaultConfig;
106 |         }
107 | 
108 |         if (config.static) config.static.dirPath = path.isAbsolute(config.static.dirPath!) ? config.static.dirPath! : path.join(process.cwd(), config.static.dirPath!);
109 |         if (config.cache!.autoRefresh!.parallelism! < 1) throw new Error(`Parallelism should be greater than 0 (${config.cache!.autoRefresh!.parallelism})`);
110 | 
111 |         super(config.ssr!);
112 |         this.config = config;
113 | 
114 |         const cLog = this.config.log;
115 |         Logger.setLevel(cLog!.level!);
116 |         Logger.configConsole(cLog!.console!.enabled!);
117 |         Logger.configFile(cLog!.file!.enabled!, cLog!.file!.dirPath!);
118 | 
119 |         const cCache = this.config.cache;
120 |         this.proxyCache = new ProxyCache(cCache!.maxEntries!, cCache!.maxByteSize!, cCache!.expirationMs!);
121 |     }
122 | 
123 |     start() {
124 |         this.startCleanUpJob();
125 |         this.startCacheJob();
126 | 
127 |         const { server } = this.listen();
128 |         
129 |         const shutDown = async () => {
130 |             Logger.info('Shutting down...');
131 | 
132 |             await this.browserShutDown();
133 | 
134 |             Logger.info('Closing the server...');
135 |             server.close(() => {
136 |                 Logger.info('Shut down completed!');
137 |                 if (this.config.forceExit) process.exit(0);
138 |             });
139 | 
140 |             if (this.config.forceExit) {
141 |                 setTimeout(() => {
142 |                     Logger.error(`Shutdown`, 'Could not shut down in time, forcefully shutting down!');
143 |                     process.exit(1);
144 |                 }, 10000);
145 |             }
146 |         };
147 |         process.on('SIGTERM', shutDown);
148 |         process.on('SIGINT', shutDown);
149 |     }
150 | 
151 |     private async startCleanUpJob() {
152 |         const cleanUpCron = this.config.ssr!.cleanUpCron;
153 |         const cleanUpTz = this.config.ssr!.cleanUpTz;
154 |         if (!cleanUpCron) return;
155 |         scheduleJob({ rule: cleanUpCron, tz: cleanUpTz }, async () => {
156 |             this.sharedBrowser?.close();
157 |         });
158 |     }
159 | 
160 |     private startCacheJob() {
161 |         const $this = this;
162 |         const cCache = this.config.cache!;
163 |         const cAutoCache = cCache.autoRefresh!;
164 | 
165 |         const enabled = cAutoCache.enabled! && cAutoCache.routes! && cAutoCache.routes!.length!;
166 |         if (!enabled) return;
167 | 
168 |         if (cAutoCache.initTimeoutMs)
169 |             setTimeout(runRefresh, cAutoCache.initTimeoutMs);
170 | 
171 |         if (cAutoCache.intervalCron)
172 |             scheduleJob({ rule: cAutoCache.intervalCron, tz: cAutoCache.intervalTz }, runRefresh);
173 | 
174 |         async function runRefresh() {
175 |             const logger = new Logger(true);
176 | 
177 |             try {
178 |                 if (!cAutoCache.shouldUse || !getOrCall(cAutoCache.shouldUse) || !cAutoCache.routes?.length) return;
179 | 
180 |                 const routesStr = '> ' + cAutoCache.routes!.map(e => e.url).join('\n> ');
181 |                 logger.info(`Refreshing Cache:\n${routesStr}`);
182 | 
183 |                 await promiseParallel(cAutoCache.routes!.map((route) => () => new Promise(async (res, rej) => {
184 |                     try {
185 |                         await promiseRetry(runProxy, cAutoCache.retries!, e => logger.warn('CacheRefresh Retry', e));
186 |                         res('ok');
187 |                     } catch (err) {
188 |                         logger.error('CacheRefresh', err);
189 |                         rej(err);
190 |                     }
191 | 
192 |                     async function runProxy() {
193 |                         const targetUrl = new URL(route.url, $this.config.targetRoute!);
194 |                         const params: ProxyParams = { isBot: cAutoCache.isBot!, cacheBypass: true, sourceUrl: route.url, targetUrl, method: route.method, headers: route.headers || {} };
195 |                         const { proxyType, result } = await $this.runProxy(params, cAutoCache.proxyOrder!, logger);
196 |                     }
197 |                 })), cAutoCache.parallelism!, true);
198 | 
199 |                 logger.info(`Cache Refreshed!`);
200 |             } catch (err) {
201 |                 logger.error('CacheRefresh', err);
202 |             } finally {
203 |                 if (cAutoCache.closeBrowser) $this.sharedBrowser?.close();
204 |             }
205 |         }
206 |     }
207 | 
208 |     private listen() {
209 |         const $this = this;
210 | 
211 |         const app = express();
212 | 
213 |         // Proxy requests
214 |         app.use('*', async (req, res, next) => {
215 |             const logger = new Logger();
216 | 
217 |             try {
218 |                 const sourceUrl = req.originalUrl;
219 |                 const targetUrl = new URL(req.originalUrl, this.config.targetRoute);
220 |                 const method = req.method;
221 |                 const headers = this.fixHeaders(req.headers);
222 | 
223 |                 const isBot = getOrCall(this.config.isBot, method, sourceUrl, headers)!;
224 | 
225 |                 logger.info(`[${method}] ${sourceUrl} | IsBot: ${isBot} | User Agent: ${headers['user-agent']}`);
226 | 
227 |                 const params: ProxyParams = { isBot, cacheBypass: false, method, sourceUrl, headers, targetUrl };
228 | 
229 |                 const { proxyType, result } = await this.runProxy(params, this.config.proxyOrder!, logger);
230 | 
231 |                 const proxyTypeParams: ProxyTypeParams = { ...params, proxyType };
232 | 
233 |                 // if (proxyType === ProxyType.Redirect) return sendRedirect(result, proxyTypeParams);
234 |                 if (result?.error != null) return sendFail(result, proxyTypeParams);
235 |                 else if (result?.text != null) return sendText(result, proxyTypeParams);
236 |                 else if (result?.stream != null) return sendStream(result, proxyTypeParams);
237 |                 else return sendFail({ ...result, error: 'No Proxy Result' }, proxyTypeParams);
238 |             } catch (err) {
239 |                 return next(err);
240 |             }
241 | 
242 |             async function sendText(result: ProxyResult, params: ProxyTypeParams) {
243 |                 res.status(result.status || 200);
244 |                 res.contentType(result.contentType!);
245 |                 setHeaders(result.headers!)
246 |                 return res.send(result.text!);
247 |             }
248 | 
249 |             async function sendStream(result: ProxyResult, params: ProxyTypeParams) {
250 |                 res.status(result.status || 200);
251 |                 res.contentType(result.contentType!);
252 |                 setHeaders(result.headers!)
253 |                 return result.stream!.on('error', err => {
254 |                     res.status(getOrCall($this.config.failStatus, params)!);
255 |                     // res.contentType('text/plain');
256 |                     // const error = Logger.errorStr(result.error!);
257 |                     return res.send();
258 |                 }).pipe(res);
259 |             }
260 | 
261 |             // async function sendRedirect(result: ProxyResult, params: ProxyTypeParams) {
262 |             //     res.status(result.status || 302);
263 |             //     setHeaders(result.headers!)
264 |             //     return res.redirect(result.text!);
265 |             // }
266 | 
267 |             async function sendFail(result: ProxyResult, params: ProxyTypeParams) {
268 |                 res.status(getOrCall($this.config.failStatus, params)!);
269 |                 res.contentType('text/plain');
270 |                 setHeaders(result.headers!)
271 |                 const errMsg = getOrCall($this.config.customError, result.error!) ?? Logger.errorStr(result.error!);
272 |                 return res.send(errMsg);
273 |             }
274 | 
275 |             function setHeaders(headers: HttpHeaders) {
276 |                 for (let key in headers) {
277 |                     try {
278 |                         res.set(key, headers[key]);
279 |                     } catch (err) {
280 |                         Logger.errorStr(`Invalid headers:\nKey: ${key}\nValue: ${headers[key]})`);
281 |                     }
282 |                 }
283 |             }
284 |         });
285 | 
286 |         // Error Handler
287 |         app.use((err: any, req: any, res: any, next: any) => {
288 |             Logger.error('Error', err, true);
289 |             res.contentType('text/plain');
290 |             res.status(err.status || 500);
291 |             const errMsg = getOrCall(this.config.customError, err) ?? Logger.errorStr(err);
292 |             res.send(errMsg);
293 |             next();
294 |         });
295 | 
296 |         let server: http.Server;
297 | 
298 |         if (this.config.httpPort) {
299 |             // HTTP Listen
300 |             server = app.listen(this.config.httpPort, this.config.hostname!, () => {
301 |                 Logger.info('----- Starting HTTP SSR Proxy -----');
302 |                 Logger.info(`Listening on http://${this.config.hostname!}:${this.config.httpPort!}`);
303 |                 Logger.info(`Proxy: ${this.config.targetRoute!}`);
304 |                 Logger.info(`DirPath: ${this.config.static!.dirPath!}`);
305 |                 Logger.info(`ProxyOrder: ${this.config.proxyOrder!}\n`);
306 |             });
307 |         } else if (this.config.httpsPort && this.config.httpsKey && this.config.httpsCert) {
308 |             // HTTPS Listen
309 |             server = https.createServer({
310 |                 key: fs.readFileSync(this.config.httpsKey),
311 |                 cert: fs.readFileSync(this.config.httpsCert),
312 |             }, app);
313 |             server.listen(this.config.httpsPort, this.config.hostname!, () => {
314 |                 Logger.info('\n----- Starting HTTPS SSR Proxy -----');
315 |                 Logger.info(`Listening on https://${this.config.hostname!}:${this.config.httpsPort!}`);
316 |                 Logger.info(`Proxy: ${this.config.targetRoute!}`);
317 |                 Logger.info(`DirPath: ${this.config.static!.dirPath!}`);
318 |                 Logger.info(`ProxyOrder: ${this.config.proxyOrder!}\n`);
319 |             });
320 |         } else {
321 |             throw new Error('Invalid Ports or Certificates');
322 |         }
323 | 
324 |         return { app, server };
325 |     }
326 | 
327 |     private async runProxy(params: ProxyParams, proxyOrder: ProxyType[], logger: Logger) {
328 |         if (!proxyOrder.length) throw new Error('Invalid Proxy Order');
329 | 
330 |         params.headers ||= {};
331 |         params.method ||= 'GET';
332 | 
333 |         let result: ProxyResult = {};
334 |         let proxyType: ProxyType = proxyOrder[0];
335 | 
336 |         for (let i in proxyOrder) {
337 |             proxyType = proxyOrder[i];
338 | 
339 |             const proxyParams = cloneDeep(this.config.reqMiddleware != null ? await this.config.reqMiddleware(params) : params);
340 | 
341 |             try {
342 |                 // const redirect = await this.checkForRedirect(proxyParams, logger);
343 |                 // if (redirect.status) return { result: redirect, proxyType: ProxyType.Redirect };
344 | 
345 |                 if (proxyType === ProxyType.SsrProxy) {
346 |                     result = await this.runSsrProxy(proxyParams, logger);
347 |                 } else if (proxyType === ProxyType.HttpProxy) {
348 |                     result = await this.runHttpProxy(proxyParams, logger);
349 |                 } else if (proxyType === ProxyType.StaticProxy) {
350 |                     result = await this.runStaticProxy(proxyParams, logger);
351 |                 } else {
352 |                     throw new Error('Invalid Proxy Type');
353 |                 }
354 |             } catch (err) {
355 |                 result = { error: err };
356 |                 params.lastError = err;
357 |             }
358 | 
359 |             // Success
360 |             if (!result.skipped && result.error == null) break;
361 | 
362 |             // Bubble up errors
363 |             if (!this.config.skipOnError && result.error != null)
364 |                 throw (typeof result.error === 'string' ? new Error(result.error) : result.error);
365 |         }
366 | 
367 |         if (this.config.resMiddleware != null) result = await this.config.resMiddleware(params, result);
368 | 
369 |         return { proxyType, result };
370 |     }
371 | 
372 |     private async runSsrProxy(params: ProxyParams, logger: Logger): Promise<ProxyResult> {
373 |         const cSsr = this.config.ssr!;
374 |         const cacheKey = `${ProxyType.SsrProxy}:${params.targetUrl}`;
375 |         const typeParams = { ...params, proxyType: ProxyType.SsrProxy };
376 | 
377 |         const shouldUse = getOrCall(cSsr.shouldUse, params)!;
378 |         if (!shouldUse) {
379 |             logger.debug(`Skipped SsrProxy: ${params.targetUrl}`);
380 |             return { skipped: true };
381 |         }
382 | 
383 |         try {
384 |             logger.info(`Using SsrProxy: ${params.targetUrl}`);
385 | 
386 |             // Try use Cache
387 | 
388 |             const cache = !params.cacheBypass && this.tryGetCache(cacheKey, typeParams, logger);
389 |             if (cache) {
390 |                 logger.info(`SSR Cache Hit`);
391 |                 return { text: cache.text, contentType: cache.contentType };
392 |             }
393 | 
394 |             // Try use SsrProxy
395 | 
396 |             let { status, text, error, headers: ssrHeaders, ttRenderMs } = await this.tryRender(params.targetUrl.toString(), params.headers, logger, params.method);
397 | 
398 |             status ||= 200;
399 |             const isSuccess = error == null;
400 | 
401 |             logger.info(`SSR Result | Render Time: ${ttRenderMs}ms | Success: ${isSuccess}${isSuccess ? '' : ` | Message: ${error}`}`);
402 | 
403 |             if (!isSuccess) return { error };
404 |             if (text == null) text = '';
405 | 
406 |             const resHeaders = {
407 |                 ...(ssrHeaders || {}),
408 |                 'Server-Timing': `Prerender;dur=${ttRenderMs};desc="Headless render time (ms)"`,
409 |             };
410 |             
411 |             const contentType = this.getContentType(params.targetUrl.pathname);
412 | 
413 |             this.trySaveCache(text, status, contentType, cacheKey, typeParams, logger);
414 | 
415 |             return { text, status, contentType, headers: resHeaders };
416 |         } catch (err: any) {
417 |             logger.error('SsrError', err);
418 |             return { error: err };
419 |         }
420 |     }
421 | 
422 |     private async runHttpProxy(params: ProxyParams, logger: Logger): Promise<ProxyResult> {
423 |         const cHttpProxy = this.config.httpProxy!;
424 |         const cacheKey = `${ProxyType.HttpProxy}:${params.method}:${params.targetUrl}`;
425 |         const typeParams = { ...params, proxyType: ProxyType.HttpProxy };
426 | 
427 |         const shouldUse = getOrCall(cHttpProxy.shouldUse, params)!;
428 |         if (!shouldUse) {
429 |             logger.debug(`Skipped HttpProxy: ${params.targetUrl}`);
430 |             return { skipped: true };
431 |         }
432 | 
433 |         try {
434 |             logger.debug(`Using HttpProxy: ${params.targetUrl}`);
435 | 
436 |             // Try use Cache
437 | 
438 |             const cache = !params.cacheBypass && this.tryGetCache(cacheKey, typeParams, logger);
439 |             if (cache) {
440 |                 logger.info(`HTTP Cache Hit`);
441 |                 return { text: cache.text, contentType: cache.contentType };
442 |             }
443 | 
444 |             // Try use HttpProxy
445 | 
446 |             // Indicate http proxy to client
447 |             for (let param of cHttpProxy.queryParams!)
448 |                 params.targetUrl.searchParams.set(param.key, param.value);
449 | 
450 |             const reqHeaders = this.fixReqHeaders(params.headers);
451 | 
452 |             logger.debug(`HttpProxy: Connecting - ${JSON.stringify(reqHeaders)}`);
453 | 
454 |             const response = await axios.request({
455 |                 url: params.targetUrl.toString(),
456 |                 method: params.method as any,
457 |                 responseType: 'stream',
458 |                 headers: reqHeaders,
459 |                 httpsAgent: new https.Agent({ rejectUnauthorized: getOrCall(cHttpProxy.unsafeHttps, params) }),
460 |                 timeout: cHttpProxy.timeout,
461 |             });
462 | 
463 |             const status = response.status;
464 | 
465 |             const resHeaders = this.fixResHeaders(response.headers);
466 | 
467 |             logger.debug(`HttpProxy: Connected - ${JSON.stringify(resHeaders)}`);
468 | 
469 |             const contentType = this.getContentType(params.targetUrl.pathname);
470 | 
471 |             this.trySaveCacheStream(response.data, status, contentType, cacheKey, typeParams, logger);
472 | 
473 |             return { status: response.status, stream: response.data, headers: resHeaders, contentType };
474 |         } catch (err: any) {
475 |             const error = err?.response?.data ? await streamToString(err.response.data).catch(err => err) : err;
476 |             logger.error('HttpProxyError', error);
477 |             return { error };
478 |         }
479 |     }
480 | 
481 |     private async runStaticProxy(params: ProxyParams, logger: Logger): Promise<ProxyResult> {
482 |         const cStatic = this.config.static!;
483 |         const cacheKey = `${ProxyType.StaticProxy}:${params.targetUrl}`;
484 |         const typeParams = { ...params, proxyType: ProxyType.StaticProxy };
485 | 
486 |         const shouldUse = getOrCall(cStatic.shouldUse, params)!;
487 |         if (!shouldUse) {
488 |             logger.debug(`Skipped StaticProxy: ${params.targetUrl}`);
489 |             return { skipped: true };
490 |         }
491 | 
492 |         try {
493 |             logger.debug(`Using StaticProxy: ${params.targetUrl}`);
494 | 
495 |             // Try use Cache
496 | 
497 |             const cache = !params.cacheBypass && this.tryGetCache(cacheKey, typeParams, logger);
498 |             if (cache) {
499 |                 logger.info(`Static Cache Hit`);
500 |                 return { text: cache.text, contentType: cache.contentType };
501 |             }
502 | 
503 |             // Try use StaticProxy
504 | 
505 |             if (cStatic.useIndexFile!(params.sourceUrl))
506 |                 params.sourceUrl = `${params.sourceUrl}/${cStatic.indexFile!}`.replace(/\/\//g, '/');
507 | 
508 |             const filePath = path.join(cStatic.dirPath!, params.sourceUrl);
509 | 
510 |             logger.debug(`Static Path: ${filePath}`);
511 | 
512 |             if (!fs.existsSync(filePath))
513 |                 throw new Error(`File Not Found: ${filePath}`);
514 | 
515 |             const fileStream = fs.createReadStream(filePath);
516 |             
517 |             const contentType = this.getContentType(filePath);
518 | 
519 |             this.trySaveCacheStream(fileStream, 200, contentType, cacheKey, typeParams, logger);
520 | 
521 |             return { stream: fileStream, status: 200, contentType };
522 |         } catch (err: any) {
523 |             logger.error('StaticError', err);
524 |             return { error: err };
525 |         }
526 |     }
527 | 
528 |     // private async checkForRedirect(proxyParams: ProxyParams, logger: Logger): Promise<ProxyResult> {
529 |     //     // TODO:
530 |     //     // cache the redirect
531 |     //     // fix targetUrl: Target (http://web-server:8080/)
532 | 
533 |     //     try {
534 |     //         const targetUrl = proxyParams.targetUrl.toString();
535 | 
536 |     //         logger.debug(`Redirect: Checking (${targetUrl})`);
537 | 
538 |     //         // Use axios with a short timeout and redirect: false
539 |     //         const response = await axios.request({
540 |     //             url: targetUrl,
541 |     //             method: 'HEAD', // HEAD request is faster than GET
542 |     //             headers: this.fixReqHeaders(proxyParams.headers),
543 |     //             maxRedirects: 0, // Don't follow redirects
544 |     //             validateStatus: (status) => status < 400 || status === 404, // Accept any status that isn't an error
545 |     //             timeout: 5000 // 5 second timeout
546 |     //         });
547 |     
548 |     //         // Check if this is a redirect status
549 |     //         if (response.status === 301 || response.status === 302 || response.status === 307 || response.status === 308) {
550 |     //             logger.info(`Redirect: Detected (${targetUrl} - ${response.status})`);
551 |     //             const location = response.headers['location'];
552 | 
553 |     //             if (location) {
554 |     //                 const redirectUrl = new URL(location, targetUrl).toString();
555 |     //                 logger.info(`Redirect: Target (${redirectUrl})`);
556 | 
557 |     //                 return {
558 |     //                     text: redirectUrl,
559 |     //                     status: response.status,
560 |     //                     headers: this.fixResHeaders(response.headers),
561 |     //                 };
562 |     //             }
563 |     //         }
564 |             
565 |     //         return {};
566 |     //     } catch (err: any) {
567 |     //         return { error: err };
568 |     //     }
569 |     // }
570 | 
571 |     private getContentType(path: string) {
572 |         const isHtml = () => /\.html$/.test(path) || !/\./.test(path)
573 |         const type = mime.lookup(path) || (isHtml() ? 'text/html' : 'text/plain');
574 |         return type;
575 |     }
576 | 
577 |     // Cache
578 | 
579 |     private tryGetCache(cacheKey: string, params: ProxyTypeParams, logger: Logger): CacheItem | null {
580 |         const cCache = this.config.cache!;
581 | 
582 |         const shouldUse = getOrCall(cCache.shouldUse, params)! && this.proxyCache?.has(cacheKey);
583 |         if (shouldUse) {
584 |             logger.debug(`Cache Hit: ${cacheKey}`);
585 |             const cache = this.proxyCache!.get(cacheKey)!;
586 | 
587 |             if (!cache) return null;
588 | 
589 |             return cache;
590 |         }
591 | 
592 |         return null;
593 |     }
594 | 
595 |     private trySaveCache(text: string, status: number, contentType: string, cacheKey: string, params: ProxyTypeParams, logger: Logger) {
596 |         const cCache = this.config.cache!;
597 | 
598 |         const shouldUse = getOrCall(cCache.shouldUse, params)! && this.proxyCache!;
599 |         if (shouldUse) {
600 |             logger.debug(`Caching: ${cacheKey}`);
601 |             this.proxyCache!.set(cacheKey, text, status, contentType);
602 |             this.tryClearCache(logger);
603 |         }
604 |     }
605 | 
606 |     private trySaveCacheStream(stream: Stream, status: number, contentType: string, cacheKey: string, params: ProxyTypeParams, logger: Logger) {
607 |         const cCache = this.config.cache!;
608 | 
609 |         const shouldUse = getOrCall(cCache.shouldUse, params)! && this.proxyCache!;
610 |         if (shouldUse) {
611 |             logger.debug(`Caching: ${cacheKey}`);
612 |             this.proxyCache!.pipe(cacheKey, stream, status, contentType)
613 |                 .then(() => this.tryClearCache(logger))
614 |                 .catch(err => logger.error('SaveCacheStream', err));
615 |         }
616 |     }
617 | 
618 |     private tryClearCache(logger: Logger) {
619 |         if (this.proxyCache!) {
620 |             const deleted = this.proxyCache.tryClear();
621 |             if (deleted.length) logger.debug(`Cache Cleared: ${JSON.stringify(deleted)}`);
622 |         }
623 |     }
624 | }
625 | 


--------------------------------------------------------------------------------