├── test ├── fixtures │ └── gif.gif ├── collapsers │ ├── binary.js │ ├── javascript.js │ ├── css.js │ └── html.js ├── helpers.js ├── plugins │ ├── html-flatten-image.js │ ├── postcss-flatten-import.js │ ├── postcss-flatten-url.js │ ├── html-flatten-script.js │ └── html-flatten-style.js └── utils │ ├── data-uri.js │ └── fetch-wrapper.js ├── .gitignore ├── src ├── customTypings │ ├── cliclopts.d.ts │ └── bole.d.ts ├── version.ts ├── utils │ ├── data-uri.ts │ ├── css-url.ts │ ├── html-rewriter.ts │ ├── httpclient.ts │ └── fetch-wrapper.ts ├── collapsers │ ├── html.ts │ ├── binary.ts │ ├── javascript.ts │ └── css.ts ├── plugins │ ├── html-flatten-image.ts │ ├── html-flatten-inline-style.ts │ ├── html-flatten-external-style.ts │ ├── postcss-flatten-import.ts │ ├── html-flatten-script.ts │ └── postcss-flatten-url.ts ├── simple.ts ├── collapsify.ts └── bin │ └── cli.ts ├── tsconfig.json ├── LICENSE ├── README.md └── package.json /test/fixtures/gif.gif: -------------------------------------------------------------------------------- 1 | GIF89a; -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | /coverage 3 | /tmp 4 | /artifacts 5 | /built 6 | /types 7 | -------------------------------------------------------------------------------- /src/customTypings/cliclopts.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'cliclopts' { 2 | export type Argument = { 3 | name: string; 4 | abbr: string; 5 | default?: any; 6 | help?: string; 7 | boolean?: boolean; 8 | }; 9 | 10 | function cliclopts(args: Argument[]): any; 11 | 12 | export default cliclopts; 13 | } 14 | -------------------------------------------------------------------------------- /src/version.ts: -------------------------------------------------------------------------------- 1 | import {readFileSync} from 'node:fs'; 2 | 3 | type PackageJson = { 4 | version: string; 5 | }; 6 | 7 | const contents = readFileSync(new URL('../package.json', import.meta.url), { 8 | encoding: 'utf8', 9 | }); 10 | const packageJson: PackageJson = JSON.parse(contents) as PackageJson; 11 | 12 | export default packageJson.version; 13 | -------------------------------------------------------------------------------- /src/utils/data-uri.ts: -------------------------------------------------------------------------------- 1 | import base64 from 'base64-js'; 2 | 3 | export function encodeSync(arrayData: Uint8Array, {contentType = ''}): string { 4 | contentType = contentType.replace(/\s+/g, ''); 5 | return `data:${contentType};base64,${base64.fromByteArray(arrayData)}`; 6 | } 7 | 8 | export function validateSync(url: string): boolean { 9 | return url.startsWith('data:'); 10 | } 11 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "noImplicitAny": true, 4 | "outDir": "./built", 5 | "allowJs": false, 6 | "module": "ES2022", 7 | "moduleResolution": "node", 8 | "target": "ES2020", 9 | "declaration": true, 10 | "strictNullChecks": true, 11 | "allowSyntheticDefaultImports": true 12 | }, 13 | "include": ["./src/**/*"], 14 | "exclude": ["node_modules/*"] 15 | } 16 | -------------------------------------------------------------------------------- /src/collapsers/html.ts: -------------------------------------------------------------------------------- 1 | import {rewriteHtml} from '../utils/html-rewriter.js'; 2 | import {type CollapsifyOptions} from '../collapsify.js'; 3 | 4 | export async function external(options: CollapsifyOptions) { 5 | const response = await options.fetch(options.resourceLocation); 6 | return collapse(await response.getAsString(), options); 7 | } 8 | 9 | async function collapse(text: string, options: CollapsifyOptions) { 10 | return rewriteHtml(text, options); 11 | } 12 | 13 | export default collapse; 14 | -------------------------------------------------------------------------------- /src/customTypings/bole.d.ts: -------------------------------------------------------------------------------- 1 | declare module 'bole' { 2 | function logger(name: string): typeof logger; 3 | namespace logger { 4 | function debug(...args: any[]): void; 5 | function info(...args: any[]): void; 6 | function warn(...args: any[]): void; 7 | function error(...args: any[]): void; 8 | } 9 | 10 | function bole(name: string): typeof logger; 11 | namespace bole { 12 | function output(args: {level: string; stream: any}): void; 13 | } 14 | 15 | export default bole; 16 | } 17 | -------------------------------------------------------------------------------- /src/collapsers/binary.ts: -------------------------------------------------------------------------------- 1 | import {type CollapsifyOptions} from '../collapsify.js'; 2 | import {encodeSync} from '../utils/data-uri.js'; 3 | 4 | async function external({fetch, resourceLocation: url}: CollapsifyOptions) { 5 | const response = await fetch(url); 6 | const contentType = response.getContentType(); 7 | return collapse(await response.getAsArray(), {contentType}); 8 | } 9 | 10 | async function collapse(bodyArray: Uint8Array, options: {contentType: string}) { 11 | return encodeSync(bodyArray, options); 12 | } 13 | 14 | collapse.external = external; 15 | 16 | export default collapse; 17 | -------------------------------------------------------------------------------- /src/utils/css-url.ts: -------------------------------------------------------------------------------- 1 | import type {Node} from 'postcss-value-parser'; 2 | import {validateSync} from './data-uri.js'; 3 | 4 | export default function cssUrl(node: Node, skipCheck: boolean) { 5 | if (node.type === 'function' && node.value === 'url') { 6 | node = node.nodes[0]; 7 | skipCheck = true; 8 | } 9 | 10 | if (node.type !== 'word' && node.type !== 'string') { 11 | return; 12 | } 13 | 14 | const url = node.value; 15 | if (!skipCheck && !/^https?:/.test(url)) { 16 | return; 17 | } 18 | 19 | if (validateSync(url)) { 20 | return; 21 | } 22 | 23 | return url; 24 | } 25 | -------------------------------------------------------------------------------- /src/plugins/html-flatten-image.ts: -------------------------------------------------------------------------------- 1 | import {type HTMLRewriter} from 'html-rewriter-wasm'; 2 | import collapseBinary from '../collapsers/binary.js'; 3 | import {type CollapsifyOptions} from '../collapsify.js'; 4 | import {validateSync} from '../utils/data-uri.js'; 5 | 6 | export default function flattenImage( 7 | rewriter: HTMLRewriter, 8 | options: CollapsifyOptions, 9 | ) { 10 | rewriter.on('img', { 11 | async element(element) { 12 | const src = element.getAttribute('src'); 13 | 14 | if (!src || validateSync(src)) { 15 | return; 16 | } 17 | 18 | const newValue = await collapseBinary.external({ 19 | fetch: options.fetch, 20 | resourceLocation: new URL(src, options.resourceLocation).toString(), 21 | }); 22 | 23 | element.setAttribute('src', newValue); 24 | }, 25 | }); 26 | } 27 | -------------------------------------------------------------------------------- /src/collapsers/javascript.ts: -------------------------------------------------------------------------------- 1 | import {minify} from 'terser'; 2 | import bole from 'bole'; 3 | import {CollapsifyError, type CollapsifyOptions} from '../collapsify.js'; 4 | 5 | const logger = bole('collapsify:collapsers:javascript'); 6 | 7 | async function external(options: CollapsifyOptions) { 8 | const response = await options.fetch(options.resourceLocation); 9 | return collapse(await response.getAsString(), options); 10 | } 11 | 12 | async function collapse( 13 | bodyString: string, 14 | options: {resourceLocation: string}, 15 | ) { 16 | try { 17 | const result = await minify({ 18 | [options.resourceLocation]: bodyString, 19 | }); 20 | return result.code; 21 | } catch (error: unknown) { 22 | logger.error(error); 23 | return bodyString; 24 | } 25 | } 26 | 27 | collapse.external = external; 28 | 29 | export default collapse; 30 | -------------------------------------------------------------------------------- /src/simple.ts: -------------------------------------------------------------------------------- 1 | import httpClient from './utils/httpclient.js'; 2 | import collapseHTML, {CollapsifyError} from './collapsify.js'; 3 | 4 | export type SimpleOptions = { 5 | forbidden: string; 6 | headers?: Record; 7 | }; 8 | 9 | export async function simpleCollapsify( 10 | resourceLocation: string, 11 | options: SimpleOptions, 12 | ) { 13 | const fetch = httpClient(options?.headers); 14 | options = Object.assign( 15 | { 16 | forbidden: 'a^', 17 | }, 18 | options, 19 | ); 20 | 21 | async function read(url: string) { 22 | const re = new RegExp(options.forbidden, 'i'); 23 | 24 | if (re.test(url)) { 25 | throw new CollapsifyError('Forbidden resource ' + url); 26 | } 27 | 28 | return fetch(url); 29 | } 30 | 31 | return collapseHTML({ 32 | fetch: read, 33 | resourceLocation, 34 | }); 35 | } 36 | -------------------------------------------------------------------------------- /src/collapsify.ts: -------------------------------------------------------------------------------- 1 | import type {Buffer} from 'node:buffer'; 2 | import {external as htmlCollapser} from './collapsers/html.js'; 3 | import {fetchWrapper} from './utils/fetch-wrapper.js'; 4 | 5 | export class CollapsifyError extends Error {} 6 | 7 | export type Response = { 8 | getStatusCode(): number; 9 | 10 | getContentType(): string; 11 | 12 | getAsString(): Promise; 13 | 14 | getAsArray(): Promise; 15 | }; 16 | 17 | export type Fetch = (url: string) => Promise; 18 | 19 | export type CollapsifyOptions = { 20 | resourceLocation: string; 21 | fetch: Fetch; 22 | }; 23 | 24 | export default async function collapsify(options: CollapsifyOptions) { 25 | return htmlCollapser({ 26 | resourceLocation: options.resourceLocation, 27 | fetch: fetchWrapper(options.fetch), 28 | }); 29 | } 30 | 31 | export {type SimpleOptions, simpleCollapsify} from './simple.js'; 32 | -------------------------------------------------------------------------------- /src/plugins/html-flatten-inline-style.ts: -------------------------------------------------------------------------------- 1 | import {type HTMLRewriter} from 'html-rewriter-wasm'; 2 | import collapseCSS from '../collapsers/css.js'; 3 | import {type CollapsifyOptions} from '../collapsify.js'; 4 | 5 | export default function flattenInlineStyle( 6 | rewriter: HTMLRewriter, 7 | options: CollapsifyOptions, 8 | ) { 9 | let currentText = ''; 10 | rewriter.on('style', { 11 | async text(text) { 12 | if (!text.lastInTextNode) { 13 | currentText += text.text; 14 | text.remove(); 15 | return; 16 | } 17 | 18 | const content = await collapseCSS(currentText, { 19 | imported: false, 20 | fetch: options.fetch, 21 | resourceLocation: options.resourceLocation, 22 | }); 23 | 24 | currentText = ''; 25 | 26 | if (typeof content !== 'string') { 27 | throw new TypeError('Wrong output from collapseCSS'); 28 | } 29 | 30 | text.replace(content, {html: true}); 31 | }, 32 | }); 33 | } 34 | -------------------------------------------------------------------------------- /src/utils/html-rewriter.ts: -------------------------------------------------------------------------------- 1 | import {HTMLRewriter} from 'html-rewriter-wasm'; 2 | import {type CollapsifyOptions} from '../collapsify.js'; 3 | import flattenExternalStyle from '../plugins/html-flatten-external-style.js'; 4 | import flattenImage from '../plugins/html-flatten-image.js'; 5 | import flattenInlineStyle from '../plugins/html-flatten-inline-style.js'; 6 | import flattenScript from '../plugins/html-flatten-script.js'; 7 | 8 | export async function rewriteHtml(html: string, options: CollapsifyOptions) { 9 | const encoder = new TextEncoder(); 10 | const decoder = new TextDecoder(); 11 | 12 | let output = ''; 13 | const rewriter = new HTMLRewriter((chunk) => { 14 | output += decoder.decode(chunk); 15 | }); 16 | 17 | flattenExternalStyle(rewriter, options); 18 | flattenInlineStyle(rewriter, options); 19 | flattenImage(rewriter, options); 20 | flattenScript(rewriter, options); 21 | 22 | await rewriter.write(encoder.encode(html)); 23 | await rewriter.end(); 24 | 25 | return output; 26 | } 27 | -------------------------------------------------------------------------------- /test/collapsers/binary.js: -------------------------------------------------------------------------------- 1 | import assert from 'power-assert'; 2 | import {describe, it} from 'mocha'; 3 | import {gifResponse, gifData} from '../helpers.js'; 4 | import collapser from '../../built/collapsers/binary.js'; 5 | 6 | describe('binary collapser', () => { 7 | it('should collapse a GIF', async () => { 8 | const encoded = await collapser(await gifData(), { 9 | contentType: 'image/gif', 10 | }); 11 | 12 | assert(typeof encoded === 'string'); 13 | assert(encoded.startsWith('data:image/gif;base64,')); 14 | }); 15 | 16 | describe('external', () => { 17 | it('should collapse an external binary', async () => { 18 | const encoded = await collapser.external({ 19 | async fetch(url) { 20 | assert(url === 'https://example.com/gif.gif'); 21 | return gifResponse(); 22 | }, 23 | resourceLocation: 'https://example.com/gif.gif', 24 | }); 25 | 26 | assert(typeof encoded === 'string'); 27 | assert(encoded.startsWith('data:image/gif;base64,')); 28 | }); 29 | }); 30 | }); 31 | -------------------------------------------------------------------------------- /src/plugins/html-flatten-external-style.ts: -------------------------------------------------------------------------------- 1 | import {type HTMLRewriter} from 'html-rewriter-wasm'; 2 | import collapseCSS from '../collapsers/css.js'; 3 | import {type CollapsifyOptions} from '../collapsify.js'; 4 | 5 | export default function flattenExternalStyle( 6 | rewriter: HTMLRewriter, 7 | options: CollapsifyOptions, 8 | ) { 9 | rewriter.on('link', { 10 | async element(element) { 11 | const rel = element.getAttribute('rel'); 12 | 13 | if (!rel || rel !== 'stylesheet') { 14 | return; 15 | } 16 | 17 | const href = element.getAttribute('href'); 18 | 19 | if (!href) { 20 | return; 21 | } 22 | 23 | const content = await collapseCSS.external({ 24 | fetch: options.fetch, 25 | resourceLocation: new URL(href, options.resourceLocation).toString(), 26 | }); 27 | 28 | if (typeof content !== 'string') { 29 | throw new TypeError('Wrong output from collapseCSS'); 30 | } 31 | 32 | element.replace(``, {html: true}); 33 | }, 34 | }); 35 | } 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 Cloudflare, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any 4 | person obtaining a copy of this software and associated 5 | documentation files (the "Software"), to deal in the 6 | Software without restriction, including without 7 | limitation the rights to use, copy, modify, merge, 8 | publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software 10 | is furnished to do so, subject to the following 11 | conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions 15 | of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF 18 | ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 19 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A 20 | PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT 21 | SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 22 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 23 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR 24 | IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 25 | DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /test/helpers.js: -------------------------------------------------------------------------------- 1 | import {promises as fs} from 'node:fs'; 2 | import assert from 'power-assert'; 3 | 4 | function response({contentType, string, array}) { 5 | return { 6 | getContentType() { 7 | if (contentType) { 8 | return contentType; 9 | } 10 | 11 | assert(false, 'unexpected getContentType call'); 12 | }, 13 | 14 | async getAsString() { 15 | if (string) { 16 | return string; 17 | } 18 | 19 | assert(false, 'unexpected getAsString call'); 20 | }, 21 | 22 | async getAsArray() { 23 | if (array) { 24 | return array; 25 | } 26 | 27 | assert(false, 'unexpected getAsArray call'); 28 | }, 29 | }; 30 | } 31 | 32 | export function binaryResponse(array, contentType) { 33 | return response({array, contentType}); 34 | } 35 | 36 | export function stringResponse(string, contentType) { 37 | return response({string, contentType}); 38 | } 39 | 40 | export async function gifData() { 41 | return fs.readFile(new URL('fixtures/gif.gif', import.meta.url)); 42 | } 43 | 44 | export async function gifResponse() { 45 | return response({ 46 | array: gifData(), 47 | contentType: 'image/gif', 48 | }); 49 | } 50 | -------------------------------------------------------------------------------- /test/collapsers/javascript.js: -------------------------------------------------------------------------------- 1 | import assert from 'power-assert'; 2 | import {describe, it} from 'mocha'; 3 | import collapser from '../../built/collapsers/javascript.js'; 4 | import {stringResponse} from '../helpers.js'; 5 | 6 | describe('JavaScript collapser', () => { 7 | it('should minify JavaScript', async () => { 8 | const encoded = await collapser('alert("foo: " + bar)', { 9 | resourceLocation: '', 10 | }); 11 | assert(typeof encoded === 'string'); 12 | }); 13 | 14 | it('should preserve JavaScript as-is if minification fails', async () => { 15 | const original = 'for: {'; 16 | const encoded = await collapser(original, { 17 | resourceLocation: '', 18 | }); 19 | assert(encoded === original); 20 | }); 21 | 22 | describe('external', () => { 23 | it('should collapse an external script', async () => { 24 | const encoded = await collapser.external({ 25 | async fetch(url) { 26 | assert(url === 'https://example.com/script.js'); 27 | return stringResponse('console.log("hello world!");'); 28 | }, 29 | resourceLocation: 'https://example.com/script.js', 30 | }); 31 | 32 | assert(typeof encoded === 'string'); 33 | }); 34 | }); 35 | }); 36 | -------------------------------------------------------------------------------- /src/collapsers/css.ts: -------------------------------------------------------------------------------- 1 | import postcss, {Result} from 'postcss'; 2 | import cssnano from 'cssnano'; 3 | import bole from 'bole'; 4 | import flattenUrl from '../plugins/postcss-flatten-url.js'; 5 | import flattenImport from '../plugins/postcss-flatten-import.js'; 6 | import {CollapsifyError, type CollapsifyOptions} from '../collapsify.js'; 7 | 8 | const logger = bole('collapsify:collapsers:css'); 9 | 10 | type CssOptions = { 11 | imported?: boolean; 12 | } & CollapsifyOptions; 13 | 14 | async function external(options: CssOptions) { 15 | const response = await options.fetch(options.resourceLocation); 16 | return collapse(await response.getAsString(), options); 17 | } 18 | 19 | async function collapse(bodyString: string, options: CssOptions) { 20 | try { 21 | const lazy = postcss() 22 | .use(flattenUrl(options)) 23 | .use(flattenImport(options)) 24 | .use(cssnano({preset: 'default'})) 25 | .process(bodyString, {from: options.resourceLocation}); 26 | const result = await lazy; 27 | 28 | for (const message of result.warnings()) { 29 | logger.warn(message); 30 | } 31 | 32 | if (options.imported) { 33 | return result; 34 | } 35 | 36 | return result.css; 37 | } catch (error: unknown) { 38 | if (error instanceof CollapsifyError) { 39 | throw error; 40 | } 41 | 42 | logger.error(error); 43 | throw new CollapsifyError('Error during CSS inlining.'); 44 | } 45 | } 46 | 47 | collapse.external = external; 48 | 49 | export default collapse; 50 | -------------------------------------------------------------------------------- /src/utils/httpclient.ts: -------------------------------------------------------------------------------- 1 | import {Buffer} from 'node:buffer'; 2 | import bole from 'bole'; 3 | import VERSION from '../version.js'; 4 | import type {Fetch, Response} from '../collapsify'; 5 | 6 | const logger = bole('collapsify:http'); 7 | 8 | const userAgent = `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:60.0) Gecko/20100101 Firefox/60.0 Collapsify/${VERSION}`; 9 | 10 | class FetchResponse implements Response { 11 | constructor( 12 | private readonly statusCode: number, 13 | private readonly contentType: string, 14 | private readonly blob: Blob, 15 | ) {} 16 | 17 | getStatusCode(): number { 18 | return this.statusCode; 19 | } 20 | 21 | getContentType(): string { 22 | return this.contentType; 23 | } 24 | 25 | async getAsString(): Promise { 26 | return this.blob.text(); 27 | } 28 | 29 | async getAsArray(): Promise { 30 | return Buffer.from(await this.blob.arrayBuffer()); 31 | } 32 | } 33 | 34 | export default function makeClient( 35 | defaultHeaders?: Record, 36 | ): Fetch { 37 | const cache = new Map(); 38 | 39 | async function nodeFetch(url: string) { 40 | logger.debug('Fetching %s.', url); 41 | 42 | const resp = await fetch(url, { 43 | headers: { 44 | 'user-agent': userAgent, 45 | ...defaultHeaders, 46 | }, 47 | }); 48 | 49 | return new FetchResponse( 50 | resp.status, 51 | resp.headers.get('content-type') ?? '', 52 | await resp.blob(), 53 | ); 54 | } 55 | 56 | return nodeFetch; 57 | } 58 | -------------------------------------------------------------------------------- /test/plugins/html-flatten-image.js: -------------------------------------------------------------------------------- 1 | import assert from 'power-assert'; 2 | import {describe, it} from 'mocha'; 3 | import {gifResponse} from '../helpers.js'; 4 | import {rewriteHtml} from '../../built/utils/html-rewriter.js'; 5 | 6 | async function test(input, expected, options) { 7 | const actual = await rewriteHtml(input, options); 8 | assert.equal(actual, expected); 9 | } 10 | 11 | describe('posthtml-flatten-image', () => { 12 | it('should flatten found image', () => { 13 | return test( 14 | '
An animated graphic!
', 15 | '
An animated graphic!
', 16 | { 17 | async fetch(url) { 18 | assert(url === 'https://example.com/gif.gif'); 19 | return gifResponse(); 20 | }, 21 | resourceLocation: 'https://example.com/page.html', 22 | }, 23 | ); 24 | }); 25 | 26 | it('should ignore inlined images', () => { 27 | return test( 28 | '
An animated graphic!
', 29 | '
An animated graphic!
', 30 | { 31 | fetch() { 32 | assert(false, 'unexpected resource resolution'); 33 | }, 34 | resourceLocation: 'https://example.com/page.html', 35 | }, 36 | ); 37 | }); 38 | }); 39 | -------------------------------------------------------------------------------- /test/plugins/postcss-flatten-import.js: -------------------------------------------------------------------------------- 1 | import postcss from 'postcss'; 2 | import assert from 'power-assert'; 3 | import {describe, it} from 'mocha'; 4 | import plugin from '../../built/plugins/postcss-flatten-import.js'; 5 | import {stringResponse} from '../helpers.js'; 6 | 7 | async function test(input, output, options = {}) { 8 | const result = await postcss([plugin(options)]).process(input, { 9 | from: options.resourceLocation, 10 | }); 11 | 12 | assert(result.css === output); 13 | } 14 | 15 | describe('postcss-flatten-import', () => { 16 | it('should flatten imports', () => { 17 | return test( 18 | '@import "fonts.css"', 19 | '@font-face{font-family:Noto Sans;font-style:normal;font-weight:400;src:local("Noto Sans")}', 20 | { 21 | async fetch(url) { 22 | assert(url === 'http://example.com/static/css/fonts.css'); 23 | return stringResponse( 24 | '@font-face {\n font-family: Noto Sans;\n font-style: normal;\n font-weight: 400;\n src: local("Noto Sans")\n}', 25 | ); 26 | }, 27 | resourceLocation: 'http://example.com/static/css/app.css', 28 | }, 29 | ); 30 | }); 31 | 32 | it('should wrap flattend imports with media query', () => { 33 | return test( 34 | '@import flatten.css screen, projection', 35 | '@media screen, projection {.flatten{color:blue}}', 36 | { 37 | async fetch(url) { 38 | assert(url === 'http://example.com/static/css/flatten.css'); 39 | return stringResponse('.flatten { color: blue }'); 40 | }, 41 | resourceLocation: 'http://example.com/static/css/app.css', 42 | }, 43 | ); 44 | }); 45 | }); 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This project has been archived, and is no longer actively maintained. 2 | See the [Cloudflare Documentation](https://developers.cloudflare.com/support/more-dashboard-apps/cloudflare-custom-pages/configuring-custom-pages-error-and-challenge/) for details on managing Custom Pages. 3 | 4 | # Collapsify [![](http://img.shields.io/npm/dm/collapsify.svg?style=flat)](https://www.npmjs.org/package/collapsify) [![](http://img.shields.io/npm/v/collapsify.svg?style=flat)](https://www.npmjs.org/package/collapsify) 5 | 6 | > Inlines all of the JavaScripts, stylesheets, images, fonts etc. of an HTML page. 7 | 8 | ## Installation 9 | 10 | ```sh 11 | npm install -g collapsify 12 | ``` 13 | 14 | ## Usage 15 | 16 | You can use the collapsify CLI like this to download and save the page into a single file like this: 17 | ```sh 18 | collapsify -o single-page.html https://my-site.com/ 19 | ``` 20 | see `collapsify -h` for all options. 21 | 22 | ## API 23 | 24 | ```javascript 25 | import {simpleCollapsify} from 'collapsify'; 26 | 27 | await simpleCollapsify('https://example.com', { 28 | headers: { 29 | 'accept-language': 'en-US' 30 | } 31 | }) 32 | .then(page => console.log(page)) 33 | .catch(err => console.error(err)); 34 | 35 | ``` 36 | 37 | The `simpleCollapsify` function takes the URL to collapse, as well as an object of options, and returns a promise that resolves to a String. 38 | 39 | ### Options 40 | 41 | * **headers**: An object of headers, to be added to each HTTP request. 42 | * **forbidden**: A regex that matches blacklisted resources that should be avoided while navigating. 43 | 44 | ## Requirements 45 | The simple mode and CLI require nodejs >= 18, as they use the global `fetch` function. 46 | -------------------------------------------------------------------------------- /test/plugins/postcss-flatten-url.js: -------------------------------------------------------------------------------- 1 | import {Buffer} from 'node:buffer'; 2 | import postcss from 'postcss'; 3 | import assert from 'power-assert'; 4 | import {describe, it} from 'mocha'; 5 | import plugin from '../../built/plugins/postcss-flatten-url.js'; 6 | import {binaryResponse} from '../helpers.js'; 7 | 8 | async function test(input, output, options = {}) { 9 | const result = await postcss([plugin(options)]).process(input, { 10 | from: options.resourceLocation, 11 | }); 12 | 13 | assert(result.css === output); 14 | } 15 | 16 | describe('postcss-flatten-url', () => { 17 | it("should ignore properties that don't contain URLs", () => { 18 | return test( 19 | '.flatten { background: #0581C1 }', 20 | '.flatten { background: #0581C1 }', 21 | ); 22 | }); 23 | 24 | it('should replace the URL in a property', () => { 25 | return test( 26 | '.flatten { background: url("example.png") }', 27 | '.flatten { background: url(data:image/png;base64,) }', 28 | { 29 | async fetch(url) { 30 | assert(url === 'http://example.com/example.png'); 31 | return binaryResponse(Buffer.from(''), 'image/png'); 32 | }, 33 | resourceLocation: 'http://example.com/', 34 | }, 35 | ); 36 | }); 37 | 38 | it('should ignore URLs from the data scheme', () => { 39 | return test( 40 | '.flatten { background: url("data:application/x-empty;base64,") }', 41 | '.flatten { background: url("data:application/x-empty;base64,") }', 42 | { 43 | fetch() { 44 | assert.fail('should not have called fetch'); 45 | }, 46 | resourceLocation: 'http://example.com/', 47 | }, 48 | ); 49 | }); 50 | }); 51 | -------------------------------------------------------------------------------- /src/plugins/postcss-flatten-import.ts: -------------------------------------------------------------------------------- 1 | import {type Plugin, Result} from 'postcss'; 2 | import valueParser from 'postcss-value-parser'; 3 | import collapseCSS from '../collapsers/css.js'; 4 | import {type CollapsifyOptions} from '../collapsify.js'; 5 | import cssURL from '../utils/css-url.js'; 6 | 7 | export default function flattenImport(options: CollapsifyOptions): Plugin { 8 | return { 9 | postcssPlugin: 'postcss-flatten-import', 10 | // eslint-disable-next-line @typescript-eslint/naming-convention 11 | async Once(css) { 12 | const tasks: Array> = []; 13 | 14 | css.walkAtRules('import', (rule) => { 15 | const parsedValue = valueParser(rule.params); 16 | const url = cssURL(parsedValue.nodes[0], true); 17 | 18 | if (!url) return; 19 | 20 | const promise = collapseCSS 21 | .external({ 22 | imported: true, 23 | fetch: options.fetch, 24 | resourceLocation: new URL(url, options.resourceLocation).toString(), 25 | }) 26 | .then((result) => { 27 | if (!(result instanceof Result)) { 28 | throw new TypeError(`postcss result wasn't a Result`); 29 | } 30 | 31 | if (parsedValue.nodes.length > 1) { 32 | rule.name = 'media'; 33 | rule.params = rule.params 34 | .slice(parsedValue.nodes[1].sourceIndex) 35 | .trim(); 36 | rule.raws.between = ' '; 37 | rule.append(result.root); 38 | } else { 39 | rule.replaceWith(result.root); 40 | } 41 | }); 42 | 43 | tasks.push(promise); 44 | }); 45 | 46 | await Promise.all(tasks); 47 | }, 48 | }; 49 | } 50 | -------------------------------------------------------------------------------- /src/utils/fetch-wrapper.ts: -------------------------------------------------------------------------------- 1 | import type {Buffer} from 'node:buffer'; 2 | import bole from 'bole'; 3 | import {CollapsifyError, type Fetch, type Response} from '../collapsify.js'; 4 | 5 | const logger = bole('collapsify:fetch'); 6 | 7 | export function fetchWrapper(fetch: Fetch): Fetch { 8 | return async (url) => { 9 | try { 10 | const response = await fetch(url); 11 | if (response.getStatusCode() >= 300) { 12 | throw new CollapsifyError( 13 | `Fetch failed, ${url} returned ${response.getStatusCode()}`, 14 | ); 15 | } 16 | 17 | return new ResponseWrapper(url, response); 18 | } catch (error: unknown) { 19 | if (error instanceof CollapsifyError) { 20 | throw error; 21 | } 22 | 23 | logger.error(error); 24 | throw new CollapsifyError(`Fetch failed, ${url} unknown error occured`); 25 | } 26 | }; 27 | } 28 | 29 | class ResponseWrapper implements Response { 30 | constructor( 31 | private readonly url: string, 32 | private readonly response: Response, 33 | ) {} 34 | 35 | getStatusCode(): number { 36 | return this.response.getStatusCode(); 37 | } 38 | 39 | getContentType(): string { 40 | return this.response.getContentType(); 41 | } 42 | 43 | async getAsString(): Promise { 44 | try { 45 | return await this.response.getAsString(); 46 | } catch (error: unknown) { 47 | logger.error(error); 48 | throw new CollapsifyError(`Couldn't read ${this.url} as string`); 49 | } 50 | } 51 | 52 | async getAsArray(): Promise { 53 | try { 54 | return await this.response.getAsArray(); 55 | } catch (error: unknown) { 56 | logger.error(error); 57 | throw new CollapsifyError(`Couldn't read ${this.url} as binary`); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "collapsify", 3 | "description": "Inlines all of the JavaScripts, stylesheets, images, fonts etc. of an HTML page.", 4 | "version": "0.8.0", 5 | "author": "Terin Stock (http://terinstock.com)", 6 | "type": "module", 7 | "bin": { 8 | "collapsify": "./built/bin/cli.js" 9 | }, 10 | "bugs": "http://github.com/cloudflare/collapsify/issues", 11 | "contributors": [ 12 | "Christopher Joel (http://scriptolo.gy)", 13 | "Terin Stock (http://terinstock.com)", 14 | "Andrew Galloni ", 15 | "Ingvar Stepanyan (https://rreverser.com)" 16 | ], 17 | "dependencies": { 18 | "base64-js": "^1.5.1", 19 | "bole": "^3.0.2", 20 | "cliclopts": "^1.1.0", 21 | "cssnano": "^5.1.7", 22 | "html-rewriter-wasm": "^0.4.1", 23 | "minimist": "^1.1.0", 24 | "postcss": "^8.2.15", 25 | "postcss-value-parser": "^4.2.0", 26 | "terser": "^5.14.2" 27 | }, 28 | "homepage": "http://github.com/cloudflare/collapsify", 29 | "keywords": [ 30 | "html", 31 | "stylesheet", 32 | "font", 33 | "css", 34 | "javascript", 35 | "js", 36 | "compile", 37 | "inline", 38 | "mobilize", 39 | "optimize", 40 | "optimization" 41 | ], 42 | "license": "MIT", 43 | "main": "built/collapsify.js", 44 | "files": [ 45 | "built" 46 | ], 47 | "repository": { 48 | "type": "git", 49 | "url": "git://github.com/cloudflare/collapsify.git" 50 | }, 51 | "devDependencies": { 52 | "mocha": "^10.0.0", 53 | "power-assert": "^1.6.0", 54 | "typescript": "^5.0.4", 55 | "xo": "^0.54.2" 56 | }, 57 | "scripts": { 58 | "build": "tsc", 59 | "start": "node ./built/bin/cli.js", 60 | "test": "xo && mocha --recursive" 61 | }, 62 | "xo": { 63 | "space": true, 64 | "prettier": true 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/plugins/html-flatten-script.ts: -------------------------------------------------------------------------------- 1 | import bole from 'bole'; 2 | import {type HTMLRewriter} from 'html-rewriter-wasm'; 3 | import collapseJavaScript from '../collapsers/javascript.js'; 4 | import {type CollapsifyOptions} from '../collapsify.js'; 5 | 6 | const logger = bole('collapsify:collapsers:html'); 7 | 8 | export default function flattenScript( 9 | rewriter: HTMLRewriter, 10 | options: CollapsifyOptions, 11 | ) { 12 | rewriter.on('script', { 13 | async element(element) { 14 | const type = element.getAttribute('type'); 15 | 16 | // Ignore non-JavaScript types. 17 | // Empty `type` should be treated just like missing one. 18 | if ( 19 | type && 20 | type !== 'text/javascript' && 21 | type !== 'application/javascript' 22 | ) { 23 | logger.debug('ignoring script of type "%s"', type); 24 | return; 25 | } 26 | 27 | const src = element.getAttribute('src'); 28 | 29 | // Ignore inline scripts here. 30 | // Unlike `type`, empty `src` should be treated as actual value. 31 | if (!src) { 32 | return; 33 | } 34 | 35 | const content = await collapseJavaScript.external({ 36 | fetch: options.fetch, 37 | resourceLocation: new URL(src, options.resourceLocation).toString(), 38 | }); 39 | 40 | // Remove original `src` attribute. 41 | element.removeAttribute('src'); 42 | element.setInnerContent(content!, {html: true}); 43 | }, 44 | }); 45 | 46 | // Gets built up, then reset when `text.lastInTextNode` is found. 47 | let innerContent = ''; 48 | rewriter.on('script', { 49 | async text(text) { 50 | innerContent += text.text; 51 | if (text.lastInTextNode) { 52 | const content = await collapseJavaScript(innerContent, { 53 | resourceLocation: '', 15 | '', 16 | { 17 | fetch() { 18 | assert(false, 'unexpected resource resolution'); 19 | }, 20 | }, 21 | ); 22 | }); 23 | 24 | it('should ignore scripts with types other than JavaScript', () => { 25 | const handlebars = 26 | ''; 27 | return test(handlebars, handlebars, { 28 | fetch() { 29 | assert(false, 'unexpected resource resolution'); 30 | }, 31 | }); 32 | }); 33 | 34 | it('should flatten inline JavaScript wraped in CDATA', () => { 35 | return test( 36 | '', 37 | '', 38 | { 39 | fetch() { 40 | assert(false, 'unexpected resource resolution'); 41 | }, 42 | }, 43 | ); 44 | }); 45 | 46 | it('should flatten external JavaScript', () => { 47 | return test( 48 | '', 49 | '', 50 | { 51 | async fetch(url) { 52 | assert(url === 'https://example.com/app.js'); 53 | return stringResponse('alert("foo" + "bar"); var a = c < b;'); 54 | }, 55 | resourceLocation: 'https://example.com/', 56 | }, 57 | ); 58 | }); 59 | }); 60 | -------------------------------------------------------------------------------- /src/plugins/postcss-flatten-url.ts: -------------------------------------------------------------------------------- 1 | import {type Plugin} from 'postcss'; 2 | import valueParser from 'postcss-value-parser'; 3 | import collapseBinary from '../collapsers/binary.js'; 4 | import {type CollapsifyOptions} from '../collapsify.js'; 5 | import cssURL from '../utils/css-url.js'; 6 | 7 | export default function flattenUrl(options: CollapsifyOptions): Plugin { 8 | return { 9 | postcssPlugin: 'postcss-flatten-url', 10 | // eslint-disable-next-line @typescript-eslint/naming-convention 11 | async Once(css) { 12 | const tasks: Array> = []; 13 | 14 | css.walkDecls((decl) => { 15 | const parsedValue = valueParser(decl.value); 16 | const newTasks: Array> = []; 17 | 18 | parsedValue.walk((node, index, nodes) => { 19 | const url = cssURL(node, false); 20 | 21 | if (!url) return; 22 | 23 | newTasks.push( 24 | collapseBinary 25 | .external({ 26 | fetch: options.fetch, 27 | resourceLocation: new URL( 28 | url, 29 | options.resourceLocation, 30 | ).toString(), 31 | }) 32 | .then((binaryString) => { 33 | // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment 34 | nodes[index] = { 35 | type: 'function', 36 | value: 'url', 37 | nodes: [ 38 | { 39 | type: 'word', 40 | value: binaryString, 41 | } as any, 42 | ], 43 | } as any; 44 | }), 45 | ); 46 | }); 47 | 48 | const promise = Promise.all(newTasks).then(() => { 49 | // eslint-disable-next-line @typescript-eslint/no-base-to-string 50 | decl.value = parsedValue.toString(); 51 | }); 52 | 53 | tasks.push(promise); 54 | }); 55 | 56 | await Promise.all(tasks); 57 | }, 58 | }; 59 | } 60 | -------------------------------------------------------------------------------- /test/utils/data-uri.js: -------------------------------------------------------------------------------- 1 | import {Buffer} from 'node:buffer'; 2 | import assert from 'power-assert'; 3 | import {describe, it} from 'mocha'; 4 | import {gifData} from '../helpers.js'; 5 | import {encodeSync, validateSync} from '../../built/utils/data-uri.js'; 6 | 7 | describe('base64 utility', () => { 8 | describe('encode', () => { 9 | it('should base64 encode ASCII text', async () => { 10 | const encoded = encodeSync(Buffer.from('plain text'), { 11 | contentType: 'text/plain', 12 | }); 13 | assert(typeof encoded === 'string'); 14 | assert(encoded.startsWith('data:text/plain;base64,')); 15 | }); 16 | 17 | it('should base64 encode Unicode text', async () => { 18 | const encoded = encodeSync(Buffer.from([0xf0, 0x9f, 0x9a, 0x97]), { 19 | contentType: 'text/plain; charset=utf-8', 20 | }); 21 | assert(typeof encoded === 'string'); 22 | assert(encoded.startsWith('data:text/plain;charset=utf-8;base64,')); 23 | }); 24 | 25 | it('should base64 encode a GIF', async () => { 26 | const encoded = encodeSync(await gifData(), { 27 | contentType: 'image/gif', 28 | }); 29 | 30 | assert(typeof encoded === 'string'); 31 | assert(encoded.startsWith('data:image/gif;base64,')); 32 | }); 33 | 34 | it('should not have spaces in the base64 string', async () => { 35 | const encoded = encodeSync(await gifData(), { 36 | contentType: 'image/gif', 37 | }); 38 | 39 | assert(typeof encoded === 'string'); 40 | assert(!/\s/.test(encoded)); 41 | }); 42 | }); 43 | 44 | describe('verifySync', () => { 45 | it('should verify ASCII text', () => { 46 | const encoded = 'data:text/plain;charset=us-ascii,plain text'; 47 | assert(validateSync(encoded)); 48 | }); 49 | 50 | it('should verify Unicode text', () => { 51 | const encoded = 'data:text/plain;charset=utf-8;base64,8J+alw=='; 52 | assert(validateSync(encoded)); 53 | }); 54 | 55 | it('should verify a GIF', () => { 56 | const encoded = 'data:image/gif;base64,R0lGODlhAQABAAAAADs='; 57 | assert(validateSync(encoded)); 58 | }); 59 | }); 60 | }); 61 | -------------------------------------------------------------------------------- /test/collapsers/css.js: -------------------------------------------------------------------------------- 1 | import {Buffer} from 'node:buffer'; 2 | import assert from 'power-assert'; 3 | import {describe, it} from 'mocha'; 4 | import {binaryResponse} from '../helpers.js'; 5 | import collapser from '../../built/collapsers/css.js'; 6 | import {CollapsifyError} from '../../built/collapsify.js'; 7 | 8 | describe('CSS collapser', () => { 9 | it('should minify CSS', async () => { 10 | const collapsed = await collapser('html, body { height: 100%; }', { 11 | fetch() { 12 | assert(false, 'unexpected resource resolution'); 13 | }, 14 | resourceLocation: 'https://example.com', 15 | }); 16 | 17 | assert(typeof collapsed === 'string'); 18 | assert(collapsed === 'body,html{height:100%}'); 19 | }); 20 | 21 | it('should reject if invalid CSS', async () => { 22 | try { 23 | await collapser('html, body {', { 24 | fetch() { 25 | assert(false, 'unexpected resource resolution'); 26 | }, 27 | resourceLocation: 'https://example.com', 28 | }); 29 | 30 | assert(false, 'unexpect Promise resolution'); 31 | } catch (error) { 32 | assert(error instanceof CollapsifyError, 'wrong error type'); 33 | assert.equal(error.message, 'Error during CSS inlining.'); 34 | } 35 | }); 36 | 37 | it('fetch error message returned', async () => { 38 | try { 39 | await collapser(`html, body { background: url('something.jpg'); }`, { 40 | fetch() { 41 | throw new CollapsifyError('Error from fetch'); 42 | }, 43 | resourceLocation: 'https://example.com', 44 | }); 45 | 46 | assert(false, 'unexpect Promise resolution'); 47 | } catch (error) { 48 | assert(error instanceof CollapsifyError, 'wrong error type'); 49 | assert.equal(error.message, 'Error from fetch'); 50 | } 51 | }); 52 | 53 | describe('external', () => { 54 | it('should collapse an external binary', async () => { 55 | const collapsed = await collapser( 56 | 'body { background: url("example.png") }', 57 | { 58 | async fetch(url) { 59 | assert(url === 'https://example.com/example.png'); 60 | return binaryResponse(Buffer.from(''), 'image/png'); 61 | }, 62 | resourceLocation: 'https://example.com', 63 | }, 64 | ); 65 | 66 | assert(collapsed.includes('data:image/png')); 67 | }); 68 | }); 69 | }); 70 | -------------------------------------------------------------------------------- /test/plugins/html-flatten-style.js: -------------------------------------------------------------------------------- 1 | import assert from 'power-assert'; 2 | import {describe, it} from 'mocha'; 3 | import {stringResponse, gifResponse} from '../helpers.js'; 4 | import {rewriteHtml} from '../../built/utils/html-rewriter.js'; 5 | 6 | async function test(input, expected, options) { 7 | const actual = await rewriteHtml(input, options); 8 | assert.equal(actual, expected); 9 | } 10 | 11 | describe('posthtml-flatten-style', () => { 12 | it('should flatten inline style', () => { 13 | return test( 14 | '', 15 | '', 16 | { 17 | async fetch(url) { 18 | assert(url === 'https://example.com/gif.gif'); 19 | return gifResponse(); 20 | }, 21 | resourceLocation: 'https://example.com', 22 | }, 23 | ); 24 | }); 25 | 26 | it('should ignore base64 URLs in inline style', () => { 27 | return test( 28 | '', 29 | '', 30 | { 31 | fetch() { 32 | assert(false, 'unexpected resource resolution'); 33 | }, 34 | resourceLocation: 'https://example.com', 35 | }, 36 | ); 37 | }); 38 | 39 | it('should flatten external stylesheets', () => { 40 | return test( 41 | '', 42 | '', 43 | { 44 | async fetch(url) { 45 | assert(url === 'https://example.com/static/css/app.css'); 46 | return stringResponse('html, body { height: 100%; }'); 47 | }, 48 | resourceLocation: 'https://example.com/page.html', 49 | }, 50 | ); 51 | }); 52 | 53 | it('should flatten resources in external stylesheets', () => { 54 | return test( 55 | '', 56 | '', 57 | { 58 | async fetch(url) { 59 | switch (url) { 60 | case 'https://example.com/static/css/app.css': { 61 | return stringResponse( 62 | 'body > .test { background: url(gif.gif) }', 63 | ); 64 | } 65 | 66 | case 'https://example.com/static/css/gif.gif': { 67 | return gifResponse(); 68 | } 69 | 70 | default: { 71 | assert(false, 'unknown resource resolution'); 72 | } 73 | } 74 | }, 75 | resourceLocation: 'https://example.com/page.html', 76 | }, 77 | ); 78 | }); 79 | }); 80 | -------------------------------------------------------------------------------- /src/bin/cli.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | /* eslint @typescript-eslint/no-unsafe-assignment: 0 */ 3 | /* eslint @typescript-eslint/no-unsafe-call: 0 */ 4 | import * as fs from 'node:fs'; 5 | import * as process from 'node:process'; 6 | import bole from 'bole'; 7 | import cliclopts, {type Argument} from 'cliclopts'; 8 | import minimist from 'minimist'; 9 | import VERSION from '../version.js'; 10 | import {simpleCollapsify} from '../simple.js'; 11 | 12 | const allowedArgs: Argument[] = [ 13 | { 14 | name: 'forbidden', 15 | abbr: 'x', 16 | default: 17 | '^(?:https?:)?(?:/+)?(localhost|(?:127|192.168|172.16|10).[0-9.]+)', 18 | help: 'Forbidden URLs (passed to the RegExp constructor).', 19 | }, 20 | { 21 | name: 'headers', 22 | abbr: 'H', 23 | default: [], 24 | help: 'Custom headers (curl style) to set on all requests.', 25 | }, 26 | { 27 | name: 'verbose', 28 | abbr: 'V', 29 | default: 1, 30 | help: 'Verbosity of logging output. 0 is errors and warnings, 1 is info, 2 is all.', 31 | }, 32 | { 33 | name: 'version', 34 | abbr: 'v', 35 | boolean: true, 36 | help: 'Print the version number.', 37 | }, 38 | { 39 | name: 'help', 40 | abbr: 'h', 41 | boolean: true, 42 | help: 'Show this usage information.', 43 | }, 44 | { 45 | name: 'output', 46 | abbr: 'o', 47 | default: '/dev/null', 48 | help: 'Destination path for the resulting output', 49 | }, 50 | ]; 51 | 52 | type Args = { 53 | help: boolean; 54 | version: boolean; 55 | forbidden: string; 56 | headers: string[]; 57 | verbose: number; 58 | output: string; 59 | }; 60 | 61 | const clopts = cliclopts(allowedArgs); 62 | const argv = minimist(process.argv.slice(2), { 63 | alias: clopts.alias(), 64 | boolean: clopts.boolean(), 65 | default: clopts.default(), 66 | }); 67 | 68 | if (argv.help) { 69 | console.log('Usage: ' + process.argv.slice(1, 2).join(' ') + ' [options]\n'); 70 | console.log('Options:'); 71 | clopts.print(); 72 | process.exit(0); 73 | } 74 | 75 | if (argv.version) { 76 | console.log('Collapsify CLI - ' + String(VERSION)); 77 | process.exit(0); 78 | } 79 | 80 | const headers: Record = {}; 81 | for (const header of argv.headers.filter(Boolean)) { 82 | const [key, value] = header.trim().split(':'); 83 | headers[key.trim()] = value.trim(); 84 | } 85 | 86 | const options = { 87 | forbidden: argv.forbidden, 88 | headers, 89 | }; 90 | 91 | const levels = 'warn info debug'.split(' '); 92 | bole.output({ 93 | level: levels[argv.verbose] || 'warn', 94 | stream: process.stderr, 95 | }); 96 | const logger = bole('collapsify-cli'); 97 | 98 | const url = argv._[0]; 99 | 100 | const output = await simpleCollapsify(url, options); 101 | logger.info(`Collapsed Size: ${String(output.length)} bytes`); 102 | fs.writeFileSync(argv.output, output); 103 | -------------------------------------------------------------------------------- /test/utils/fetch-wrapper.js: -------------------------------------------------------------------------------- 1 | import {Buffer} from 'node:buffer'; 2 | import assert from 'power-assert'; 3 | import {describe, it} from 'mocha'; 4 | import {CollapsifyError} from '../../built/collapsify.js'; 5 | import {fetchWrapper} from '../../built/utils/fetch-wrapper.js'; 6 | 7 | describe('fetch-wrapper', () => { 8 | it('bad status code throws error', async () => { 9 | try { 10 | await fetchWrapper(() => new FakeResponse({statusCode: 404}))( 11 | 'http://exmaple.com', 12 | ); 13 | assert.fail('should have thrown'); 14 | } catch (error) { 15 | assertError(error, 'Fetch failed, http://exmaple.com returned 404'); 16 | } 17 | }); 18 | 19 | it('failed getAsString', async () => { 20 | try { 21 | const response = await fetchWrapper( 22 | () => new FakeResponse({text: Promise.reject()}), 23 | )('http://exmaple.com'); 24 | await response.getAsString(); 25 | assert.fail('should have thrown'); 26 | } catch (error) { 27 | assertError(error, `Couldn't read http://exmaple.com as string`); 28 | } 29 | }); 30 | 31 | it('failed getAsArray', async () => { 32 | try { 33 | const response = await fetchWrapper( 34 | () => new FakeResponse({text: Promise.reject()}), 35 | )('http://exmaple.com'); 36 | await response.getAsArray(); 37 | assert.fail('should have thrown'); 38 | } catch (error) { 39 | assertError(error, `Couldn't read http://exmaple.com as binary`); 40 | } 41 | }); 42 | 43 | it('can read properties', async () => { 44 | const response = await fetchWrapper( 45 | () => 46 | new FakeResponse({ 47 | statusCode: 200, 48 | contentType: 'text/plain', 49 | text: Promise.resolve('some text'), 50 | binary: Buffer.from([0x01, 0x02]), 51 | }), 52 | )('http://exmaple.com'); 53 | assert.equal(response.getStatusCode(), 200); 54 | assert.equal(response.getContentType(), 'text/plain'); 55 | assert.equal(await response.getAsString(), 'some text'); 56 | assert( 57 | Buffer.from([0x01, 0x02]).equals(await response.getAsArray()), 58 | 'array response equal', 59 | ); 60 | }); 61 | }); 62 | 63 | function assertError(error, message, type = CollapsifyError) { 64 | assert(error instanceof type, 'incorrect error type'); 65 | assert.equal(error.message, message); 66 | } 67 | 68 | class FakeResponse { 69 | constructor({ 70 | statusCode = 200, 71 | contentType = '', 72 | text = Promise.reject(), 73 | binary = Promise.reject(), 74 | }) { 75 | this.statusCode = statusCode; 76 | this.contentType = contentType; 77 | this.text = text; 78 | this.binary = binary; 79 | } 80 | 81 | getStatusCode() { 82 | return this.statusCode; 83 | } 84 | 85 | getContentType() { 86 | return this.contentType; 87 | } 88 | 89 | getAsString() { 90 | return this.text; 91 | } 92 | 93 | getAsArray() { 94 | return this.binary; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /test/collapsers/html.js: -------------------------------------------------------------------------------- 1 | import {Buffer} from 'node:buffer'; 2 | import assert from 'power-assert'; 3 | import {describe, it} from 'mocha'; 4 | import {binaryResponse, stringResponse} from '../helpers.js'; 5 | import collapser, {external} from '../../built/collapsers/html.js'; 6 | 7 | describe('html collapser', () => { 8 | it('should collapse a script tag', async () => { 9 | const collapsed = await collapser( 10 | '', 11 | { 12 | fetch() { 13 | assert(false, 'unexpected resource resolution'); 14 | }, 15 | }, 16 | ); 17 | 18 | assert(typeof collapsed === 'string'); 19 | assert( 20 | collapsed === 21 | '', 22 | ); 23 | }); 24 | 25 | it('should collapse an image', async () => { 26 | const collapsed = await collapser( 27 | '', 28 | { 29 | async fetch(url) { 30 | assert(url === 'https://example.org/foobar.png'); 31 | return binaryResponse(Buffer.from(''), 'image/png'); 32 | }, 33 | resourceLocation: 'https://example.com', 34 | }, 35 | ); 36 | 37 | assert(typeof collapsed === 'string'); 38 | assert( 39 | collapsed === 40 | '', 41 | ); 42 | }); 43 | 44 | it('should collapse an external HTML page', async () => { 45 | const collapsed = await external({ 46 | async fetch(url) { 47 | switch (url) { 48 | case 'https://terinstock.com': { 49 | return stringResponse( 50 | '

Hi.

', 51 | ); 52 | } 53 | 54 | case 'https://terinstock.com/avatar.jpeg': { 55 | return binaryResponse(Buffer.from(''), 'image/jpeg'); 56 | } 57 | 58 | default: { 59 | throw new assert.AssertionError('unknown resource resolution'); 60 | } 61 | } 62 | }, 63 | resourceLocation: 'https://terinstock.com', 64 | }); 65 | 66 | assert(typeof collapsed === 'string'); 67 | assert( 68 | collapsed === 69 | '

Hi.

', 70 | ); 71 | }); 72 | 73 | // Over 1kb the text is split into multiple chunks 74 | it('testing inline style block greater than 1kb', async () => { 75 | const bigContent = Array.from({length: 2048}).fill('a').join(''); 76 | const collapsed = await collapser( 77 | ``, 83 | { 84 | async fetch() { 85 | assert.fail('no request expected'); 86 | }, 87 | resourceLocation: 'https://example.com', 88 | }, 89 | ); 90 | 91 | assert.equal(typeof collapsed, 'string'); 92 | assert.equal( 93 | collapsed, 94 | ``, 95 | ); 96 | }); 97 | 98 | // Over 1kb the text is split into multiple chunks 99 | it('testing inline script block greater than 1kb', async () => { 100 | const bigContent = Array.from({length: 2048}).fill('a').join(''); 101 | const collapsed = await collapser( 102 | ``, 109 | { 110 | async fetch() { 111 | assert.fail('no request expected'); 112 | }, 113 | resourceLocation: 'https://example.com', 114 | }, 115 | ); 116 | 117 | assert.equal(typeof collapsed, 'string'); 118 | assert.equal( 119 | collapsed, 120 | ``, 121 | ); 122 | }); 123 | }); 124 | --------------------------------------------------------------------------------