├── .husky ├── pre-commit └── commit-msg ├── packages ├── core │ ├── tsconfig.json │ ├── src │ │ ├── node.ts │ │ ├── iife.ts │ │ ├── index.ts │ │ ├── types │ │ │ ├── utils.ts │ │ │ ├── strategy.ts │ │ │ ├── functionTypes.ts │ │ │ ├── index.ts │ │ │ ├── workerTypes.ts │ │ │ └── fileHashChunks.ts │ │ ├── node │ │ │ ├── index.ts │ │ │ ├── nodeWorkerPool.ts │ │ │ ├── nodeWorkerWrapper.ts │ │ │ ├── nodeUtils.ts │ │ │ └── nodeHashWorker.ts │ │ ├── browser │ │ │ ├── index.ts │ │ │ ├── browserWorkerPool.ts │ │ │ ├── browserUtils.ts │ │ │ ├── browserWorkerWrapper.ts │ │ │ └── browserHashWorker.ts │ │ ├── worker │ │ │ ├── browser.worker.ts │ │ │ └── node.worker.ts │ │ └── shared │ │ │ ├── index.ts │ │ │ ├── constant.ts │ │ │ ├── helper.ts │ │ │ ├── arrayBufferService.ts │ │ │ ├── baseWorkerWrapper.ts │ │ │ ├── baseHashWorker.ts │ │ │ ├── workerService.ts │ │ │ ├── utils.ts │ │ │ ├── merkleTree.ts │ │ │ └── baseWorkerPool.ts │ ├── LICENSE │ ├── CHANGELOG.md │ ├── jest.config.ts │ ├── package.json │ ├── rollup.config.ts │ ├── __tests__ │ │ └── node │ │ │ ├── shared │ │ │ ├── baseWorkerWrapper.spec.ts │ │ │ ├── helper.spec.ts │ │ │ ├── utils.spec.ts │ │ │ ├── arrayBufferService.spec.ts │ │ │ ├── workerService.spec.ts │ │ │ ├── merkleTree.spec.ts │ │ │ └── baseHashWorker.spec.ts │ │ │ ├── node │ │ │ └── nodeUtils.spec.ts │ │ │ └── browser │ │ │ ├── browserUtils.spec.ts │ │ │ └── browserWorkerPool.spec.ts │ ├── README-zh.md │ └── README.md ├── benchmark │ ├── src │ │ ├── node.ts │ │ ├── index.ts │ │ ├── shared │ │ │ ├── constant.ts │ │ │ ├── types.ts │ │ │ ├── helper.ts │ │ │ └── benchmark.ts │ │ ├── node │ │ │ ├── nodeHelper.ts │ │ │ └── nodeBenchmark.ts │ │ └── browser │ │ │ └── browserBenchmark.ts │ ├── tsconfig.json │ ├── tsup.config.ts │ ├── CHANGELOG.md │ ├── package.json │ ├── README-zh.md │ └── README.md └── playground │ ├── react-webpack-demo │ ├── .babelrc │ ├── tsconfig.json │ ├── public │ │ └── index.html │ ├── webpack.config.js │ ├── package.json │ └── src │ │ └── index.tsx │ ├── vue-vite-demo │ ├── src │ │ ├── main.ts │ │ ├── App.vue │ │ └── hooks │ │ │ └── useFileHashInfo.ts │ ├── tsconfig.json │ ├── vite.config.ts │ ├── index.html │ └── package.json │ ├── node-demo │ ├── tsup.config.ts │ ├── package.json │ ├── tsconfig.json │ └── src │ │ └── index.ts │ ├── benchmark-demo │ ├── tsup.config.ts │ ├── src │ │ └── index.ts │ ├── tsconfig.json │ └── package.json │ ├── react-rsbuild-demo │ ├── tsconfig.json │ ├── rsbuild.config.ts │ ├── src │ │ ├── index.tsx │ │ ├── env.d.ts │ │ └── App.tsx │ └── package.json │ └── iife-demo │ ├── package.json │ ├── prepare.js │ ├── index.html │ └── server.js ├── .gitattributes ├── .lintstagedrc ├── .prettierignore ├── .vscode └── settings.json ├── .prettierrc ├── .changeset ├── config.json └── README.md ├── codecov.yml ├── scripts ├── syncReadme.js ├── fileCopier.js └── clear.js ├── tsconfig.json ├── .github └── workflows │ └── ci.yml ├── pnpm-workspace.yaml ├── turbo.json ├── .gitignore ├── LICENSE ├── commitlint.config.js ├── eslint.config.mjs ├── package.json ├── README-zh.md └── README.md /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | npx lint-staged 2 | -------------------------------------------------------------------------------- /.husky/commit-msg: -------------------------------------------------------------------------------- 1 | npx --no-install commitlint --edit "$1" -------------------------------------------------------------------------------- /packages/core/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json" 3 | } 4 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.lintstagedrc: -------------------------------------------------------------------------------- 1 | { 2 | "*.{js,jsx,ts,tsx}": ["eslint --fix", "prettier --write"] 3 | } 4 | -------------------------------------------------------------------------------- /packages/benchmark/src/node.ts: -------------------------------------------------------------------------------- 1 | export type * from './shared/types' 2 | export * from './node/nodeBenchmark' 3 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | /dist/ 2 | /node_modules/ 3 | /coverage/ 4 | /.idea/ 5 | /.husky/ 6 | /.github/ 7 | /.changeset/ 8 | -------------------------------------------------------------------------------- /packages/benchmark/src/index.ts: -------------------------------------------------------------------------------- 1 | export type * from './shared/types' 2 | export * from './browser/browserBenchmark' 3 | -------------------------------------------------------------------------------- /packages/benchmark/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../tsconfig.json", 3 | "include": ["src/**/*.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /packages/benchmark/src/shared/constant.ts: -------------------------------------------------------------------------------- 1 | export const FILE_PATH = './data.txt' 2 | export const FILE_NAME = 'data.txt' 3 | -------------------------------------------------------------------------------- /packages/core/src/node.ts: -------------------------------------------------------------------------------- 1 | export * from './node/index' 2 | export * from './shared/index' 3 | export * from './types/index' 4 | -------------------------------------------------------------------------------- /packages/core/src/iife.ts: -------------------------------------------------------------------------------- 1 | // 让 iife 有一个 namespace 的类型提示 2 | import * as HashWorker from '.' 3 | 4 | export default HashWorker 5 | -------------------------------------------------------------------------------- /packages/core/src/index.ts: -------------------------------------------------------------------------------- 1 | export * from './browser/index' 2 | export * from './shared/index' 3 | export * from './types/index' 4 | -------------------------------------------------------------------------------- /packages/core/src/types/utils.ts: -------------------------------------------------------------------------------- 1 | export type RequiredWithExclude = Required> & Pick 2 | -------------------------------------------------------------------------------- /packages/core/src/types/strategy.ts: -------------------------------------------------------------------------------- 1 | export enum Strategy { 2 | md5 = 'md5', 3 | xxHash64 = 'xxHash64', 4 | xxHash128 = 'xxHash128', 5 | } 6 | -------------------------------------------------------------------------------- /packages/playground/react-webpack-demo/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["@babel/preset-env", "@babel/preset-react", "@babel/preset-typescript"] 3 | } 4 | -------------------------------------------------------------------------------- /packages/playground/vue-vite-demo/src/main.ts: -------------------------------------------------------------------------------- 1 | import { createApp } from 'vue' 2 | import App from './App.vue' 3 | 4 | createApp(App).mount('#app') 5 | -------------------------------------------------------------------------------- /packages/core/src/types/functionTypes.ts: -------------------------------------------------------------------------------- 1 | export type Resolve = (value: T | PromiseLike) => void 2 | export type Reject = (reason?: any) => void 3 | -------------------------------------------------------------------------------- /packages/playground/vue-vite-demo/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../../tsconfig.json", 3 | "compilerOptions": { 4 | "types": ["vite/client"] 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /packages/core/src/node/index.ts: -------------------------------------------------------------------------------- 1 | export * from './nodeWorkerPool' 2 | export * from './nodeUtils' 3 | export * from './nodeWorkerWrapper' 4 | export * from './nodeHashWorker' 5 | -------------------------------------------------------------------------------- /packages/playground/node-demo/tsup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'tsup' 2 | 3 | export default defineConfig({ 4 | entry: ['src/index.ts'], 5 | format: ['esm'], 6 | }) 7 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "cSpell.words": [ 3 | "Chks", 4 | "Merkle", 5 | "rimraf", 6 | "rsbuild", 7 | "tsup", 8 | "vite", 9 | "vitejs" 10 | ] 11 | } -------------------------------------------------------------------------------- /packages/core/src/browser/index.ts: -------------------------------------------------------------------------------- 1 | export * from './browserWorkerPool' 2 | export * from './browserUtils' 3 | export * from './browserWorkerWrapper' 4 | export * from './browserHashWorker' 5 | -------------------------------------------------------------------------------- /packages/playground/benchmark-demo/tsup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'tsup' 2 | 3 | export default defineConfig({ 4 | entry: ['src/index.ts'], 5 | format: ['esm'], 6 | }) 7 | -------------------------------------------------------------------------------- /packages/playground/vue-vite-demo/vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite' 2 | import vue from '@vitejs/plugin-vue' 3 | 4 | export default defineConfig({ 5 | plugins: [vue()], 6 | }) 7 | -------------------------------------------------------------------------------- /packages/playground/react-rsbuild-demo/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../../tsconfig.json", 3 | "compilerOptions": { 4 | "jsx": "react", 5 | "types": ["react", "react-dom"] 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /packages/playground/react-webpack-demo/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../../tsconfig.json", 3 | "compilerOptions": { 4 | "jsx": "react", 5 | "types": ["react", "react-dom"] 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /packages/playground/iife-demo/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "browser-demo", 3 | "private": true, 4 | "license": "MIT", 5 | "scripts": { 6 | "play": "node prepare.js && node server.js" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /packages/core/src/types/index.ts: -------------------------------------------------------------------------------- 1 | export type * from './fileHashChunks' 2 | export type * from './functionTypes' 3 | export type * from './utils' 4 | 5 | export * from './strategy' 6 | export * from './workerTypes' 7 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 100, 3 | "semi": false, 4 | "singleQuote": true, 5 | "trailingComma": "all", 6 | "proseWrap": "never", 7 | "htmlWhitespaceSensitivity": "strict", 8 | "endOfLine": "auto" 9 | } 10 | -------------------------------------------------------------------------------- /packages/core/src/worker/browser.worker.ts: -------------------------------------------------------------------------------- 1 | /// 2 | import { calculateHashInWorker } from '../shared' 3 | 4 | addEventListener('message', async ({ data }) => { 5 | const res = await calculateHashInWorker(data) 6 | postMessage(res, [data.chunk]) 7 | }) 8 | -------------------------------------------------------------------------------- /packages/benchmark/tsup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'tsup' 2 | 3 | export default defineConfig({ 4 | entry: { 5 | index: 'src/index.ts', 6 | node: 'src/node.ts', 7 | }, 8 | format: ['esm', 'cjs'], 9 | external: ['hash-worker'], 10 | dts: true, 11 | }) 12 | -------------------------------------------------------------------------------- /packages/core/src/worker/node.worker.ts: -------------------------------------------------------------------------------- 1 | import { calculateHashInWorker } from '../shared' 2 | import { parentPort } from 'worker_threads' 3 | 4 | parentPort?.on('message', async (data) => { 5 | const res = await calculateHashInWorker(data) 6 | parentPort?.postMessage(res, [data.chunk]) 7 | }) 8 | -------------------------------------------------------------------------------- /.changeset/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://unpkg.com/@changesets/config@3.0.5/schema.json", 3 | "changelog": "@changesets/cli/changelog", 4 | "commit": false, 5 | "fixed": [], 6 | "linked": [], 7 | "access": "restricted", 8 | "baseBranch": "main", 9 | "updateInternalDependencies": "patch", 10 | "ignore": [] 11 | } 12 | -------------------------------------------------------------------------------- /packages/core/src/shared/index.ts: -------------------------------------------------------------------------------- 1 | export * from './arrayBufferService' 2 | export * from './baseHashWorker' 3 | export * from './baseWorkerPool' 4 | export * from './baseWorkerWrapper' 5 | export * from './constant' 6 | export * from './helper' 7 | export * from './merkleTree' 8 | export * from './utils' 9 | export * from './workerService' 10 | -------------------------------------------------------------------------------- /packages/playground/react-rsbuild-demo/rsbuild.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from '@rsbuild/core' 2 | import { pluginReact } from '@rsbuild/plugin-react' 3 | 4 | export default defineConfig({ 5 | plugins: [pluginReact()], 6 | server: { 7 | port: 8889, 8 | open: false, 9 | }, 10 | html: { 11 | title: 'React Rsbuild Demo', 12 | }, 13 | }) 14 | -------------------------------------------------------------------------------- /packages/playground/react-webpack-demo/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | React Webpack Demo 7 | 8 | 9 |
10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /packages/playground/vue-vite-demo/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Vue Vite Demo 7 | 8 | 9 |
10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /packages/playground/benchmark-demo/src/index.ts: -------------------------------------------------------------------------------- 1 | import { benchmark, BenchmarkOptions } from 'hash-worker-benchmark/node' 2 | import { Strategy } from 'hash-worker/node' 3 | 4 | const options: BenchmarkOptions = { 5 | sizeInMB: 100, 6 | workerCountTobeTest: [4, 4, 4, 6, 6, 6, 8, 8, 8], 7 | strategy: Strategy.md5, 8 | } 9 | 10 | benchmark(options).then(() => {}) 11 | -------------------------------------------------------------------------------- /packages/core/src/shared/constant.ts: -------------------------------------------------------------------------------- 1 | import { md5, xxhash64, xxhash128 } from 'hash-wasm' 2 | import { Strategy } from '../types' 3 | 4 | export const DEFAULT_MAX_WORKERS = 8 5 | 6 | export type HashStrategy = Strategy 7 | export const HASH_FUNCTIONS = { 8 | [Strategy.md5]: md5, 9 | [Strategy.xxHash64]: xxhash64, 10 | [Strategy.xxHash128]: xxhash128, 11 | } as const 12 | -------------------------------------------------------------------------------- /packages/playground/react-rsbuild-demo/src/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import ReactDOM from 'react-dom/client' 3 | import App from './App' 4 | 5 | const rootEl = document.getElementById('root') 6 | if (rootEl) { 7 | const root = ReactDOM.createRoot(rootEl) 8 | root.render( 9 | 10 | 11 | , 12 | ) 13 | } 14 | -------------------------------------------------------------------------------- /packages/benchmark/src/shared/types.ts: -------------------------------------------------------------------------------- 1 | import { HashWorkerOptions, Strategy } from 'hash-worker' 2 | 3 | export interface BenchmarkOptions { 4 | sizeInMB?: number // 默认: 测试文件 500MB 5 | strategy?: Strategy // 默认: 使用 MD5 作为 hash 策略 6 | workerCountTobeTest?: number[] // 默认: 1, 4, 8, 12 线程各测三次 7 | } 8 | 9 | export interface NormalizeOptions { 10 | sizeInMB: number 11 | params: HashWorkerOptions[] 12 | } 13 | -------------------------------------------------------------------------------- /packages/benchmark/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # hash-worker-benchmark 2 | 3 | ## 2.0.0 4 | 5 | ### Major Changes 6 | 7 | - release version 2.0.0 8 | 9 | ## 1.0.0 10 | 11 | ### Major Changes 12 | 13 | - release version 1.0.0 14 | 15 | ### Patch Changes 16 | 17 | - Updated dependencies 18 | - hash-worker@1.0.0 19 | 20 | ## 0.0.1 21 | 22 | ### Patch Changes 23 | 24 | - Updated dependencies 25 | - hash-worker@0.1.3 26 | -------------------------------------------------------------------------------- /packages/playground/benchmark-demo/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../../tsconfig.json", 3 | "compilerOptions": { 4 | "moduleResolution": "bundler", 5 | "module": "ESNext", 6 | "outDir": "./dist", // 生成的 JavaScript 文件输出目录 7 | "rootDir": "./src", // TypeScript 源文件目录 8 | "sourceMap": false, // 不生成 .map 文件 9 | "declaration": false // 不生成 .d.ts 文件 10 | }, 11 | "include": ["src/**/*.ts"] 12 | } 13 | -------------------------------------------------------------------------------- /packages/playground/node-demo/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "node-demo", 3 | "private": true, 4 | "type": "module", 5 | "license": "MIT", 6 | "scripts": { 7 | "play": "node dist/index.js", 8 | "build:node-demo": "pnpm rm:dist && tsup --config tsup.config.ts", 9 | "rm:dist": "rimraf ./dist" 10 | }, 11 | "dependencies": { 12 | "hash-worker": "workspace:*" 13 | }, 14 | "devDependencies": {} 15 | } 16 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | require_ci_to_pass: true 3 | 4 | coverage: 5 | status: 6 | project: 7 | default: 8 | target: 90% 9 | threshold: 2% 10 | base: auto 11 | if_ci_failed: error 12 | branches: 13 | - 'master' 14 | 15 | comment: 16 | layout: 'reach, diff, flags, files' 17 | behavior: default 18 | require_changes: false 19 | branches: 20 | - 'master' 21 | -------------------------------------------------------------------------------- /packages/playground/node-demo/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../../../tsconfig.json", 3 | "compilerOptions": { 4 | "moduleResolution": "bundler", // 宽松规则, 配合打包工具 5 | "module": "ESNext", // 支持最新的模块系统 6 | "outDir": "./dist", // 生成的 JavaScript 文件输出目录 7 | "rootDir": "./src", // TypeScript 源文件目录 8 | "sourceMap": false, // 不生成 .map 文件 9 | "declaration": false // 不生成 .d.ts 文件 10 | }, 11 | "include": ["src/**/*.ts"] 12 | } 13 | -------------------------------------------------------------------------------- /packages/playground/react-rsbuild-demo/src/env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | /** 4 | * Imports the SVG file as a React component. 5 | * @requires [@rsbuild/plugin-svgr](https://npmjs.com/package/@rsbuild/plugin-svgr) 6 | */ 7 | declare module '*.svg?react' { 8 | import type React from 'react' 9 | const ReactComponent: React.FunctionComponent> 10 | export default ReactComponent 11 | } 12 | -------------------------------------------------------------------------------- /packages/core/src/node/nodeWorkerPool.ts: -------------------------------------------------------------------------------- 1 | import { Worker as NodeWorker } from 'worker_threads' 2 | import { BaseWorkerPool } from '../shared' 3 | import { NodeWorkerWrapper } from './nodeWorkerWrapper' 4 | 5 | export class NodeWorkerPool extends BaseWorkerPool { 6 | createWorker(): NodeWorkerWrapper { 7 | return new NodeWorkerWrapper( 8 | new NodeWorker(new URL('./worker/node.worker.mjs', import.meta.url)), 9 | ) 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /packages/playground/vue-vite-demo/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "vue-demo", 3 | "private": true, 4 | "type": "module", 5 | "license": "MIT", 6 | "scripts": { 7 | "play": "vite --port 8888" 8 | }, 9 | "dependencies": { 10 | "hash-worker": "workspace:*", 11 | "hash-worker-benchmark": "workspace:*", 12 | "vue": "catalog:" 13 | }, 14 | "devDependencies": { 15 | "@vitejs/plugin-vue": "catalog:", 16 | "vite": "catalog:" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /packages/core/src/browser/browserWorkerPool.ts: -------------------------------------------------------------------------------- 1 | import { BaseWorkerPool } from '../shared' 2 | import { BrowserWorkerWrapper } from './browserWorkerWrapper' 3 | 4 | export class BrowserWorkerPool extends BaseWorkerPool { 5 | createWorker(): BrowserWorkerWrapper { 6 | return new BrowserWorkerWrapper( 7 | // 指向打包后的 worker 路径 8 | new Worker(new URL('./worker/browser.worker.mjs', import.meta.url), { type: 'module' }), 9 | ) 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /packages/playground/benchmark-demo/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "benchmark", 3 | "private": true, 4 | "type": "module", 5 | "license": "MIT", 6 | "scripts": { 7 | "play": "node dist/index.js", 8 | "build:benchmark-demo": "pnpm rm:dist && tsup --config tsup.config.ts", 9 | "rm:dist": "rimraf ./dist" 10 | }, 11 | "dependencies": { 12 | "hash-worker": "workspace:*", 13 | "hash-worker-benchmark": "workspace:*" 14 | }, 15 | "devDependencies": {} 16 | } 17 | -------------------------------------------------------------------------------- /packages/playground/node-demo/src/index.ts: -------------------------------------------------------------------------------- 1 | import { getFileHashChunks, HashWorkerOptions, HashWorkerResult, Strategy } from 'hash-worker/node' 2 | 3 | const param: HashWorkerOptions = { 4 | filePath: '/home/tkunl/下载/docker-desktop-amd64.deb', 5 | config: { 6 | strategy: Strategy.md5, 7 | workerCount: 6, 8 | }, 9 | } 10 | 11 | function main() { 12 | getFileHashChunks(param).then((res: HashWorkerResult) => { 13 | console.log(res) 14 | }) 15 | } 16 | 17 | main() 18 | -------------------------------------------------------------------------------- /packages/playground/react-rsbuild-demo/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "react-rsbuild-demo", 3 | "private": true, 4 | "license": "MIT", 5 | "scripts": { 6 | "play": "rsbuild dev" 7 | }, 8 | "dependencies": { 9 | "react": "catalog:", 10 | "react-dom": "catalog:", 11 | "hash-worker": "workspace:*" 12 | }, 13 | "devDependencies": { 14 | "@rsbuild/core": "catalog:", 15 | "@rsbuild/plugin-react": "catalog:", 16 | "@types/react": "catalog:", 17 | "@types/react-dom": "catalog:" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /scripts/syncReadme.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const { copyFiles } = require('./fileCopier') 3 | 4 | // 获取项目的根目录路径 5 | const rootDir = process.cwd() 6 | 7 | // 定义要复制的文件列表 8 | const filesToCopy = [ 9 | { 10 | src: path.resolve(rootDir, 'README.md'), 11 | dest: path.resolve(rootDir, 'packages/core/README.md'), 12 | }, 13 | { 14 | src: path.resolve(rootDir, 'README-zh.md'), 15 | dest: path.resolve(rootDir, 'packages/core/README-zh.md'), 16 | }, 17 | ] 18 | 19 | // 执行文件复制 20 | copyFiles(filesToCopy) 21 | -------------------------------------------------------------------------------- /.changeset/README.md: -------------------------------------------------------------------------------- 1 | # Changesets 2 | 3 | Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works 4 | with multi-package repos, or single-package repos to help you version and publish your code. You can 5 | find the full documentation for it [in our repository](https://github.com/changesets/changesets) 6 | 7 | We have a quick list of common questions to get you started engaging with this project in 8 | [our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md) 9 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2022", 4 | "module": "ESNext", 5 | "sourceMap": true, 6 | "declaration": true, // 是否包含 d.ts 文件 7 | "strict": true, // Ts 的严格模式 8 | "esModuleInterop": true, // 可以使用 es6 的导入语法导入 cjs 模块 9 | "forceConsistentCasingInFileNames": true, // 确保文件大小写一致来确保引用的一致性 10 | "skipLibCheck": true, // 跳过第三方库的 d.ts 类型文件检查 11 | "lib": ["esnext", "dom"], // 指定编译过程中需要包括的库文件列表, 这些库文件是声明 js 运行时和 dom 中可用的全局变量的类型 12 | "moduleResolution": "Bundler" // 定义依赖解析策略, 默认是 node 模块解析策略 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /packages/playground/iife-demo/prepare.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const { copyFiles } = require('../../../scripts/fileCopier') 3 | 4 | const rootDir = process.cwd() 5 | 6 | // 定义要复制的文件列表 7 | const filesToCopy = [ 8 | { 9 | src: path.resolve(rootDir, '../../core/dist/global.js'), 10 | dest: path.resolve(rootDir, 'global.js'), 11 | }, 12 | { 13 | src: path.resolve(rootDir, '../../core/dist/worker/browser.worker.mjs'), 14 | dest: path.resolve(rootDir, './worker/browser.worker.mjs'), 15 | }, 16 | ] 17 | 18 | // 执行文件复制 19 | copyFiles(filesToCopy) 20 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Workflow for Codecov 2 | on: [push] 3 | jobs: 4 | run: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - uses: actions/checkout@v4 8 | - name: Install berry 9 | run: corepack enable 10 | - uses: actions/setup-node@v3 11 | with: 12 | node-version: 22 13 | - name: Install Dependencies 14 | run: pnpm install --no-frozen-lockfile 15 | - name: Run Test 16 | run: pnpm test 17 | - name: Upload coverage reports to Codecov 18 | uses: codecov/codecov-action@v4.0.1 19 | with: 20 | token: ${{ secrets.CODECOV_TOKEN }} 21 | -------------------------------------------------------------------------------- /packages/core/src/types/workerTypes.ts: -------------------------------------------------------------------------------- 1 | import { Strategy } from './strategy' 2 | 3 | export enum WorkerStatusEnum { 4 | RUNNING = 'running', 5 | WAITING = 'waiting', 6 | ERROR = 'error', 7 | } 8 | 9 | export interface WorkerReq { 10 | chunk: ArrayBuffer 11 | strategy: Strategy 12 | } 13 | 14 | export interface WorkerRes { 15 | result: T 16 | chunk: ArrayBuffer 17 | } 18 | 19 | export type TaskResult = 20 | | { success: true; data: T; index: number } 21 | | { success: false; error: Error; index: number } 22 | 23 | export interface TaskConfig { 24 | timeout?: number 25 | // 重试次数预留, 目前未实现 26 | retries?: number 27 | } 28 | -------------------------------------------------------------------------------- /packages/playground/react-webpack-demo/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | 3 | module.exports = { 4 | entry: './src/index.tsx', 5 | output: { 6 | filename: 'bundle.js', 7 | path: path.resolve(__dirname, 'dist'), 8 | publicPath: '/', 9 | }, 10 | resolve: { 11 | extensions: ['.ts', '.tsx', '.js'], 12 | }, 13 | module: { 14 | rules: [ 15 | { 16 | test: /\.(ts|tsx)$/, 17 | use: 'babel-loader', 18 | exclude: /node_modules/, 19 | }, 20 | ], 21 | }, 22 | devServer: { 23 | static: { 24 | directory: path.join(__dirname, 'public'), 25 | }, 26 | compress: true, 27 | port: 8890, 28 | historyApiFallback: true, 29 | }, 30 | } 31 | -------------------------------------------------------------------------------- /packages/playground/react-webpack-demo/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "react-webpack-demo", 3 | "private": true, 4 | "license": "MIT", 5 | "scripts": { 6 | "play": "webpack serve --mode development" 7 | }, 8 | "dependencies": { 9 | "react": "catalog:", 10 | "react-dom": "catalog:", 11 | "hash-worker": "workspace:*" 12 | }, 13 | "devDependencies": { 14 | "@babel/core": "catalog:", 15 | "@babel/preset-env": "catalog:", 16 | "@babel/preset-react": "catalog:", 17 | "@babel/preset-typescript": "catalog:", 18 | "@types/react": "catalog:", 19 | "@types/react-dom": "catalog:", 20 | "babel-loader": "catalog:", 21 | "webpack": "catalog:", 22 | "webpack-cli": "catalog:", 23 | "webpack-dev-server": "catalog:" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /pnpm-workspace.yaml: -------------------------------------------------------------------------------- 1 | packages: 2 | - "packages/core" 3 | - "packages/benchmark" 4 | - "packages/playground/*" 5 | 6 | catalog: 7 | hash-wasm: "^4.12.0" 8 | ts-node: "^10.9.2" 9 | vue: "^3.5.18" 10 | "@vitejs/plugin-vue": "^6.0.1" 11 | vite: "^7.0.6" 12 | react: "^18.2.0" 13 | react-dom: "^18.2.0" 14 | "@babel/core": "^7.26.0" 15 | "@babel/preset-env": "^7.26.0" 16 | "@babel/preset-react": "^7.26.3" 17 | "@babel/preset-typescript": "^7.26.0" 18 | "@types/react": "^18.2.0" 19 | "@types/react-dom": "^18.2.0" 20 | babel-loader: "^9.2.1" 21 | webpack: "^5.97.1" 22 | webpack-cli: "^6.0.1" 23 | webpack-dev-server: "^5.2.0" 24 | typescript: "^5.8.3" 25 | tsup: "^8.4.0" 26 | rimraf: "^6.0.1" 27 | "@rsbuild/core": "^1.4.3" 28 | "@rsbuild/plugin-react": "^1.3.3" 29 | -------------------------------------------------------------------------------- /turbo.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://turbo.build/schema.json", 3 | "tasks": { 4 | "build": { 5 | "dependsOn": ["^build"], 6 | "outputs": ["dist/**"], 7 | "outputLogs": "new-only" 8 | }, 9 | "build:benchmark": { 10 | "dependsOn": ["^build", "^build:benchmark"], 11 | "outputs": ["dist/**"], 12 | "outputLogs": "new-only" 13 | }, 14 | "build:benchmark-demo": { 15 | "dependsOn": ["^build:benchmark", "^build:benchmark-demo"], 16 | "outputs": ["dist/**"], 17 | "outputLogs": "new-only" 18 | }, 19 | "build:node-demo": { 20 | "dependsOn": ["^build", "^build:node-demo"], 21 | "outputs": ["dist/**"], 22 | "outputLogs": "new-only" 23 | }, 24 | "build:all": { 25 | "dependsOn": ["build", "build:benchmark", "build:benchmark-demo", "build:node-demo"], 26 | "outputs": ["dist/**"], 27 | "outputLogs": "new-only" 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /packages/playground/vue-vite-demo/src/App.vue: -------------------------------------------------------------------------------- 1 | 19 | 20 | 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See http://help.github.com/ignore-files/ for more about ignoring files. 2 | 3 | # Compiled output, 排除打包产物 4 | dist 5 | /output 6 | /tmp 7 | /out-tsc 8 | /bazel-out 9 | 10 | # 排除子项目的打包产物 11 | **/output 12 | **/.turbo 13 | 14 | # Node, 忽略任意层级下的 node_modules 目录 15 | **/node_modules/ 16 | npm-debug.log 17 | yarn-error.log 18 | 19 | # 忽略覆盖率 20 | **/coverage/ 21 | 22 | # IDEs and editors 23 | .idea/ 24 | .project 25 | .classpath 26 | .c9/ 27 | *.launch 28 | .settings/ 29 | *.sublime-workspace 30 | 31 | # Visual Studio Code 32 | .vscode/* 33 | !.vscode/settings.json 34 | !.vscode/tasks.json 35 | !.vscode/launch.json 36 | !.vscode/extensions.json 37 | .history/* 38 | 39 | # Miscellaneous 40 | /.angular/cache 41 | .sass-cache/ 42 | /connect.lock 43 | /libpeerconnection.log 44 | testem.log 45 | /typings 46 | 47 | # System files 48 | .DS_Store 49 | Thumbs.db 50 | 51 | # browser-demo 52 | /packages/playground/iife-demo/global.js 53 | /packages/playground/iife-demo/worker/* 54 | -------------------------------------------------------------------------------- /packages/benchmark/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hash-worker-benchmark", 3 | "version": "1.0.0", 4 | "private": true, 5 | "type": "module", 6 | "types": "./dist/index.d.ts", 7 | "main": "./dist/index.cjs", 8 | "module": "./dist/index.js", 9 | "exports": { 10 | ".": { 11 | "types": "./dist/index.d.ts", 12 | "import": "./dist/index.js", 13 | "require": "./dist/index.cjs" 14 | }, 15 | "./browser": { 16 | "types": "./dist/index.d.ts", 17 | "import": "./dist/index.js", 18 | "require": "./dist/index.cjs" 19 | }, 20 | "./node": { 21 | "types": "./dist/node.d.ts", 22 | "import": "./dist/node.js", 23 | "require": "./dist/node.cjs" 24 | } 25 | }, 26 | "scripts": { 27 | "dev": "tsup --config tsup.config.ts --watch", 28 | "build:benchmark": "pnpm rm:dist && tsup --config tsup.config.ts", 29 | "rm:dist": "rimraf ./dist" 30 | }, 31 | "license": "MIT", 32 | "dependencies": { 33 | "hash-worker": "workspace:*" 34 | }, 35 | "devDependencies": {} 36 | } 37 | -------------------------------------------------------------------------------- /packages/benchmark/README-zh.md: -------------------------------------------------------------------------------- 1 | ## Introduce for benchmark 2 | 3 | 该项目用于测试 Hash worker 在不同线程下的哈希计算速度 4 | 5 | 它同时支持 `浏览器` 环境和 `Node.js` 环境 6 | 7 | ### Usage 8 | 9 | ```ts 10 | import { benchmark, BenchmarkOptions } from 'hash-worker-benchmark' 11 | 12 | // options 是可选的 13 | const options: BenchmarkOptions = {} 14 | benchmark(options) 15 | ``` 16 | 17 | ### Options 18 | 19 | **BenchmarkOptions** 20 | 21 | | filed | type | default | description | 22 | | ------------------- | -------- | --------------------------------------- |---------------------| 23 | | sizeInMB | number | 500 | 用于测试的文件大小 (MB) | 24 | | strategy | Strategy | Strategy.md5 | hash 计算策略 | 25 | | workerCountTobeTest | number[] | [1, 1, 1, 4, 4, 4, 8, 8, 8, 12, 12, 12] | 1/4/8/12 线程下各测试 3 次 | 26 | 27 | ```ts 28 | // strategy.ts 29 | export enum Strategy { 30 | md5 = 'md5', 31 | crc32 = 'crc32', 32 | xxHash64 = 'xxHash64', 33 | mixed = 'mixed', 34 | } 35 | ``` 36 | ### LICENSE 37 | 38 | [MIT](./../../LICENSE) 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Tkunl 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /packages/core/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Tkunl 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /packages/core/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # hash-worker 2 | 3 | ## 2.0.1 4 | 5 | ### Patch Changes 6 | 7 | - 修复发包产物的问题, 在使用了 pnpm 的 catalog 功能时, 应该使用 pnpm publish 而不是 npm publish 8 | 9 | ## 2.0.0 10 | 11 | ### Major Changes 12 | 13 | - feat: 移除了 crc32 和 mix 策略的支持, 因为 crc32 非常容易导致 hash 碰撞, 现在只支持 md5, xxHash64, xxHash128 14 | 15 | - feat: 更新了 getFileHashChunks 方法的相关类型命名 16 | 17 | ## 1.1.3 18 | 19 | ### Patch Changes 20 | 21 | - 1.1.1 版本打包产物没有问题, 运行报错是需要将 hash-worker 排除在 vite 的预构建之外 22 | 23 | - chore: 更新了 playground 中的相关依赖, 更新了子包与主包之间的依赖管理 24 | 25 | ## 1.1.2 26 | 27 | ### Patch Changes 28 | 29 | - 1.1.1 版本重构后, 打包产物存在问题, 在 playground 中未测试出, 从 npm 中安装包或使用 yalc 链接本地打包产物后, 正常使用会报错, 暂时回滚 1.0.1 的代码, 待修复后, 发布 1.1.3 版本 30 | 31 | ## 1.1.1 32 | 33 | ### Minor Changes 34 | 35 | - refator: 重构了 core 包, 拆分不同环境的打包产物, 不需要在配置 Vite/Webpack 中排除 node 相关依赖 36 | 37 | - feat: 现在可以自定义构建 MerkleTree 的 hash 方法 38 | 39 | ## 1.0.1 40 | 41 | ### Minor Changes 42 | 43 | - fix: 当 0 <= chunkSize < 1 时, 导致分片函数死循环的问题 44 | 45 | ## 1.0.0 46 | 47 | ### Major Changes 48 | 49 | - release version 1.0.0 50 | 51 | ## 0.1.3 52 | 53 | ### Minor Changes 54 | 55 | - feat: 添加了 Webpack 下报错的解决方案, 升级了项目到最新的依赖 -------------------------------------------------------------------------------- /packages/playground/react-rsbuild-demo/src/App.tsx: -------------------------------------------------------------------------------- 1 | import React, { useCallback, useRef } from 'react' 2 | import { getFileHashChunks, HashWorkerOptions, HashWorkerResult, Strategy } from 'hash-worker' 3 | 4 | const App = () => { 5 | const fileRef = useRef(null) 6 | 7 | const handleInputChange = useCallback((e: React.ChangeEvent) => { 8 | const files = e.currentTarget.files 9 | if (files?.[0]) { 10 | fileRef.current = files[0] 11 | } 12 | }, []) 13 | 14 | const handleGetHash = useCallback(() => { 15 | const param: HashWorkerOptions = { 16 | file: fileRef.current!, 17 | config: { 18 | workerCount: 6, 19 | strategy: Strategy.md5, 20 | isShowLog: true, 21 | }, 22 | } 23 | 24 | getFileHashChunks(param).then((data: HashWorkerResult) => { 25 | console.log(data) 26 | alert('Calculation complete, please check the console!') 27 | }) 28 | }, []) 29 | 30 | return ( 31 | <> 32 |
Hello
33 | 34 | 35 | 36 | ) 37 | } 38 | 39 | export default App 40 | -------------------------------------------------------------------------------- /commitlint.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | extends: ['@commitlint/config-conventional'], 3 | rules: { 4 | 'type-enum': [ 5 | // type枚举 6 | 2, 7 | 'always', 8 | [ 9 | 'build', // 编译相关的修改，例如发布版本、对项目构建或者依赖的改动 10 | 'feat', // 新功能 11 | 'fix', // 修补bug 12 | 'docs', // 文档修改 13 | 'style', // 代码格式修改, 注意不是 css 修改 14 | 'refactor', // 重构 15 | 'perf', // 优化相关，比如提升性能、体验 16 | 'test', // 测试用例修改 17 | 'revert', // 代码回滚 18 | 'ci', // 持续集成修改 19 | 'config', // 配置修改 20 | 'chore', // 其他改动 21 | ], 22 | ], 23 | 'type-empty': [2, 'never'], // never: type 不能为空; always: type 必须为空 24 | 'type-case': [0, 'always', 'lower-case'], // type 必须小写，upper-case大写，camel-case小驼峰，kebab-case短横线，pascal-case大驼峰，等等 25 | 'scope-empty': [0], 26 | 'scope-case': [0], 27 | 'subject-empty': [2, 'never'], // subject不能为空 28 | 'subject-case': [0], 29 | 'subject-full-stop': [0, 'never', '.'], // subject 以.为结束标记 30 | 'header-max-length': [2, 'always', 72], // header 最长72 31 | 'body-leading-blank': [0], // body 换行 32 | 'footer-leading-blank': [0, 'always'], // footer 以空行开头 33 | }, 34 | } 35 | -------------------------------------------------------------------------------- /packages/playground/vue-vite-demo/src/hooks/useFileHashInfo.ts: -------------------------------------------------------------------------------- 1 | import { ref } from 'vue' 2 | import { 3 | destroyWorkerPool, 4 | getFileHashChunks, 5 | HashWorkerOptions, 6 | HashWorkerResult, 7 | Strategy, 8 | } from 'hash-worker' 9 | 10 | export function useFileHashInfo() { 11 | const file = ref() 12 | 13 | function handleInputChange(e: Event) { 14 | const target = e.target as HTMLInputElement 15 | if (target.files) { 16 | file.value = target.files[0] 17 | } 18 | } 19 | 20 | function handleGetHash() { 21 | const param: HashWorkerOptions = { 22 | file: file.value!, 23 | config: { 24 | workerCount: 6, 25 | strategy: Strategy.md5, 26 | isShowLog: true, 27 | // hashFn: async (hLeft, hRight?) => (hRight ? md5(hLeft + hRight) : hLeft) 28 | }, 29 | } 30 | 31 | getFileHashChunks(param).then((res: HashWorkerResult) => { 32 | console.log(res) 33 | alert('Calculation complete, please check the console!') 34 | }) 35 | } 36 | 37 | function handleDestroyWorkerPool() { 38 | destroyWorkerPool() 39 | } 40 | 41 | return { 42 | handleInputChange, 43 | handleGetHash, 44 | handleDestroyWorkerPool, 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /packages/playground/iife-demo/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | IIFE Demo 8 | 9 | 10 | 11 | 12 | 13 | 35 | 36 |
37 |

Hello

38 |
39 |

如果你在使用 Chrome 浏览器, 并且在控制台日志中发现了报错输出

40 |

这可能是因为你开启了 Vue.js devtools 或 React Developer Tools 插件

41 |

关闭它们, 错误就会消失.

42 |
43 | 44 | 45 |
46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /packages/benchmark/README.md: -------------------------------------------------------------------------------- 1 | ## Introduce for benchmark 2 | 3 | This project is used to test the hash calculation speed of Hash worker in different threads. 4 | 5 | It supports both `Browser` and `Node.js` environments. 6 | 7 | ### Usage 8 | 9 | ```ts 10 | import { benchmark, BenchmarkOptions } from 'hash-worker-benchmark' 11 | 12 | // options is optional. 13 | const options: BenchmarkOptions = {} 14 | benchmark(options) 15 | ``` 16 | 17 | ### Options 18 | 19 | **BenchmarkOptions** 20 | 21 | | filed | type | default | description | 22 | | ------------------- | -------- | --------------------------------------- | -------------------------- | 23 | | sizeInMB | number | 500 | File size for testing (MB) | 24 | | strategy | Strategy | Strategy.md5 | Hash computation strategy | 25 | | workerCountTobeTest | number[] | [1, 1, 1, 4, 4, 4, 8, 8, 8, 12, 12, 12] | Hashing performance was measured 3 times in each of the 1/4/8/12 threads | 26 | 27 | ```ts 28 | // strategy.ts 29 | export enum Strategy { 30 | md5 = 'md5', 31 | crc32 = 'crc32', 32 | xxHash64 = 'xxHash64', 33 | mixed = 'mixed', 34 | } 35 | ``` 36 | ### LICENSE 37 | 38 | [MIT](./../../LICENSE) 39 | -------------------------------------------------------------------------------- /packages/core/src/types/fileHashChunks.ts: -------------------------------------------------------------------------------- 1 | import { Strategy } from '.' 2 | import { HashFn } from '../shared' 3 | 4 | export interface Config { 5 | chunkSize?: number // 分片大小 MB 6 | workerCount?: number // worker 线程数量 7 | strategy?: Strategy // hash 计算策略 8 | isCloseWorkerImmediately?: boolean // 是否在计算 hash 后立即关闭 worker 9 | isShowLog?: boolean // 是否显示 log 10 | hashFn?: HashFn // 自定义 MerkleTree hash 合并方法 11 | timeout?: number // 单个 worker 任务的超时时间（毫秒），默认无超时 12 | } 13 | 14 | export interface FileMetaInfo { 15 | name: string // 文件名 16 | size: number // 文件大小 KB 17 | lastModified: number // 时间戳 18 | type: string // 文件的后缀名 19 | } 20 | 21 | export interface HashWorkerResult { 22 | chunksBlob?: Blob[] // 文件分片的 Blob[] 23 | chunksHash: string[] // 文件分片的 Hash[] 24 | merkleHash: string // 文件的 merkleHash 25 | metadata: FileMetaInfo // 文件的 metadata 26 | } 27 | 28 | interface BaseParam { 29 | config?: Config 30 | } 31 | interface BrowserEnvParam extends BaseParam { 32 | file: File // 待计算 Hash 的文件 (浏览器环境) 33 | filePath?: never // 当 file 存在时，filePath 不能存在 34 | } 35 | interface NodeEnvParam extends BaseParam { 36 | file?: never // 当 filePath 存在时，file 不能存在 37 | filePath: string // 待计算 Hash 的文件的 URL (Node 环境) 38 | } 39 | /** 使用交叉类型确保 file 和 filePath 二者之一必须存在 */ 40 | export type HashWorkerOptions = BrowserEnvParam | NodeEnvParam 41 | -------------------------------------------------------------------------------- /packages/playground/react-webpack-demo/src/index.tsx: -------------------------------------------------------------------------------- 1 | import { getFileHashChunks, HashWorkerOptions, HashWorkerResult, Strategy } from 'hash-worker' 2 | import React, { useCallback, useRef } from 'react' 3 | import ReactDOM from 'react-dom/client' 4 | 5 | function App() { 6 | const fileRef = useRef(null) 7 | 8 | const handleInputChange = useCallback((e: React.ChangeEvent) => { 9 | const files = e.currentTarget.files 10 | if (files?.[0]) { 11 | fileRef.current = files[0] 12 | } 13 | }, []) 14 | 15 | const handleGetHash = useCallback(() => { 16 | const param: HashWorkerOptions = { 17 | file: fileRef.current!, 18 | config: { 19 | workerCount: 6, 20 | strategy: Strategy.md5, 21 | isShowLog: true, 22 | }, 23 | } 24 | 25 | getFileHashChunks(param).then((data: HashWorkerResult) => { 26 | console.log(data) 27 | alert('Calculation complete, please check the console!') 28 | }) 29 | }, []) 30 | 31 | return ( 32 | <> 33 |
Hello
34 | 35 | 36 | 37 | ) 38 | } 39 | 40 | ReactDOM.createRoot(document.querySelector('#app')!).render( 41 | 42 | 43 | , 44 | ) 45 | -------------------------------------------------------------------------------- /packages/core/src/browser/browserUtils.ts: -------------------------------------------------------------------------------- 1 | import { FileMetaInfo } from '../types' 2 | import { getFileExtension } from '../shared/utils' 3 | 4 | /** 5 | * 将 File 转成 ArrayBuffer 6 | * 注意: Blob 无法直接移交到 Worker 中, 所以需要放到主线程中执行 7 | * @param chunks 8 | */ 9 | export async function getArrayBufFromBlobs(chunks: Blob[]) { 10 | return Promise.all(chunks.map((chunk) => chunk.arrayBuffer())) 11 | } 12 | 13 | /** 14 | * 分割文件 15 | * @param file 16 | * @param baseSize 默认分块大小为 1MB 17 | */ 18 | export function sliceFile(file: File, baseSize = 1) { 19 | if (baseSize <= 0) throw Error('baseSize must be greater than 0') 20 | const chunkSize = Math.max(1, baseSize * 1048576) // 1MB = 1024 * 1024 21 | const chunks: Blob[] = [] 22 | let startPos = 0 23 | if (file.size === 0) { 24 | // 空文件返回一个空 Blob 25 | return [file.slice(0, 0)] 26 | } 27 | while (startPos < file.size) { 28 | chunks.push(file.slice(startPos, startPos + chunkSize)) 29 | startPos += chunkSize 30 | } 31 | return chunks 32 | } 33 | 34 | /** 35 | * 获取文件元数据 36 | * @param file 文件 37 | */ 38 | export async function getFileMetadataInBrowser(file: File): Promise { 39 | const fileType = getFileExtension(file.name) 40 | 41 | return { 42 | name: file.name, 43 | size: file.size / 1024, 44 | lastModified: file.lastModified, 45 | type: fileType, 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /packages/benchmark/src/node/nodeHelper.ts: -------------------------------------------------------------------------------- 1 | import { createWriteStream, unlinkSync } from 'fs' 2 | import { randomBytes } from 'crypto' 3 | 4 | export async function createMockFileInLocal(filePath: string, sizeInMB: number): Promise { 5 | const generateRandomData = (size: number) => randomBytes(size).toString('hex') 6 | 7 | const stream = createWriteStream(filePath) 8 | const size = 1024 * 1024 * sizeInMB // 总大小转换为字节 9 | const chunkSize = 1024 * 512 // 每次写入512KB 10 | 11 | let written = 0 12 | 13 | return new Promise((resolve, reject) => { 14 | stream.on('error', reject) 15 | const write = (): void => { 16 | let ok = true 17 | do { 18 | const chunk = generateRandomData(chunkSize > size - written ? size - written : chunkSize) 19 | written += chunk.length / 2 // 更新已写入的长度，除以2因为hex字符串表示的字节长度是实际长度的一半 20 | 21 | if (written >= size) { 22 | // 如果达到或超过预定大小，则写入最后一个块并结束 23 | stream.write(chunk, () => stream.end()) 24 | resolve() 25 | } else { 26 | // 否则，继续写入 27 | ok = stream.write(chunk) 28 | } 29 | } while (written < size && ok) 30 | if (written < size) { 31 | // 'drain' 事件会在可以安全地继续写入数据到流中时触发 32 | stream.once('drain', write) 33 | } 34 | } 35 | write() 36 | }) 37 | } 38 | 39 | export async function deleteLocalFile(path: string) { 40 | unlinkSync(path) 41 | } 42 | -------------------------------------------------------------------------------- /scripts/fileCopier.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs') 2 | const path = require('path') 3 | 4 | /** 5 | * 复制文件的函数 6 | * @param {string} src - 源文件路径 7 | * @param {string} dest - 目标文件路径 8 | */ 9 | function copyFile(src, dest) { 10 | // 如果目标文件已存在，则先删除 11 | if (fs.existsSync(dest)) { 12 | try { 13 | fs.unlinkSync(dest) 14 | console.log(`Deleted existing file at ${dest}`) 15 | } catch (err) { 16 | console.error(`Error deleting file at ${dest}:`, err) 17 | return 18 | } 19 | } 20 | 21 | const readStream = fs.createReadStream(src) 22 | const writeStream = fs.createWriteStream(dest) 23 | 24 | readStream.on('error', (err) => { 25 | console.error(`Error reading file from ${src}:`, err) 26 | }) 27 | 28 | writeStream.on('error', (err) => { 29 | console.error(`Error writing file to ${dest}:`, err) 30 | }) 31 | 32 | writeStream.on('finish', () => { 33 | console.log(`Successfully copied ${src} to ${dest}`) 34 | }) 35 | 36 | readStream.pipe(writeStream) 37 | } 38 | 39 | /** 40 | * 复制多个文件的函数 41 | * @param {Array<{src: string, dest: string}>} files - 包含源文件和目标文件路径的对象数组 42 | */ 43 | function copyFiles(files) { 44 | files.forEach(({ src, dest }) => { 45 | // 确保目标目录存在 46 | const destDir = path.dirname(dest) 47 | if (!fs.existsSync(destDir)) { 48 | fs.mkdirSync(destDir, { recursive: true }) 49 | } 50 | 51 | copyFile(src, dest) 52 | }) 53 | } 54 | 55 | // 使用 module.exports 导出 copyFiles 函数 56 | module.exports = { copyFiles } 57 | -------------------------------------------------------------------------------- /packages/playground/iife-demo/server.js: -------------------------------------------------------------------------------- 1 | const http = require('http') 2 | const fs = require('fs') 3 | const path = require('path') 4 | 5 | const server = http.createServer((req, res) => { 6 | // 获取请求的文件路径 7 | let filePath = '.' + req.url 8 | if (filePath === './') { 9 | filePath = './index.html' 10 | } 11 | 12 | // 获取文件扩展名 13 | const extname = String(path.extname(filePath)).toLowerCase() 14 | const mimeTypes = { 15 | '.html': 'text/html', 16 | '.js': 'application/javascript', 17 | '.mjs': 'application/javascript', 18 | '.css': 'text/css', 19 | '.png': 'image/png', 20 | '.jpg': 'image/jpg', 21 | '.json': 'application/json', 22 | } 23 | 24 | const contentType = mimeTypes[extname] || 'application/octet-stream' 25 | 26 | // 读取文件 27 | fs.readFile(filePath, (error, content) => { 28 | if (error) { 29 | if (error.code === 'ENOENT') { 30 | // 如果文件不存在，返回 404 页面 31 | fs.readFile('./404.html', (error404, content404) => { 32 | res.writeHead(404, { 'Content-Type': 'text/html' }) 33 | res.end(content404, 'utf-8') 34 | }) 35 | } else { 36 | // 其他错误，返回 500 页面 37 | res.writeHead(500) 38 | res.end(`Server Error: ${error.code}`) 39 | } 40 | } else { 41 | // 成功读取文件，返回内容 42 | res.writeHead(200, { 'Content-Type': contentType }) 43 | res.end(content, 'utf-8') 44 | } 45 | }) 46 | }) 47 | 48 | const PORT = process.env.PORT || 8891 49 | server.listen(PORT, () => { 50 | console.log(`Server running at http://localhost:${PORT}/`) 51 | }) 52 | -------------------------------------------------------------------------------- /packages/core/jest.config.ts: -------------------------------------------------------------------------------- 1 | import type { Config } from 'jest' 2 | const projectsConfigWrapper = (configs: Record[]): any[] => 3 | configs.map((config) => ({ 4 | ...config, 5 | coveragePathIgnorePatterns: ['/__tests__/fixture/'], 6 | preset: 'ts-jest', 7 | moduleNameMapper: { 8 | '^(\\.{1,2}/)*src/(.*)$': '/src/$2', 9 | }, 10 | transform: { 11 | '^.+\\.tsx?$': [ 12 | // 为了解决报错: The 'import.meta' meta-property is only allowed when the '--module' option is 'es2020', 'esnext', or 'system'. 13 | // make ts-jest happy ^_^ 14 | 'ts-jest', 15 | { 16 | diagnostics: { 17 | ignoreCodes: [1343], 18 | }, 19 | astTransformers: { 20 | before: [ 21 | { 22 | path: 'ts-jest-mock-import-meta', 23 | // 此处 url 可随便符合 url 格式的字符串, 因为在运行 Jest 时, Worker 会被 Mock Worker 换掉 24 | options: { metaObjectReplacement: { url: 'https://test.com' } }, 25 | }, 26 | ], 27 | }, 28 | }, 29 | ], 30 | }, 31 | })) 32 | 33 | const config: Config = { 34 | collectCoverage: true, 35 | coverageReporters: ['lcov'], 36 | projects: projectsConfigWrapper([ 37 | { 38 | displayName: 'node', 39 | testEnvironment: 'node', 40 | testMatch: ['**/__tests__/node/**/*.spec.ts'], 41 | }, 42 | // TODO: 此处貌似暂时没有用到 43 | { 44 | displayName: 'browser', 45 | testEnvironment: 'jsdom', 46 | testMatch: ['**/__tests__/browser/**/*.spec.ts'], 47 | }, 48 | ]), 49 | } 50 | 51 | export default config 52 | -------------------------------------------------------------------------------- /packages/core/src/shared/helper.ts: -------------------------------------------------------------------------------- 1 | import { DEFAULT_MAX_WORKERS, HASH_FUNCTIONS } from './constant' 2 | import { HashFn, MerkleTree } from './merkleTree' 3 | import { Config, Strategy, WorkerReq, WorkerRes } from '../types' 4 | 5 | export async function getMerkleRootHashByChunks(hashList: string[], hashFn?: HashFn) { 6 | const merkleTree = new MerkleTree(hashFn) 7 | await merkleTree.init(hashList) 8 | return merkleTree.getRootHash() 9 | } 10 | 11 | export function mergeConfig(paramConfig?: Config) { 12 | const { chunkSize, workerCount, strategy, isCloseWorkerImmediately, isShowLog } = 13 | paramConfig ?? {} 14 | 15 | return { 16 | chunkSize: chunkSize ?? 10, 17 | workerCount: workerCount ?? DEFAULT_MAX_WORKERS, 18 | strategy: strategy ?? Strategy.xxHash128, 19 | isCloseWorkerImmediately: isCloseWorkerImmediately ?? true, 20 | isShowLog: isShowLog ?? false, 21 | } 22 | } 23 | 24 | export async function calculateHashInWorker(req: WorkerReq): Promise> { 25 | const { chunk: buf, strategy } = req 26 | const data = new Uint8Array(buf) 27 | 28 | const hashFn = HASH_FUNCTIONS[strategy] 29 | if (!hashFn) { 30 | throw new Error(`calculateHashInWorker: Unsupported strategy: ${strategy}`) 31 | } 32 | 33 | const hash = await hashFn(data) 34 | return { result: hash, chunk: buf } 35 | } 36 | 37 | export async function getChunksHashSingle(strategy: Strategy, arrayBuffer: ArrayBuffer) { 38 | const uint8Array = new Uint8Array(arrayBuffer) 39 | const hashFn = HASH_FUNCTIONS[strategy] 40 | 41 | if (!hashFn) { 42 | throw new Error(`getChunksHashSingle: Unsupported strategy: ${strategy}`) 43 | } 44 | 45 | return [await hashFn(uint8Array)] 46 | } 47 | -------------------------------------------------------------------------------- /packages/benchmark/src/shared/helper.ts: -------------------------------------------------------------------------------- 1 | import { BenchmarkOptions } from './types' 2 | import { Strategy } from 'hash-worker' 3 | 4 | export function normalizeBenchmarkOptions(options: BenchmarkOptions): Required { 5 | const defaultWorkerCountTobeTest = [1, 1, 1, 4, 4, 4, 8, 8, 8, 12, 12, 12] 6 | const { sizeInMB, strategy, workerCountTobeTest } = options 7 | 8 | const normalizeOptions = { 9 | sizeInMB: sizeInMB ?? 500, 10 | strategy: strategy ?? Strategy.md5, 11 | workerCountTobeTest: workerCountTobeTest ?? defaultWorkerCountTobeTest, 12 | } 13 | 14 | const { workerCountTobeTest: _workerCountTobeTest } = normalizeOptions 15 | 16 | if ( 17 | _workerCountTobeTest.length === 0 || 18 | _workerCountTobeTest.find((num: number) => num <= 0 || num > 32 || !Number.isInteger(num)) 19 | ) { 20 | throw new Error('Illegal workerCount') 21 | } 22 | 23 | return normalizeOptions 24 | } 25 | 26 | export async function sleep(ms: number) { 27 | await new Promise((rs) => setTimeout(() => rs(), ms)) 28 | } 29 | 30 | export function createMockFile(fileName: string, sizeInMB: number): File { 31 | // 每 MB 大约为 1048576 字节 32 | const size = sizeInMB * 1048576 33 | const buffer = new ArrayBuffer(size) 34 | const view = new Uint8Array(buffer) 35 | 36 | // 填充随机内容 37 | for (let i = 0; i < size; i++) { 38 | // 随机填充每个字节，这里是填充 0-255 的随机数 39 | // 实际应用中，你可能需要调整生成随机数的方式以达到所需的随机性 40 | view[i] = Math.floor(Math.random() * 256) 41 | } 42 | 43 | // 将 ArrayBuffer 转换为Blob 44 | const blob = new Blob([view], { type: 'application/octet-stream' }) 45 | 46 | // 将 Blob 转换为File 47 | return new File([blob], fileName, { type: 'application/octet-stream' }) 48 | } 49 | -------------------------------------------------------------------------------- /packages/core/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hash-worker", 3 | "version": "2.0.1", 4 | "description": "hash-worker is a tool for quickly calculating file's hash", 5 | "author": "https://github.com/Tkunl", 6 | "repository": "https://github.com/Tkunl/hash-worker", 7 | "type": "module", 8 | "types": "./dist/index.d.ts", 9 | "main": "./dist/index.cjs.js", 10 | "module": "./dist/index.esm.js", 11 | "exports": { 12 | ".": { 13 | "types": "./dist/index.d.ts", 14 | "import": "./dist/index.esm.js", 15 | "require": "./dist/index.cjs.js" 16 | }, 17 | "./browser": { 18 | "types": "./dist/index.d.ts", 19 | "import": "./dist/index.esm.js", 20 | "require": "./dist/index.cjs.js" 21 | }, 22 | "./node": { 23 | "types": "./dist/node.d.ts", 24 | "import": "./dist/node.mjs", 25 | "require": "./dist/node.cjs" 26 | }, 27 | "./global": { 28 | "types": "./dist/global.d.ts", 29 | "default": "./dist/global.js" 30 | }, 31 | "./worker/browser.worker.mjs": "./dist/worker/browser.worker.mjs" 32 | }, 33 | "files": [ 34 | "dist", 35 | "README.md", 36 | "README-zh.md", 37 | "package.json", 38 | "LICENSE" 39 | ], 40 | "scripts": { 41 | "dev": "rollup --config rollup.config.ts --configPlugin swc3 --watch", 42 | "build": "pnpm rm:dist && rollup --config rollup.config.ts --configPlugin swc3", 43 | "test": "jest --coverage", 44 | "rm:dist": "rimraf ./dist" 45 | }, 46 | "keywords": [ 47 | "hash-worker", 48 | "hash" 49 | ], 50 | "license": "MIT", 51 | "dependencies": { 52 | "hash-wasm": "catalog:" 53 | }, 54 | "publishConfig": { 55 | "registry": "https://registry.npmjs.org/", 56 | "access": "public" 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /eslint.config.mjs: -------------------------------------------------------------------------------- 1 | import typescriptEslint from '@typescript-eslint/eslint-plugin' 2 | import prettier from 'eslint-plugin-prettier' 3 | import globals from 'globals' 4 | import tsParser from '@typescript-eslint/parser' 5 | import path from 'node:path' 6 | import { fileURLToPath } from 'node:url' 7 | import js from '@eslint/js' 8 | import { FlatCompat } from '@eslint/eslintrc' 9 | 10 | const __filename = fileURLToPath(import.meta.url) 11 | const __dirname = path.dirname(__filename) 12 | const compat = new FlatCompat({ 13 | baseDirectory: __dirname, 14 | recommendedConfig: js.configs.recommended, 15 | allConfig: js.configs.all, 16 | }) 17 | 18 | export default [ 19 | { 20 | ignores: ['**/node_modules/', '**/output/', '**/dist/', '**/playground/'], 21 | }, 22 | ...compat.extends( 23 | 'eslint:recommended', 24 | 'plugin:@typescript-eslint/recommended', 25 | 'plugin:prettier/recommended', 26 | ), 27 | { 28 | plugins: { 29 | '@typescript-eslint': typescriptEslint, 30 | prettier, 31 | }, 32 | 33 | languageOptions: { 34 | globals: { 35 | ...globals.browser, 36 | ...globals.node, 37 | }, 38 | 39 | parser: tsParser, 40 | ecmaVersion: 2022, 41 | sourceType: 'module', 42 | }, 43 | 44 | rules: { 45 | 'prettier/prettier': 'error', 46 | '@typescript-eslint/no-unused-vars': 'error', 47 | '@typescript-eslint/no-explicit-any': 'off', 48 | '@typescript-eslint/ban-ts-comment': 'off', 49 | '@typescript-eslint/no-require-imports': 'off', 50 | 51 | '@typescript-eslint/no-unused-expressions': [ 52 | 'error', 53 | { 54 | allowShortCircuit: true, 55 | allowTernary: true, 56 | allowTaggedTemplates: true, 57 | }, 58 | ], 59 | }, 60 | }, 61 | ] 62 | -------------------------------------------------------------------------------- /packages/core/src/browser/browserWorkerWrapper.ts: -------------------------------------------------------------------------------- 1 | import { BaseWorkerWrapper, obtainBuf, generateUUID } from '../shared' 2 | import { WorkerReq, TaskConfig } from '../types' 3 | 4 | export class BrowserWorkerWrapper extends BaseWorkerWrapper { 5 | constructor(worker: Worker) { 6 | super(worker) 7 | } 8 | 9 | run(param: WorkerReq, index: number, config?: TaskConfig): Promise { 10 | const taskId = generateUUID() 11 | this.setRunning(taskId) 12 | 13 | return new Promise((resolve, reject) => { 14 | // 设置超时时间 15 | if (config?.timeout) { 16 | this.setTimeout(config.timeout, reject, taskId) 17 | } 18 | 19 | const cleanup = () => { 20 | this.worker.onmessage = null 21 | this.worker.onerror = null 22 | } 23 | 24 | this.worker.onmessage = (event: MessageEvent) => { 25 | cleanup() 26 | this.handleMessage(event.data, resolve, reject, index) 27 | } 28 | this.worker.onerror = (event: ErrorEvent) => { 29 | cleanup() 30 | this.handleError(reject, event.error || new Error('Unknown worker error')) 31 | } 32 | 33 | this.worker.postMessage(param, [obtainBuf(param)]) 34 | }) 35 | } 36 | 37 | protected createTimeout( 38 | timeoutMs: number, 39 | reject: (reason: any) => void, 40 | taskId: string, 41 | ): number { 42 | return window.setTimeout(() => { 43 | if (this.currentTaskId === taskId) { 44 | this.handleError(reject, new Error(`Worker task timeout after ${timeoutMs}ms`)) 45 | } 46 | }, timeoutMs) 47 | } 48 | 49 | protected clearTimeout(timeoutId: number): void { 50 | window.clearTimeout(timeoutId) 51 | } 52 | 53 | protected cleanupEventListeners(): void { 54 | this.worker.onmessage = null 55 | this.worker.onerror = null 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /packages/core/src/node/nodeWorkerWrapper.ts: -------------------------------------------------------------------------------- 1 | import { Worker as NodeWorker } from 'worker_threads' 2 | import { BaseWorkerWrapper, obtainBuf, generateUUID } from '../shared' 3 | import { WorkerReq, TaskConfig } from '../types' 4 | 5 | export class NodeWorkerWrapper extends BaseWorkerWrapper { 6 | constructor(worker: NodeWorker) { 7 | super(worker) 8 | worker.setMaxListeners(1024) 9 | } 10 | 11 | run(param: WorkerReq, index: number, config?: TaskConfig): Promise { 12 | const taskId = generateUUID() 13 | this.setRunning(taskId) 14 | 15 | return new Promise((resolve, reject) => { 16 | // 设置超时时间 17 | if (config?.timeout) { 18 | this.setTimeout(config.timeout, reject, taskId) 19 | } 20 | 21 | const cleanup = () => { 22 | this.worker.removeAllListeners('message') 23 | this.worker.removeAllListeners('error') 24 | } 25 | 26 | this.worker 27 | .on('message', (data) => { 28 | cleanup() 29 | this.handleMessage(data, resolve, reject, index) 30 | }) 31 | .on('error', (error) => { 32 | cleanup() 33 | this.handleError(reject, error) 34 | }) 35 | 36 | this.worker.postMessage(param, [obtainBuf(param)]) 37 | }) 38 | } 39 | 40 | protected createTimeout( 41 | timeoutMs: number, 42 | reject: (reason: any) => void, 43 | taskId: string, 44 | ): NodeJS.Timeout { 45 | return setTimeout(() => { 46 | if (this.currentTaskId === taskId) { 47 | this.handleError(reject, new Error(`Worker task timeout after ${timeoutMs}ms`)) 48 | } 49 | }, timeoutMs) 50 | } 51 | 52 | protected clearTimeout(timeoutId: NodeJS.Timeout): void { 53 | clearTimeout(timeoutId) 54 | } 55 | 56 | protected cleanupEventListeners(): void { 57 | this.worker.removeAllListeners('message') 58 | this.worker.removeAllListeners('error') 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /packages/core/src/shared/arrayBufferService.ts: -------------------------------------------------------------------------------- 1 | import { WorkerReq } from '../types' 2 | 3 | /** 4 | * ArrayBuffer 服务类 5 | * 用于管理 Worker 线程间传输的 ArrayBuffer 对象 6 | */ 7 | class ArrayBufferService { 8 | private arrayBuffers: ArrayBuffer[] = [] 9 | 10 | /** 11 | * 初始化缓冲区数组 12 | * @param buffers - ArrayBuffer 数组 13 | */ 14 | init(buffers: ArrayBuffer[]): void { 15 | if (!Array.isArray(buffers)) { 16 | throw new Error('Buffers must be an array') 17 | } 18 | this.arrayBuffers = [...buffers] // 创建副本以避免外部修改 19 | } 20 | 21 | /** 22 | * 从 WorkerReq 中提取 ArrayBuffer 23 | * @param param - Worker 请求参数 24 | * @returns 提取的 ArrayBuffer 25 | */ 26 | extractArrayBuffer(param: WorkerReq): ArrayBuffer { 27 | if (!param || !param.chunk) { 28 | throw new Error('Invalid WorkerReq: chunk is required') 29 | } 30 | if (!(param.chunk instanceof ArrayBuffer)) { 31 | throw new Error('Invalid chunk: must be an ArrayBuffer') 32 | } 33 | return param.chunk 34 | } 35 | 36 | /** 37 | * 恢复指定索引位置的 ArrayBuffer 38 | * @param options - 包含缓冲区和索引的选项 39 | */ 40 | restoreArrayBuffer(options: { buf: ArrayBuffer; index: number }): void { 41 | const { index, buf } = options 42 | 43 | if (!buf || !(buf instanceof ArrayBuffer)) { 44 | throw new Error('Invalid buffer: must be an ArrayBuffer') 45 | } 46 | 47 | if (!Number.isInteger(index) || index < 0) { 48 | throw new Error('Invalid index: must be a non-negative integer') 49 | } 50 | 51 | if (index >= this.arrayBuffers.length) { 52 | throw new Error(`Index ${index} is out of bounds (array length: ${this.arrayBuffers.length})`) 53 | } 54 | 55 | this.arrayBuffers[index] = buf 56 | } 57 | 58 | /** 59 | * 清空缓冲区数组 60 | */ 61 | clear(): void { 62 | this.arrayBuffers = [] 63 | } 64 | } 65 | 66 | const instance = new ArrayBufferService() 67 | 68 | export const initBufService = (buffers: ArrayBuffer[]) => instance.init(buffers) 69 | export const obtainBuf = (param: WorkerReq) => instance.extractArrayBuffer(param) 70 | export const restoreBuf = (options: { buf: ArrayBuffer; index: number }) => 71 | instance.restoreArrayBuffer(options) 72 | export const clearBufService = () => instance.clear() 73 | -------------------------------------------------------------------------------- /packages/benchmark/src/shared/benchmark.ts: -------------------------------------------------------------------------------- 1 | import { HashWorkerOptions, Strategy } from 'hash-worker' 2 | import { BenchmarkOptions, NormalizeOptions } from './types' 3 | import { sleep } from './helper' 4 | 5 | export abstract class Benchmark { 6 | private preSpeed: number[] = [] 7 | private preWorkerCount = 1 8 | 9 | protected abstract buildParams(options: BenchmarkOptions): NormalizeOptions 10 | protected abstract logInitialMsg(): void 11 | protected abstract logStrategy(strategy?: Strategy): void 12 | protected abstract logAvgSpeed(averageSpeed: number): void 13 | protected abstract logCurSpeed(overTime: number, workerCount: number, speed: number): void 14 | protected abstract logCompletion(): void 15 | protected abstract getFileHashChunks(param: HashWorkerOptions): Promise 16 | 17 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 18 | protected async createMockFileInLocal(sizeInMB: number) {} 19 | protected async deleteLocalFile() {} 20 | 21 | async run(options: BenchmarkOptions = {}) { 22 | this.logInitialMsg() 23 | const normalizeOptions = this.buildParams(options) 24 | const { sizeInMB, params } = normalizeOptions 25 | await this.createMockFileInLocal(sizeInMB) 26 | this.logStrategy(normalizeOptions.params[0].config?.strategy) 27 | 28 | for (const param of params) { 29 | const workerCount = param.config!.workerCount! 30 | workerCount !== this.preWorkerCount && this.getAverageSpeed(workerCount) 31 | const beforeDate = Date.now() 32 | await this.getFileHashChunks(param) 33 | const overTime = Date.now() - beforeDate 34 | const speed = sizeInMB / (overTime / 1000) 35 | workerCount === this.preWorkerCount && this.preSpeed.push(speed) 36 | this.logCurSpeed(overTime, workerCount, speed) 37 | await sleep(1000) 38 | } 39 | this.getAverageSpeed(this.preWorkerCount) 40 | this.deleteLocalFile() 41 | this.logCompletion() 42 | } 43 | 44 | private getAverageSpeed(workerCount = 0) { 45 | if (this.preSpeed.length === 0) return 46 | const averageSpeed = this.preSpeed.reduce((acc, cur) => acc + cur, 0) / this.preSpeed.length 47 | this.logAvgSpeed(averageSpeed) 48 | this.preWorkerCount = workerCount 49 | this.preSpeed.length = 0 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /packages/core/rollup.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'rollup' 2 | import { dts } from 'rollup-plugin-dts' 3 | import { nodeResolve } from '@rollup/plugin-node-resolve' 4 | import { swc } from 'rollup-plugin-swc3' 5 | 6 | const bundleName = 'HashWorker' 7 | 8 | // 一般来说现代项目不需要自行压缩这些 cjs/esm 模块，因为现代构建工具会自动处理 9 | // 其次发包发布压缩的包意义在于减少安装大小，但是实际上这个行为可有可无 10 | // 关于 iife/umd 面向现代的前端提供 iife 就可以了。 11 | // 因此你不需要过多复杂的配置。 12 | 13 | export default defineConfig([ 14 | // 浏览器 esm 产物 15 | { 16 | input: 'src/index.ts', 17 | output: [ 18 | { file: 'dist/index.esm.js', format: 'esm', exports: 'named' }, 19 | { file: 'dist/index.cjs.js', format: 'cjs', exports: 'named' }, 20 | ], 21 | plugins: [nodeResolve(), swc({ sourceMaps: true })], 22 | }, 23 | // 浏览器 esm 类型产物 24 | { 25 | input: 'src/index.ts', 26 | output: { file: 'dist/index.d.ts' }, 27 | plugins: [dts()], 28 | }, 29 | // 浏览器 iife 产物 30 | { 31 | input: 'src/iife.ts', 32 | output: { file: 'dist/global.js', format: 'iife', name: bundleName }, 33 | plugins: [nodeResolve(), swc({ sourceMaps: true })], 34 | }, 35 | // 浏览器 iife 类型产物 36 | { 37 | input: 'src/iife.ts', 38 | output: { file: 'dist/global.d.ts', format: 'es' }, 39 | plugins: [dts()], 40 | external: ['worker_threads'], 41 | }, 42 | // node esm, cjs 产物 43 | { 44 | input: 'src/node.ts', 45 | output: [ 46 | { file: 'dist/node.mjs', format: 'esm', exports: 'named' }, 47 | { file: 'dist/node.cjs', format: 'cjs', exports: 'named' }, 48 | ], 49 | plugins: [nodeResolve(), swc({ sourceMaps: true })], 50 | external: ['worker_threads'], 51 | }, 52 | // node 类型产物 53 | { 54 | input: 'src/node.ts', 55 | output: { file: 'dist/node.d.ts' }, 56 | plugins: [dts()], 57 | external: ['worker_threads'], 58 | }, 59 | // browser worker 60 | { 61 | input: 'src/worker/browser.worker.ts', 62 | output: { file: 'dist/worker/browser.worker.mjs', format: 'esm' }, 63 | plugins: [nodeResolve(), swc({ sourceMaps: true })], 64 | }, 65 | // node worker 66 | { 67 | input: 'src/worker/node.worker.ts', 68 | output: { file: 'dist/worker/node.worker.mjs', format: 'esm' }, 69 | plugins: [nodeResolve(), swc({ sourceMaps: true })], 70 | external: ['worker_threads'], 71 | }, 72 | ]) 73 | -------------------------------------------------------------------------------- /packages/benchmark/src/browser/browserBenchmark.ts: -------------------------------------------------------------------------------- 1 | import { Strategy, HashWorkerOptions, getFileHashChunks } from 'hash-worker' 2 | import { Benchmark } from '../shared/benchmark' 3 | import { BenchmarkOptions, NormalizeOptions } from '../shared/types' 4 | import { createMockFile, normalizeBenchmarkOptions } from '../shared/helper' 5 | import { FILE_NAME } from '../shared/constant' 6 | 7 | class BrowserBenchmark extends Benchmark { 8 | private readonly yellow = 'color: #FFB049;' 9 | 10 | protected buildParams(options: BenchmarkOptions): NormalizeOptions { 11 | const { sizeInMB, strategy, workerCountTobeTest } = normalizeBenchmarkOptions(options) 12 | console.log(`Creating mock file ...`) 13 | const mockFile = createMockFile(FILE_NAME, sizeInMB) 14 | 15 | return { 16 | sizeInMB, 17 | params: workerCountTobeTest.map((workerCount) => ({ 18 | file: mockFile, 19 | config: { 20 | workerCount, 21 | strategy, 22 | isCloseWorkerImmediately: false, 23 | }, 24 | })), 25 | } 26 | } 27 | protected logInitialMsg(): void { 28 | console.log('%cHash Worker Benchmark 🎯', this.yellow) 29 | } 30 | 31 | protected logStrategy(strategy?: Strategy): void { 32 | console.log(`Running benchmark for %c${strategy} %cstrategy 🚀`, this.yellow, '') 33 | } 34 | 35 | protected logAvgSpeed(averageSpeed: number): void { 36 | console.log(`Average speed: %c${averageSpeed.toFixed(2)} Mb/s`, this.yellow) 37 | } 38 | 39 | protected logCurSpeed(overTime: number, workerCount: number, speed: number): void { 40 | console.log( 41 | `Get file hash in: %c${overTime} ms%c by using %c${workerCount} worker%c, speed: %c${speed.toFixed(2)} Mb/s`, 42 | this.yellow, // 为 overTime 设置黄色 43 | '', // 重置为默认颜色 44 | this.yellow, // 为 workerCount 设置黄色 45 | '', // 重置为默认颜色 46 | this.yellow, // 为 speed 设置黄色 47 | ) 48 | } 49 | 50 | protected logCompletion(): void { 51 | console.log('%cDone 🎈', this.yellow) 52 | alert('Please check the console for benchmark information ~') 53 | } 54 | 55 | protected async getFileHashChunks(param: HashWorkerOptions): Promise { 56 | await getFileHashChunks(param) 57 | } 58 | } 59 | 60 | const instance = new BrowserBenchmark() 61 | export const benchmark = instance.run.bind(instance) 62 | -------------------------------------------------------------------------------- /packages/benchmark/src/node/nodeBenchmark.ts: -------------------------------------------------------------------------------- 1 | import * as chalk from 'chalk' 2 | import { HashWorkerOptions, Strategy, getFileHashChunks } from 'hash-worker/node' 3 | import { Benchmark } from '../shared/benchmark' 4 | import { FILE_PATH } from '../shared/constant' 5 | import { BenchmarkOptions, NormalizeOptions } from '../shared/types' 6 | import { normalizeBenchmarkOptions } from '../shared/helper' 7 | import { createMockFileInLocal, deleteLocalFile } from './nodeHelper' 8 | 9 | class NodeBenchmark extends Benchmark { 10 | chalkYellow = chalk.default.hex('#FFB049') 11 | 12 | protected buildParams(options: BenchmarkOptions): NormalizeOptions { 13 | const { sizeInMB, strategy, workerCountTobeTest } = normalizeBenchmarkOptions(options) 14 | return { 15 | sizeInMB, 16 | params: workerCountTobeTest.map((workerCount) => ({ 17 | filePath: FILE_PATH, 18 | config: { 19 | workerCount, 20 | strategy, 21 | isCloseWorkerImmediately: false, 22 | }, 23 | })), 24 | } 25 | } 26 | 27 | protected logInitialMsg(): void { 28 | console.log(`${this.chalkYellow!('Hash Worker Benchmark')} 🎯`) 29 | } 30 | 31 | protected logStrategy(strategy?: Strategy): void { 32 | console.log(`Running benchmark for ${this.chalkYellow!(strategy + ' strategy')} 🚀`) 33 | } 34 | 35 | protected logAvgSpeed(averageSpeed: number): void { 36 | console.log(`Average speed: ${this.chalkYellow!(averageSpeed.toFixed(2) + 'Mb/s')}`) 37 | } 38 | 39 | protected logCurSpeed(overTime: number, workerCount: number, speed: number): void { 40 | console.log( 41 | `Get file hash in: ${this.chalkYellow!(overTime + ' ms')} by using ${this.chalkYellow!(workerCount) + ' worker'}, ` + 42 | `speed: ${this.chalkYellow!(speed.toFixed(2) + ' Mb/s')}`, 43 | ) 44 | } 45 | 46 | protected logCompletion(): void { 47 | console.log(this.chalkYellow!('Done ') + '🎈') 48 | process.exit(0) 49 | } 50 | 51 | protected async getFileHashChunks(param: HashWorkerOptions): Promise { 52 | await getFileHashChunks(param) 53 | } 54 | 55 | override createMockFileInLocal(sizeInMB: number) { 56 | console.log(`Creating mock file ...`) 57 | return createMockFileInLocal(FILE_PATH, sizeInMB) 58 | } 59 | 60 | override deleteLocalFile() { 61 | return deleteLocalFile(FILE_PATH) 62 | } 63 | } 64 | 65 | const instance = new NodeBenchmark() 66 | export const benchmark = instance.run.bind(instance) 67 | -------------------------------------------------------------------------------- /scripts/clear.js: -------------------------------------------------------------------------------- 1 | const rimraf = require('rimraf') 2 | const path = require('path') 3 | 4 | const nodeModulesDir = [ 5 | '', 6 | 'packages/benchmark/', 7 | 'packages/core/', 8 | 'packages/playground/benchmark-demo/', 9 | 'packages/playground/node-demo/', 10 | 'packages/playground/react-webpack-demo/', 11 | 'packages/playground/vue-vite-demo/', 12 | ].map((dir) => dir + 'node_modules') 13 | 14 | const distToBeBundled = [ 15 | 'packages/benchmark/', 16 | 'packages/core/', 17 | 'packages/playground/benchmark-demo/', 18 | 'packages/playground/node-demo/', 19 | ] 20 | 21 | const distDir = distToBeBundled.map((dir) => dir + 'dist') 22 | const turboCacheDir = distToBeBundled.map((dir) => dir + '.turbo') 23 | const iifeDemoDeps = [ 24 | 'packages/playground/iife-demo/global.js', 25 | 'packages/playground/iife-demo/worker', 26 | ] 27 | 28 | const coverageDir = ['packages/core/coverage'] 29 | 30 | // 主函数来删除所有路径，并处理错误 31 | function removePaths(paths) { 32 | paths.forEach((path) => { 33 | try { 34 | rimraf.sync(path) 35 | console.log(`Successfully deleted: ${path}`) 36 | } catch (err) { 37 | console.error(`Failed to delete: ${path}, Error: ${err.message}`) 38 | } 39 | }) 40 | 41 | console.log('All deletion attempts have been processed.') 42 | } 43 | 44 | function processArgs() { 45 | const args = process.argv.slice(2) 46 | let pattern = '' 47 | 48 | args.forEach((arg) => { 49 | const [key, value] = arg.split('=') 50 | if (key === '--pattern') { 51 | pattern = value 52 | } 53 | }) 54 | return pattern 55 | } 56 | 57 | ;(() => { 58 | const startTime = Date.now() // 记录开始时间 59 | const pattern = processArgs() // 获取执行参数 60 | 61 | // 定义目录映射 62 | const dirMap = { 63 | node_modules: nodeModulesDir, 64 | dist: distDir, 65 | cache: turboCacheDir, 66 | coverage: coverageDir, 67 | } 68 | 69 | let pathsToDelete = [] 70 | 71 | if (pattern === 'all') { 72 | pathsToDelete = [ 73 | ...nodeModulesDir, 74 | ...distDir, 75 | ...turboCacheDir, 76 | ...iifeDemoDeps, 77 | ...coverageDir, 78 | ] 79 | } else if (dirMap[pattern]) { 80 | pathsToDelete = dirMap[pattern] 81 | } 82 | 83 | // 解析路径并删除 84 | pathsToDelete = pathsToDelete.map((p) => path.resolve(process.cwd(), p)) 85 | 86 | removePaths(pathsToDelete) 87 | 88 | const endTime = Date.now() // 记录结束时间 89 | const timeTaken = endTime - startTime // 计算总耗时 90 | console.log(`Total time taken: ${timeTaken}ms`) 91 | })() 92 | -------------------------------------------------------------------------------- /packages/core/src/shared/baseWorkerWrapper.ts: -------------------------------------------------------------------------------- 1 | import { restoreBuf } from './arrayBufferService' 2 | import { Reject, Resolve, WorkerReq, WorkerRes, WorkerStatusEnum, TaskConfig } from '../types' 3 | 4 | type WorkerLike = { terminate: () => void } 5 | 6 | export abstract class BaseWorkerWrapper< 7 | TWorker extends WorkerLike = WorkerLike, 8 | TTimeout = NodeJS.Timeout, 9 | > { 10 | status: WorkerStatusEnum 11 | protected worker: TWorker 12 | protected currentTaskId: string | null = null 13 | protected timeoutId: TTimeout | null = null 14 | 15 | constructor(worker: TWorker) { 16 | this.worker = worker 17 | this.status = WorkerStatusEnum.WAITING 18 | } 19 | 20 | abstract run(param: WorkerReq, index: number, config?: TaskConfig): Promise 21 | 22 | terminate() { 23 | this.cleanup() 24 | this.cleanupEventListeners() 25 | this.worker.terminate() 26 | } 27 | 28 | private cleanup() { 29 | if (this.timeoutId) { 30 | this.clearTimeout(this.timeoutId) 31 | this.timeoutId = null 32 | } 33 | this.currentTaskId = null 34 | this.status = WorkerStatusEnum.WAITING 35 | } 36 | 37 | protected abstract cleanupEventListeners(): void 38 | 39 | protected setRunning(taskId: string) { 40 | this.currentTaskId = taskId 41 | this.status = WorkerStatusEnum.RUNNING 42 | } 43 | 44 | protected setError() { 45 | this.status = WorkerStatusEnum.ERROR 46 | this.cleanup() 47 | } 48 | 49 | protected isCurrentTask(taskId: string): boolean { 50 | return this.currentTaskId === taskId 51 | } 52 | 53 | protected handleMessage( 54 | workerRes: WorkerRes, 55 | resolve: Resolve, 56 | reject: Reject, 57 | index: number, 58 | ) { 59 | try { 60 | restoreBuf({ buf: workerRes.chunk, index }) 61 | this.cleanup() 62 | resolve(workerRes.result) 63 | } catch (error) { 64 | this.handleError( 65 | reject, 66 | error instanceof Error ? error : new Error('Unknown error in handleMessage'), 67 | ) 68 | } 69 | } 70 | 71 | protected handleError(reject: Reject, error: Error) { 72 | this.setError() 73 | reject(error) 74 | } 75 | 76 | protected setTimeout(timeoutMs: number, reject: Reject, taskId: string) { 77 | this.timeoutId = this.createTimeout(timeoutMs, reject, taskId) 78 | } 79 | 80 | protected abstract createTimeout(timeoutMs: number, reject: Reject, taskId: string): TTimeout 81 | protected abstract clearTimeout(timeoutId: TTimeout): void 82 | } 83 | -------------------------------------------------------------------------------- /packages/core/src/shared/baseHashWorker.ts: -------------------------------------------------------------------------------- 1 | import { WorkerService } from './workerService' 2 | import { Config, FileMetaInfo, HashWorkerOptions, HashWorkerResult } from '../types' 3 | 4 | type ProcessFileProps = { 5 | file?: File 6 | filePath?: string 7 | config: Required 8 | } 9 | 10 | type ProcessFileResult = Promise<{ chunksBlob?: Blob[]; chunksHash: string[]; fileHash: string }> 11 | 12 | type GetFileMetadataProps = { 13 | file?: File 14 | filePath?: string 15 | } 16 | 17 | export abstract class BaseHashWorker { 18 | protected workerService: WorkerService | null = null 19 | protected curWorkerCount: number = 0 20 | 21 | protected abstract normalizeParams(param: HashWorkerOptions): Required 22 | protected abstract processFile({ file, filePath, config }: ProcessFileProps): ProcessFileResult 23 | protected abstract getFileMetadata({ 24 | file, 25 | filePath, 26 | }: GetFileMetadataProps): Promise 27 | protected abstract createWorkerService(workerCount: number): WorkerService 28 | 29 | async getFileHashChunks(param: HashWorkerOptions): Promise { 30 | const { config, file, filePath } = this.normalizeParams(param) 31 | const requiredConfig = config as Required 32 | const { isCloseWorkerImmediately, isShowLog, workerCount } = requiredConfig 33 | if (this.workerService === null) { 34 | this.workerService = this.createWorkerService(workerCount) 35 | this.curWorkerCount = workerCount 36 | } 37 | if (this.curWorkerCount !== workerCount) { 38 | this.workerService.adjustWorkerPoolSize(workerCount) 39 | this.curWorkerCount = workerCount 40 | } 41 | const metadata = await this.getFileMetadata({ file, filePath }) 42 | 43 | let beforeTime: number = 0 44 | let overTime: number = 0 45 | isShowLog && (beforeTime = Date.now()) 46 | const fileInfo = await this.processFile({ 47 | file, 48 | filePath, 49 | config: requiredConfig, 50 | }) 51 | isShowLog && (overTime = Date.now() - beforeTime) 52 | isShowLog && 53 | console.log( 54 | `Generated file Merkle hash in ${overTime}ms using ${config.workerCount} worker(s) with ${requiredConfig.strategy} strategy, processing speed: ${(metadata.size / 1024 / (overTime / 1000)).toFixed(2)} MB/s`, 55 | ) 56 | isCloseWorkerImmediately && this.destroyWorkerPool() 57 | 58 | return { 59 | chunksBlob: fileInfo.chunksBlob, 60 | chunksHash: fileInfo.chunksHash, 61 | merkleHash: fileInfo.fileHash, 62 | metadata, 63 | } 64 | } 65 | 66 | destroyWorkerPool() { 67 | this.workerService && this.workerService.terminate() 68 | this.workerService = null 69 | this.curWorkerCount = 0 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /packages/core/src/browser/browserHashWorker.ts: -------------------------------------------------------------------------------- 1 | import { 2 | BaseHashWorker, 3 | getArrParts, 4 | getChunksHashSingle, 5 | getMerkleRootHashByChunks, 6 | mergeConfig, 7 | runAsyncFuncSerialized, 8 | WorkerService, 9 | } from '../shared' 10 | import { Config, HashWorkerOptions, RequiredWithExclude } from '../types' 11 | import { getArrayBufFromBlobs, getFileMetadataInBrowser, sliceFile } from './browserUtils' 12 | import { BrowserWorkerPool } from './browserWorkerPool' 13 | 14 | class BrowserHashWorker extends BaseHashWorker { 15 | protected createWorkerService(workerCount: number): WorkerService { 16 | return new WorkerService(new BrowserWorkerPool(workerCount)) 17 | } 18 | 19 | protected normalizeParams(param: HashWorkerOptions) { 20 | if (!param.file) { 21 | throw new Error('The file attribute is required in browser environment') 22 | } 23 | 24 | return >{ 25 | ...param, 26 | config: mergeConfig(param.config), 27 | } 28 | } 29 | 30 | protected async processFile({ 31 | file, 32 | config, 33 | }: { 34 | file?: File 35 | config: RequiredWithExclude 36 | }) { 37 | const _file = file! 38 | const { chunkSize, strategy, workerCount, hashFn, timeout } = config 39 | 40 | const chunksBlob = sliceFile(_file, chunkSize) 41 | let chunksHash: string[] = [] 42 | 43 | const singleChunkProcessor = async () => { 44 | const arrayBuffer = await chunksBlob[0].arrayBuffer() 45 | chunksHash = await getChunksHashSingle(strategy, arrayBuffer) 46 | } 47 | 48 | const multipleChunksProcessor = async () => { 49 | let chunksBuf: ArrayBuffer[] = [] 50 | // 将文件分片进行分组, 组内任务并行执行, 组外任务串行执行 51 | const chunksPart = getArrParts(chunksBlob, workerCount) 52 | const tasks = chunksPart.map((part) => async () => { 53 | // 手动释放上一次用于计算 Hash 的 ArrayBuffer 54 | chunksBuf.length = 0 55 | chunksBuf = await getArrayBufFromBlobs(part) 56 | // 传递超时配置给 getHashForFiles 57 | const taskConfig = timeout ? { timeout } : undefined 58 | return this.workerService!.getHashForFiles(chunksBuf, strategy, taskConfig) 59 | }) 60 | 61 | chunksHash = await runAsyncFuncSerialized(tasks) 62 | chunksBuf.length = 0 63 | } 64 | 65 | chunksBlob.length === 1 ? await singleChunkProcessor() : await multipleChunksProcessor() 66 | const fileHash = await getMerkleRootHashByChunks(chunksHash, hashFn) 67 | 68 | return { 69 | chunksBlob, 70 | chunksHash, 71 | fileHash, 72 | } 73 | } 74 | 75 | protected getFileMetadata({ file }: { file?: File }) { 76 | return getFileMetadataInBrowser(file!) 77 | } 78 | } 79 | 80 | const instance = new BrowserHashWorker() 81 | export const getFileHashChunks = instance.getFileHashChunks.bind(instance) 82 | export const destroyWorkerPool = instance.destroyWorkerPool.bind(instance) 83 | -------------------------------------------------------------------------------- /packages/core/src/shared/workerService.ts: -------------------------------------------------------------------------------- 1 | import { BaseWorkerPool } from './baseWorkerPool' 2 | import { clearBufService, initBufService } from './arrayBufferService' 3 | import { Strategy, WorkerReq, TaskConfig } from '../types' 4 | 5 | /** 6 | * 工作服务类，封装了基于 Worker 的批量哈希计算功能 7 | */ 8 | export class WorkerService { 9 | private pool: BaseWorkerPool | null = null 10 | 11 | constructor(pool: BaseWorkerPool) { 12 | this.pool = pool 13 | } 14 | 15 | /** 16 | * 为文件块计算哈希值 17 | * @param chunks - 要计算哈希的数据块数组 18 | * @param strategy - 哈希计算策略 19 | * @param config - 任务配置（可选） 20 | * @returns 返回每个块对应的哈希值数组 21 | * @throws 如果有任务失败，会抛出包含详细错误信息的错误 22 | */ 23 | async getHashForFiles( 24 | chunks: ArrayBuffer[], 25 | strategy: Strategy, 26 | config?: TaskConfig, 27 | ): Promise { 28 | if (!this.pool) { 29 | throw new Error('WorkerService has been terminated') 30 | } 31 | 32 | if (chunks.length === 0) { 33 | return [] 34 | } 35 | 36 | const params: WorkerReq[] = chunks.map((chunk) => ({ 37 | chunk, 38 | strategy, 39 | })) 40 | 41 | // 初始化缓冲区服务以支持 Worker 间的数据传输 42 | initBufService(chunks) 43 | 44 | const results = await this.pool.exec(params, config) 45 | 46 | // 收集所有错误和成功结果 47 | const hashResults: string[] = [] 48 | const errors: Array<{ index: number; error: Error }> = [] 49 | 50 | for (const result of results) { 51 | if (result.success) { 52 | hashResults[result.index] = result.data 53 | } else { 54 | errors.push({ index: result.index, error: result.error }) 55 | } 56 | } 57 | 58 | // 如果有错误，抛出包含所有错误信息的详细错误 59 | if (errors.length > 0) { 60 | const errorMessage = errors 61 | .map(({ index, error }) => `Chunk ${index}: ${error.message}`) 62 | .join('; ') 63 | throw new Error(`Hash calculation failed for ${errors.length} chunks: ${errorMessage}`) 64 | } 65 | 66 | clearBufService() 67 | return hashResults 68 | } 69 | 70 | /** 71 | * 调整工作池中的 Worker 数量 72 | * @param workerCount - 新的 Worker 数量 73 | * @throws 如果服务已被终止，会抛出错误 74 | */ 75 | adjustWorkerPoolSize(workerCount: number): void { 76 | if (!this.pool) { 77 | throw new Error('WorkerService has been terminated') 78 | } 79 | 80 | if (workerCount < 1) { 81 | throw new Error('Worker count must be at least 1') 82 | } 83 | 84 | this.pool.adjustPool(workerCount) 85 | } 86 | 87 | /** 88 | * 获取工作池状态信息 89 | * @returns 工作池的状态统计信息，如果服务已终止则返回 null 90 | */ 91 | getPoolStatus() { 92 | return this.pool?.getPoolStatus() || null 93 | } 94 | 95 | /** 96 | * 检查服务是否处于活动状态 97 | * @returns 如果服务可用则返回 true，否则返回 false 98 | */ 99 | isActive(): boolean { 100 | return this.pool !== null 101 | } 102 | 103 | /** 104 | * 终止工作服务，清理所有资源 105 | */ 106 | terminate(): void { 107 | if (this.pool) { 108 | this.pool.terminate() 109 | this.pool = null 110 | } 111 | clearBufService() 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "private": true, 3 | "packageManager": "pnpm@10.13.1", 4 | "scripts": { 5 | "dev:core": "pnpm -F hash-worker run dev", 6 | "dev:benchmark": "pnpm -F hash-worker-benchmark run dev", 7 | "build:core": "turbo run build", 8 | "build:benchmark": "turbo run build:benchmark", 9 | "build:node-demo": "turbo run build:node-demo", 10 | "build:benchmark-demo": "turbo run build:benchmark-demo", 11 | "build:all": "turbo run build:all", 12 | "play-benchmark": "pnpm run build:benchmark-demo && pnpm -F benchmark run play", 13 | "play-node": "pnpm run build:node-demo && pnpm -F node-demo run play", 14 | "play-iife": "pnpm run build:core && pnpm -F browser-demo run play", 15 | "play-vue-vite": "pnpm run build:core && pnpm -F vue-demo run play", 16 | "play-react-webpack": "pnpm run build:core && pnpm -F react-webpack-demo run play", 17 | "play-react-rsbuild": "pnpm run build:core && pnpm -F react-rsbuild-demo run play", 18 | "check-updates": "pnpm outdated", 19 | "check-updates:vue-demo": "pnpm -F vue-demo outdated", 20 | "check-updates:react-demo": "pnpm -F react-demo outdated", 21 | "test": "pnpm -F hash-worker run test", 22 | "lint": "eslint --fix", 23 | "format": "prettier --write '**/*.{js,jsx,ts,tsx,json}'", 24 | "check-format": "prettier --check '**/*.{js,jsx,ts,tsx,json}'", 25 | "prepare": "husky", 26 | "pre-commit": "lint-staged", 27 | "commitlint": "commitlint --config commitlint.config.js -e -V", 28 | "sync-readme-to-core": "node scripts/syncReadme.js", 29 | "clear:node_modules": "node scripts/clear.js --pattern=node_modules", 30 | "clear:dist": "node scripts/clear.js --pattern=dist", 31 | "clear:cache": "node scripts/clear.js --pattern=cache", 32 | "clear:coverage": "node scripts/clear.js --pattern=coverage", 33 | "clear:all": "node scripts/clear.js --pattern=all" 34 | }, 35 | "keywords": [ 36 | "hash-worker", 37 | "hash" 38 | ], 39 | "author": "Tkunl", 40 | "license": "MIT", 41 | "devDependencies": { 42 | "@commitlint/cli": "^19.8.0", 43 | "@commitlint/config-conventional": "^19.8.0", 44 | "@eslint/eslintrc": "^3.3.1", 45 | "@eslint/js": "^9.25.0", 46 | "@jest/types": "^29.6.3", 47 | "@rollup/plugin-node-resolve": "^16.0.1", 48 | "@swc/core": "^1.11.21", 49 | "@types/jest": "^29.5.12", 50 | "@types/node": "^22.16.0", 51 | "@typescript-eslint/eslint-plugin": "^8.30.1", 52 | "@typescript-eslint/parser": "^8.30.1", 53 | "browserslist": "^4.24.4", 54 | "chalk": "^5.4.1", 55 | "eslint": "^9.25.0", 56 | "eslint-config-prettier": "^10.1.2", 57 | "eslint-plugin-prettier": "^5.2.6", 58 | "globals": "^16.0.0", 59 | "husky": "^9.1.7", 60 | "jest": "^29.7.0", 61 | "jest-environment-jsdom": "^29.7.0", 62 | "lint-staged": "^15.5.1", 63 | "prettier": "^3.5.3", 64 | "rimraf": "catalog:", 65 | "rollup": "^4.40.0", 66 | "ts-node": "catalog:", 67 | "rollup-plugin-dts": "^6.2.1", 68 | "rollup-plugin-swc3": "^0.12.1", 69 | "ts-jest": "^29.1.5", 70 | "ts-jest-mock-import-meta": "^1.2.0", 71 | "tsup": "catalog:", 72 | "tsx": "^4.19.3", 73 | "turbo": "^2.5.0", 74 | "typescript": "catalog:" 75 | }, 76 | "pnpm": { 77 | "onlyBuiltDependencies": [ 78 | "@swc/core", 79 | "esbuild" 80 | ] 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /packages/core/src/node/nodeUtils.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs' 2 | import fsp from 'fs/promises' 3 | import path from 'path' 4 | import { FileMetaInfo } from '../types' 5 | import { getFileExtension } from '../shared/utils' 6 | 7 | /** 8 | * 读取一个文件并将它转成 ArrayBuffer 9 | * @param filePath 文件路径 10 | * @param start 起始位置(字节) 11 | * @param end 结束位置(字节) 12 | */ 13 | export async function readFileAsArrayBuffer( 14 | filePath: string, 15 | start: number, 16 | end: number, 17 | ): Promise { 18 | try { 19 | const readStream = fs.createReadStream(filePath, { start, end }) 20 | const chunks: any[] = [] 21 | return new Promise((resolve, reject) => { 22 | readStream.on('data', (chunk) => { 23 | chunks.push(chunk) // 收集数据块 24 | }) 25 | 26 | readStream.on('end', () => { 27 | const buf = Buffer.concat(chunks) // 合并所有数据块构成 Buffer 28 | const arrayBuf = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) 29 | resolve(arrayBuf) 30 | }) 31 | 32 | readStream.on('error', (error) => { 33 | reject(new Error(`Failed to read file: ${error.message}`)) 34 | }) 35 | }) 36 | } catch (error) { 37 | throw new Error( 38 | `Failed to create read stream: ${error instanceof Error ? error.message : 'Unknown error'}`, 39 | ) 40 | } 41 | } 42 | 43 | /** 44 | * 分割文件, 获取每个分片的起止位置 45 | * @param filePath 文件路径 46 | * @param baseSize 默认分块大小为 1MB 47 | */ 48 | export async function getFileSliceLocations(filePath: string, baseSize = 1) { 49 | if (baseSize <= 0) throw new Error('baseSize must be greater than 0') 50 | 51 | try { 52 | const chunkSize = Math.max(1, baseSize * 1048576) // 1MB = 1024 * 1024 53 | const stats = await fsp.stat(filePath) 54 | const fileSize = stats.size // Bytes 字节 55 | const sliceLocation: [number, number][] = [] 56 | 57 | for (let cur = 0; cur < fileSize; cur += chunkSize) { 58 | const end = Math.min(cur + chunkSize - 1, fileSize - 1) 59 | sliceLocation.push([cur, end]) 60 | } 61 | 62 | return { sliceLocation, endLocation: fileSize } 63 | } catch (error) { 64 | throw new Error( 65 | `Failed to get file slice locations: ${error instanceof Error ? error.message : 'Unknown error'}`, 66 | ) 67 | } 68 | } 69 | 70 | /** 71 | * 获取文件元数据 72 | * @param filePath 文件路径 73 | */ 74 | export async function getFileMetadata(filePath: string): Promise { 75 | try { 76 | const stats = await fsp.stat(filePath) 77 | const fileName = path.basename(filePath) 78 | 79 | // 使用改进的文件扩展名检测逻辑 80 | const fileType = getFileExtension(fileName) 81 | 82 | return { 83 | name: fileName, 84 | size: stats.size / 1024, // 转换为 KB，与接口文档保持一致 85 | lastModified: stats.mtime.getTime(), 86 | type: fileType, 87 | } 88 | } catch (error) { 89 | if (error instanceof Error && 'code' in error) { 90 | const fsError = error as NodeJS.ErrnoException 91 | if (fsError.code === 'ENOENT') { 92 | throw new Error(`File not found: ${filePath}`) 93 | } else if (fsError.code === 'EACCES') { 94 | throw new Error(`Permission denied: ${filePath}`) 95 | } 96 | } 97 | throw new Error( 98 | `Failed to get file metadata: ${error instanceof Error ? error.message : 'Unknown error'}`, 99 | ) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /packages/core/src/node/nodeHashWorker.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs' 2 | import path from 'path' 3 | import { 4 | BaseHashWorker, 5 | getArrParts, 6 | getChunksHashSingle, 7 | getMerkleRootHashByChunks, 8 | mergeConfig, 9 | runAsyncFuncSerialized, 10 | WorkerService, 11 | } from '../shared' 12 | import { Config, HashWorkerOptions, RequiredWithExclude } from '../types' 13 | import { NodeWorkerPool } from './nodeWorkerPool' 14 | import { getFileMetadata, getFileSliceLocations, readFileAsArrayBuffer } from './nodeUtils' 15 | 16 | class NodeHashWorker extends BaseHashWorker { 17 | protected createWorkerService(workerCount: number): WorkerService { 18 | return new WorkerService(new NodeWorkerPool(workerCount)) 19 | } 20 | 21 | protected normalizeParams(param: HashWorkerOptions) { 22 | if (!param.filePath) { 23 | throw new Error('The filePath attribute is required in node environment') 24 | } 25 | let _filePath = param.filePath 26 | try { 27 | if (!path.isAbsolute(_filePath)) { 28 | _filePath = path.resolve(_filePath) 29 | } 30 | const stats = fs.statSync(_filePath) 31 | if (!stats.isFile()) { 32 | throw new Error('Invalid filePath: Path does not point to a file') 33 | } 34 | } catch (err) { 35 | const error = err as NodeJS.ErrnoException 36 | if (error.code === 'ENOENT') { 37 | throw new Error('Invalid filePath: File does not exist') 38 | } 39 | throw err 40 | } 41 | return >{ 42 | ...param, 43 | config: mergeConfig(param.config), 44 | } 45 | } 46 | 47 | protected async processFile({ 48 | filePath, 49 | config, 50 | }: { 51 | filePath?: string 52 | config: RequiredWithExclude 53 | }) { 54 | const { chunkSize, strategy, workerCount, hashFn, timeout } = config 55 | const _filePath = filePath! 56 | 57 | // 文件分片 58 | const { sliceLocation, endLocation } = await getFileSliceLocations(_filePath, chunkSize) 59 | let chunksHash: string[] = [] 60 | 61 | const singleChunkProcessor = async () => { 62 | const arrayBuffer = await readFileAsArrayBuffer(_filePath, 0, endLocation) 63 | chunksHash = await getChunksHashSingle(strategy, arrayBuffer) 64 | } 65 | 66 | const multipleChunksProcessor = async () => { 67 | let chunksBuf: ArrayBuffer[] = [] 68 | // 分组后的起始分割位置 69 | const sliceLocationPart = getArrParts<[number, number]>(sliceLocation, workerCount) 70 | const tasks = sliceLocationPart.map((partArr) => async () => { 71 | // 手动释放上一次用于计算 Hash 的 ArrayBuffer 72 | chunksBuf.length = 0 73 | chunksBuf = await Promise.all( 74 | partArr.map((part) => readFileAsArrayBuffer(_filePath, part[0], part[1])), 75 | ) 76 | 77 | // 传递超时配置给 getHashForFiles 78 | const taskConfig = timeout ? { timeout } : undefined 79 | return this.workerService!.getHashForFiles(chunksBuf, strategy, taskConfig) 80 | }) 81 | 82 | chunksHash = await runAsyncFuncSerialized(tasks) 83 | chunksBuf.length = 0 84 | } 85 | 86 | sliceLocation.length === 1 ? await singleChunkProcessor() : await multipleChunksProcessor() 87 | const fileHash = await getMerkleRootHashByChunks(chunksHash, hashFn) 88 | 89 | return { 90 | chunksHash, 91 | fileHash, 92 | } 93 | } 94 | 95 | protected getFileMetadata({ filePath }: { filePath?: string }) { 96 | return getFileMetadata(filePath!) 97 | } 98 | } 99 | 100 | const instance = new NodeHashWorker() 101 | export const getFileHashChunks = instance.getFileHashChunks.bind(instance) 102 | export const destroyWorkerPool = instance.destroyWorkerPool.bind(instance) 103 | -------------------------------------------------------------------------------- /packages/core/src/shared/utils.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * @param chunks 原始数组 3 | * @param size 分 part 大小 4 | * @example 5 | * [1, 2, 3, 4] => [[1, 2], [3, 4]] 6 | */ 7 | export function getArrParts(chunks: T[], size: number): T[][] { 8 | // 添加输入验证 9 | if (!Array.isArray(chunks)) { 10 | throw new TypeError('chunks must be an array') 11 | } 12 | 13 | if (!Number.isInteger(size) || size <= 0) { 14 | throw new RangeError('size must be a positive integer') 15 | } 16 | 17 | // 空数组直接返回 18 | if (chunks.length === 0) { 19 | return [] 20 | } 21 | 22 | const result: T[][] = [] 23 | for (let i = 0; i < chunks.length; i += size) { 24 | result.push(chunks.slice(i, i + size)) 25 | } 26 | return result 27 | } 28 | 29 | /** 30 | * 按顺序串行执行多个返回数组的异步函数，并合并所有结果到一个扁平数组中 31 | * @param tasks - 由异步函数组成的数组，每个函数需返回一个 Promise，其解析值为 T 类型的数组 32 | * @returns Promise 对象，解析后为所有任务结果的合并数组（T 类型） 33 | * @throws {TypeError} 当 tasks 不是数组时 34 | * @throws {Error} 当某个任务执行失败时 35 | * @example 36 | * runAsyncFuncSerialized([ 37 | * () => Promise.resolve([1, 2]), 38 | * () => Promise.resolve([3, 4]) 39 | * ]).then(console.log); // 输出 [1, 2, 3, 4] 40 | */ 41 | export async function runAsyncFuncSerialized(tasks: (() => Promise)[]): Promise { 42 | // 输入验证 43 | if (!Array.isArray(tasks)) { 44 | throw new TypeError('tasks must be an array') 45 | } 46 | 47 | // 空数组直接返回 48 | if (tasks.length === 0) { 49 | return [] 50 | } 51 | 52 | const results: T[] = [] 53 | 54 | for (let i = 0; i < tasks.length; i++) { 55 | const task = tasks[i] 56 | 57 | // 验证每个任务是否为函数 58 | if (typeof task !== 'function') { 59 | throw new TypeError(`Task at index ${i} is not a function`) 60 | } 61 | 62 | try { 63 | const result = await task() 64 | 65 | // 验证结果是否为数组 66 | if (!Array.isArray(result)) { 67 | throw new TypeError(`Task at index ${i} did not return an array`) 68 | } 69 | 70 | results.push(...result) 71 | } catch (error) { 72 | // 包装错误信息，提供更多上下文 73 | throw new Error( 74 | `Task at index ${i} failed: ${error instanceof Error ? error.message : String(error)}`, 75 | ) 76 | } 77 | } 78 | 79 | return results 80 | } 81 | 82 | /** 83 | * 生成符合 UUID v4 标准的唯一标识符 84 | * @returns 标准格式的 UUID 字符串 (xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx) 85 | * @example 86 | * generateUUID(); // 输出类似 "f47ac10b-58cc-4372-a567-0e02b2c3d479" 87 | */ 88 | export function generateUUID(): string { 89 | return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function (c) { 90 | const r = (Math.random() * 16) | 0 91 | const v = c === 'x' ? r : (r & 0x3) | 0x8 92 | return v.toString(16) 93 | }) 94 | } 95 | 96 | /** 97 | * 获取文件的完整扩展名，支持复合扩展名 98 | * @param fileName 文件名 99 | * @returns 文件扩展名，如 .txt, .tar.gz, .min.js 等 100 | * @example 101 | * getFileExtension('test.txt'); // 输出 ".txt" 102 | * getFileExtension('archive.tar.gz'); // 输出 ".tar.gz" 103 | * getFileExtension('script.min.js'); // 输出 ".min.js" 104 | * getFileExtension('.hidden'); // 输出 "" 105 | */ 106 | export function getFileExtension(fileName: string): string { 107 | if (!fileName || !fileName.includes('.') || fileName.startsWith('.') || fileName.endsWith('.')) { 108 | return '' 109 | } 110 | 111 | // 常见的复合扩展名列表（按从长到短排序以确保正确匹配） 112 | const compoundExtensions = [ 113 | '.tar.gz', 114 | '.tar.bz2', 115 | '.tar.xz', 116 | '.tar.lz', 117 | '.tar.Z', 118 | '.min.js', 119 | '.min.css', 120 | '.min.html', 121 | '.spec.js', 122 | '.spec.ts', 123 | '.test.js', 124 | '.test.ts', 125 | '.d.ts', 126 | '.map.js', 127 | '.backup.sql', 128 | '.log.gz', 129 | ] 130 | 131 | const lowerFileName = fileName.toLowerCase() 132 | 133 | // 检查是否匹配复合扩展名 134 | for (const ext of compoundExtensions) { 135 | if (lowerFileName.endsWith(ext)) { 136 | return ext 137 | } 138 | } 139 | 140 | // 如果没有匹配的复合扩展名，使用单一扩展名逻辑 141 | const lastDot = fileName.lastIndexOf('.') 142 | if (lastDot > 0 && lastDot < fileName.length - 1) { 143 | return '.' + fileName.slice(lastDot + 1) 144 | } 145 | 146 | return '' 147 | } 148 | -------------------------------------------------------------------------------- /packages/core/src/shared/merkleTree.ts: -------------------------------------------------------------------------------- 1 | import { md5 } from 'hash-wasm' 2 | 3 | /** 4 | * 表示 Merkle 树中的节点 5 | */ 6 | interface IMerkleNode { 7 | /** 节点的哈希值 */ 8 | hash: string 9 | /** 左子节点 */ 10 | left: IMerkleNode | null 11 | /** 右子节点 */ 12 | right: IMerkleNode | null 13 | } 14 | 15 | /** 16 | * 表示完整的 Merkle 树结构 17 | */ 18 | interface IMerkleTree { 19 | /** 树的根节点 */ 20 | root: IMerkleNode 21 | /** 树的叶子节点 */ 22 | leafs: IMerkleNode[] 23 | } 24 | 25 | /** 26 | * Merkle 树节点的实现 27 | */ 28 | export class MerkleNode implements IMerkleNode { 29 | hash: string 30 | left: IMerkleNode | null 31 | right: IMerkleNode | null 32 | 33 | constructor(hash: string, left: IMerkleNode | null = null, right: IMerkleNode | null = null) { 34 | if (!hash || typeof hash !== 'string') { 35 | throw new Error('哈希值必须是非空字符串') 36 | } 37 | this.hash = hash 38 | this.left = left 39 | this.right = right 40 | } 41 | } 42 | 43 | /** 44 | * 用于组合两个哈希值的哈希函数类型 45 | */ 46 | export type HashFn = (leftHash: string, rightHash?: string) => Promise 47 | 48 | /** 49 | * Merkle 树的实现 50 | */ 51 | export class MerkleTree implements IMerkleTree { 52 | root!: IMerkleNode // 使用延迟初始化，避免在构造函数中创建无效节点 53 | leafs: IMerkleNode[] = [] 54 | private hashFn: HashFn = async (leftHash, rightHash?) => 55 | rightHash ? await md5(leftHash + rightHash) : leftHash 56 | 57 | constructor(hashFn?: HashFn) { 58 | if (hashFn) { 59 | this.hashFn = hashFn 60 | } 61 | } 62 | 63 | /** 64 | * 使用哈希字符串初始化树 65 | */ 66 | async init(hashList: string[]): Promise 67 | /** 68 | * 使用现有节点初始化树 69 | */ 70 | async init(leafNodes: IMerkleNode[]): Promise 71 | async init(nodes: string[] | IMerkleNode[]): Promise { 72 | if (!Array.isArray(nodes)) { 73 | throw new Error('输入必须是数组') 74 | } 75 | 76 | if (nodes.length === 0) { 77 | throw new Error('无法使用空输入创建 Merkle 树') 78 | } 79 | 80 | // 验证输入数据 81 | if (typeof nodes[0] === 'string') { 82 | const hashStrings = nodes as string[] 83 | this.validateHashStrings(hashStrings) 84 | this.leafs = hashStrings.map((hash) => new MerkleNode(hash)) 85 | } else { 86 | const nodeArray = nodes as IMerkleNode[] 87 | this.validateNodes(nodeArray) 88 | this.leafs = [...nodeArray] // 创建副本以避免外部修改 89 | } 90 | 91 | this.root = await this.buildTree() 92 | } 93 | 94 | /** 95 | * 获取树的根哈希值 96 | */ 97 | getRootHash(): string { 98 | if (!this.root) { 99 | throw new Error('Merkle 树尚未初始化，请先调用 init() 方法') 100 | } 101 | return this.root.hash 102 | } 103 | 104 | /** 105 | * 从叶子节点构建 Merkle 树 106 | */ 107 | private async buildTree(): Promise { 108 | if (this.leafs.length === 0) { 109 | throw new Error('无法在没有叶子节点的情况下构建树') 110 | } 111 | 112 | let currentLevelNodes = [...this.leafs] // 使用副本进行操作 113 | 114 | while (currentLevelNodes.length > 1) { 115 | const parentNodes: IMerkleNode[] = [] 116 | 117 | for (let i = 0; i < currentLevelNodes.length; i += 2) { 118 | const leftNode = currentLevelNodes[i] 119 | const rightNode = i + 1 < currentLevelNodes.length ? currentLevelNodes[i + 1] : null 120 | 121 | try { 122 | const parentHash = await this.calculateHash({ 123 | leftHash: leftNode.hash, 124 | rightHash: rightNode?.hash, 125 | }) 126 | parentNodes.push(new MerkleNode(parentHash, leftNode, rightNode)) 127 | } catch (error) { 128 | throw new Error(`计算父节点哈希失败: ${error}`) 129 | } 130 | } 131 | 132 | currentLevelNodes = parentNodes 133 | } 134 | 135 | return currentLevelNodes[0] 136 | } 137 | 138 | /** 139 | * 计算父节点的哈希值 140 | */ 141 | private async calculateHash({ 142 | leftHash, 143 | rightHash, 144 | }: { 145 | leftHash: string 146 | rightHash?: string 147 | }): Promise { 148 | try { 149 | return await this.hashFn(leftHash, rightHash) 150 | } catch (error) { 151 | throw new Error(`哈希计算失败: ${error}`) 152 | } 153 | } 154 | 155 | /** 156 | * 验证哈希字符串数组 157 | */ 158 | private validateHashStrings(hashes: string[]): void { 159 | for (let i = 0; i < hashes.length; i++) { 160 | if (!hashes[i] || typeof hashes[i] !== 'string') { 161 | throw new Error(`索引 ${i} 处的哈希值无效: 必须是非空字符串`) 162 | } 163 | } 164 | } 165 | 166 | /** 167 | * 验证节点数组 168 | */ 169 | private validateNodes(nodes: IMerkleNode[]): void { 170 | for (let i = 0; i < nodes.length; i++) { 171 | const node = nodes[i] 172 | if (!node || !node.hash || typeof node.hash !== 'string') { 173 | throw new Error(`索引 ${i} 处的节点无效: 必须具有有效的 hash 属性`) 174 | } 175 | } 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /packages/core/__tests__/node/shared/baseWorkerWrapper.spec.ts: -------------------------------------------------------------------------------- 1 | // Mock the arrayBufferService module 2 | jest.mock('../../../src/shared/arrayBufferService', () => ({ 3 | restoreBuf: jest.fn(), 4 | })) 5 | 6 | import { BaseWorkerWrapper } from '../../../src/shared/baseWorkerWrapper' 7 | import { WorkerStatusEnum, WorkerReq, WorkerRes } from '../../../src/types' 8 | 9 | // 创建一个具体的实现类用于测试 10 | class TestWorkerWrapper extends BaseWorkerWrapper<{ terminate: () => void }, NodeJS.Timeout> { 11 | run(param: WorkerReq, index: number): Promise { 12 | return new Promise((resolve, reject) => { 13 | // 模拟异步操作 14 | setTimeout(() => { 15 | const mockResult = { data: 'test result' } as T 16 | const mockResponse: WorkerRes = { 17 | result: mockResult, 18 | chunk: param.chunk, 19 | } 20 | this.handleMessage(mockResponse, resolve, reject, index) 21 | }, 10) 22 | }) 23 | } 24 | 25 | // 实现抽象方法 26 | protected cleanupEventListeners(): void { 27 | // 测试用的空实现 28 | } 29 | 30 | protected createTimeout( 31 | timeoutMs: number, 32 | reject: (reason?: any) => void, 33 | taskId: string, 34 | ): NodeJS.Timeout { 35 | return setTimeout(() => { 36 | reject(new Error(`Task ${taskId} timed out after ${timeoutMs}ms`)) 37 | }, timeoutMs) 38 | } 39 | 40 | protected clearTimeout(timeoutId: NodeJS.Timeout): void { 41 | clearTimeout(timeoutId) 42 | } 43 | 44 | // 暴露受保护的方法用于测试 45 | public testHandleMessage( 46 | workerRes: WorkerRes, 47 | resolve: (value: any) => void, 48 | reject: (reason?: any) => void, 49 | index: number, 50 | ) { 51 | this.handleMessage(workerRes, resolve, reject, index) 52 | } 53 | 54 | public testHandleError(reject: (reason?: any) => void, error: Error) { 55 | this.handleError(reject, error) 56 | } 57 | } 58 | 59 | describe('BaseWorkerWrapper', () => { 60 | let mockWorker: { terminate: () => void } 61 | let wrapper: TestWorkerWrapper 62 | 63 | beforeEach(() => { 64 | mockWorker = { 65 | terminate: jest.fn(), 66 | } 67 | wrapper = new TestWorkerWrapper(mockWorker) 68 | }) 69 | 70 | describe('constructor', () => { 71 | it('应该正确初始化状态和worker', () => { 72 | expect(wrapper.status).toBe(WorkerStatusEnum.WAITING) 73 | expect((wrapper as any).worker).toBe(mockWorker) 74 | }) 75 | }) 76 | 77 | describe('run', () => { 78 | it('应该能够运行并返回结果', async () => { 79 | const mockParam: WorkerReq = { 80 | chunk: new ArrayBuffer(8), 81 | strategy: 'md5' as any, 82 | } 83 | const index = 0 84 | 85 | const result = await wrapper.run(mockParam, index) 86 | 87 | expect(result).toEqual({ data: 'test result' }) 88 | }) 89 | 90 | it('应该在运行过程中更新状态', async () => { 91 | const mockParam: WorkerReq = { 92 | chunk: new ArrayBuffer(8), 93 | strategy: 'md5' as any, 94 | } 95 | const index = 0 96 | 97 | // 创建一个新的包装器来测试状态变化 98 | const testWrapper = new TestWorkerWrapper(mockWorker) 99 | 100 | const runPromise = testWrapper.run(mockParam, index) 101 | 102 | // 等待异步操作完成 103 | await runPromise 104 | 105 | // 运行完成后状态应该回到 WAITING 106 | expect(testWrapper.status).toBe(WorkerStatusEnum.WAITING) 107 | }) 108 | }) 109 | 110 | describe('terminate', () => { 111 | it('应该调用worker的terminate方法', () => { 112 | wrapper.terminate() 113 | expect(mockWorker.terminate).toHaveBeenCalledTimes(1) 114 | }) 115 | }) 116 | 117 | describe('handleMessage', () => { 118 | it('应该正确处理消息并更新状态', () => { 119 | const mockResolve = jest.fn() 120 | const mockReject = jest.fn() 121 | const mockResponse: WorkerRes = { 122 | result: 'test result', 123 | chunk: new ArrayBuffer(8), 124 | } 125 | const index = 0 126 | 127 | wrapper.testHandleMessage(mockResponse, mockResolve, mockReject, index) 128 | 129 | expect(wrapper.status).toBe(WorkerStatusEnum.WAITING) 130 | expect(mockResolve).toHaveBeenCalledWith('test result') 131 | }) 132 | }) 133 | 134 | describe('handleError', () => { 135 | it('应该正确处理错误并更新状态', () => { 136 | const mockReject = jest.fn() 137 | const testError = new Error('Test error') 138 | 139 | wrapper.testHandleError(mockReject, testError) 140 | 141 | expect(wrapper.status).toBe(WorkerStatusEnum.WAITING) 142 | expect(mockReject).toHaveBeenCalledWith(testError) 143 | }) 144 | }) 145 | 146 | describe('WorkerStatusEnum', () => { 147 | it('应该包含正确的状态值', () => { 148 | expect(WorkerStatusEnum.RUNNING).toBe('running') 149 | expect(WorkerStatusEnum.WAITING).toBe('waiting') 150 | }) 151 | }) 152 | 153 | describe('类型兼容性', () => { 154 | it('应该能够接受任何具有terminate方法的worker', () => { 155 | const customWorker = { 156 | terminate: () => console.log('custom terminate'), 157 | customMethod: () => 'custom', 158 | } 159 | 160 | const customWrapper = new TestWorkerWrapper(customWorker) 161 | expect((customWrapper as any).worker).toBe(customWorker) 162 | }) 163 | }) 164 | }) 165 | -------------------------------------------------------------------------------- /packages/core/__tests__/node/shared/helper.spec.ts: -------------------------------------------------------------------------------- 1 | import { 2 | getMerkleRootHashByChunks, 3 | mergeConfig, 4 | calculateHashInWorker, 5 | getChunksHashSingle, 6 | } from '../../../src/shared/helper' 7 | import { Strategy, Config, WorkerReq } from '../../../src/types' 8 | import { DEFAULT_MAX_WORKERS } from '../../../src/shared/constant' 9 | 10 | // Mock hash-wasm functions 11 | jest.mock('hash-wasm', () => ({ 12 | md5: jest.fn().mockResolvedValue('mock-md5-hash'), 13 | xxhash128: jest.fn().mockResolvedValue('mock-xxHash128-hash'), 14 | })) 15 | 16 | describe('helper', () => { 17 | beforeEach(() => { 18 | jest.clearAllMocks() 19 | }) 20 | 21 | describe('getMerkleRootHashByChunks', () => { 22 | it('应该使用默认 hash 函数计算 Merkle 根哈希', async () => { 23 | const hashList = ['hash1', 'hash2', 'hash3'] 24 | const result = await getMerkleRootHashByChunks(hashList) 25 | 26 | expect(result).toBeDefined() 27 | expect(typeof result).toBe('string') 28 | }) 29 | 30 | it('应该使用自定义 hash 函数计算 Merkle 根哈希', async () => { 31 | const hashList = ['hash1', 'hash2'] 32 | const customHashFn = jest.fn().mockResolvedValue('custom-hash') 33 | const result = await getMerkleRootHashByChunks(hashList, customHashFn) 34 | 35 | expect(result).toBeDefined() 36 | expect(typeof result).toBe('string') 37 | }) 38 | 39 | it('应该处理空哈希列表', async () => { 40 | await expect(getMerkleRootHashByChunks([])).rejects.toThrow('无法使用空输入创建 Merkle 树') 41 | }) 42 | 43 | it('应该处理单个哈希', async () => { 44 | const result = await getMerkleRootHashByChunks(['single-hash']) 45 | expect(result).toBeDefined() 46 | }) 47 | }) 48 | 49 | describe('mergeConfig', () => { 50 | it('应该使用默认配置当没有提供参数时', () => { 51 | const config = mergeConfig() 52 | 53 | expect(config).toEqual({ 54 | chunkSize: 10, 55 | workerCount: DEFAULT_MAX_WORKERS, 56 | strategy: Strategy.xxHash128, 57 | isCloseWorkerImmediately: true, 58 | isShowLog: false, 59 | }) 60 | }) 61 | 62 | it('应该合并部分配置参数', () => { 63 | const partialConfig: Config = { 64 | chunkSize: 20, 65 | strategy: Strategy.xxHash128, 66 | isShowLog: true, 67 | } 68 | 69 | const config = mergeConfig(partialConfig) 70 | 71 | expect(config).toEqual({ 72 | chunkSize: 20, 73 | workerCount: DEFAULT_MAX_WORKERS, 74 | strategy: Strategy.xxHash128, 75 | isCloseWorkerImmediately: true, 76 | isShowLog: true, 77 | }) 78 | }) 79 | 80 | it('应该合并所有配置参数', () => { 81 | const fullConfig: Config = { 82 | chunkSize: 15, 83 | workerCount: 4, 84 | strategy: Strategy.md5, 85 | isCloseWorkerImmediately: false, 86 | isShowLog: true, 87 | } 88 | 89 | const config = mergeConfig(fullConfig) 90 | 91 | expect(config).toEqual(fullConfig) 92 | }) 93 | 94 | it('应该处理 undefined 配置', () => { 95 | const config = mergeConfig(undefined) 96 | 97 | expect(config).toEqual({ 98 | chunkSize: 10, 99 | workerCount: DEFAULT_MAX_WORKERS, 100 | strategy: Strategy.xxHash128, 101 | isCloseWorkerImmediately: true, 102 | isShowLog: false, 103 | }) 104 | }) 105 | }) 106 | 107 | describe('calculateHashInWorker', () => { 108 | it('应该使用 md5 策略计算哈希', async () => { 109 | const req: WorkerReq = { 110 | chunk: new ArrayBuffer(8), 111 | strategy: Strategy.md5, 112 | } 113 | 114 | const result = await calculateHashInWorker(req) 115 | 116 | expect(result).toEqual({ 117 | result: 'mock-md5-hash', 118 | chunk: req.chunk, 119 | }) 120 | }) 121 | 122 | it('应该使用 xxHash128 策略计算哈希', async () => { 123 | const req: WorkerReq = { 124 | chunk: new ArrayBuffer(32), 125 | strategy: Strategy.xxHash128, 126 | } 127 | 128 | const result = await calculateHashInWorker(req) 129 | 130 | expect(result).toEqual({ 131 | result: 'mock-xxHash128-hash', 132 | chunk: req.chunk, 133 | }) 134 | }) 135 | 136 | it('应该抛出错误当使用不支持的策略时', async () => { 137 | const req: WorkerReq = { 138 | chunk: new ArrayBuffer(8), 139 | strategy: 'unsupported' as Strategy, 140 | } 141 | 142 | await expect(calculateHashInWorker(req)).rejects.toThrow('Unsupported strategy: unsupported') 143 | }) 144 | 145 | it('应该正确处理空的 ArrayBuffer', async () => { 146 | const req: WorkerReq = { 147 | chunk: new ArrayBuffer(0), 148 | strategy: Strategy.md5, 149 | } 150 | 151 | const result = await calculateHashInWorker(req) 152 | 153 | expect(result).toEqual({ 154 | result: 'mock-md5-hash', 155 | chunk: req.chunk, 156 | }) 157 | }) 158 | }) 159 | 160 | describe('getChunksHashSingle', () => { 161 | it('应该使用 md5 策略计算单个哈希', async () => { 162 | const arrayBuffer = new ArrayBuffer(8) 163 | const result = await getChunksHashSingle(Strategy.md5, arrayBuffer) 164 | 165 | expect(result).toEqual(['mock-md5-hash']) 166 | }) 167 | 168 | it('应该使用 xxHash128 策略计算单个哈希', async () => { 169 | const arrayBuffer = new ArrayBuffer(32) 170 | const result = await getChunksHashSingle(Strategy.xxHash128, arrayBuffer) 171 | 172 | expect(result).toEqual(['mock-xxHash128-hash']) 173 | }) 174 | 175 | it('应该抛出错误当使用不支持的策略时', async () => { 176 | const arrayBuffer = new ArrayBuffer(8) 177 | 178 | await expect(getChunksHashSingle('unsupported' as Strategy, arrayBuffer)).rejects.toThrow( 179 | 'Unsupported strategy: unsupported', 180 | ) 181 | }) 182 | 183 | it('应该处理空的 ArrayBuffer', async () => { 184 | const arrayBuffer = new ArrayBuffer(0) 185 | const result = await getChunksHashSingle(Strategy.md5, arrayBuffer) 186 | 187 | expect(result).toEqual(['mock-md5-hash']) 188 | }) 189 | }) 190 | }) 191 | -------------------------------------------------------------------------------- /README-zh.md: -------------------------------------------------------------------------------- 1 | # Hash Worker [![npm package](https://img.shields.io/npm/v/hash-worker.svg)](https://www.npmjs.com/package/hash-worker) [![Bundle size](https://badgen.net/bundlephobia/minzip/hash-worker)](https://bundlephobia.com/result?p=hash-worker) [![codecov](https://codecov.io/gh/Tkunl/hash-worker/graph/badge.svg?token=G7GYAPEPYS)](https://codecov.io/gh/Tkunl/hash-worker) ![GitHub License](https://img.shields.io/github/license/Tkunl/hash-worker) 2 | 3 |

4 | 5 |

6 | 7 | ## Introduce 8 | 9 | [English Document](./README.md) 10 | 11 | **Hash-worker** 是一个用于快速计算文件哈希值的库。 12 | 13 | 它基于 hash-wasm 且利用了 WebWorker 进行并行计算，从而加快了计算文件分片的计算速度。 14 | 15 | Hash-worker 支持两种哈希计算算法：`md5`, `xxHash64`, `xxHash128`。 16 | 17 | 同时支持 `浏览器` 和 `Node.js` 环境。 18 | 19 | > [!WARNING] 20 | > Hash-worker 计算出的 MerkleHash 是基于文件块哈希值构建的 MerkleTree 的根哈希值。请注意，这并不直接等同于文件本身的哈希值。 21 | 22 | ## Install 23 | 24 | ```bash 25 | $ pnpm install hash-worker 26 | ``` 27 | 28 | ## Usage 29 | 30 | > [!WARNING] 31 | > 如果你在使用 `Vite` 作为构建工具, 需要在 `Vite` 的配置文件中, 添加如下配置, 用于将 hash-worker 排除 vite 的预构建行为 32 | 33 | ```js 34 | // vite.config.js 35 | import { defineConfig } from 'vite' 36 | import vue from '@vitejs/plugin-vue' 37 | 38 | export default defineConfig({ 39 | plugins: [vue()], 40 | // other configurations ... 41 | optimizeDeps: { 42 | exclude: ['hash-worker'] // new added.. 43 | } 44 | }) 45 | ``` 46 | 47 | ### Global 48 | 49 | ```html 50 | 51 | 52 | 55 | ``` 56 | 57 | 其中 `global.js` 和 `browser.worker.mjs` 是执行 `package.json` 中的 `build:core` 后的打包产物 58 | 59 | 打包产物位于 `packages/core/dist` 目录 60 | 61 | ### ESM 62 | 63 | > [!WARNING] 64 | > 在浏览器环境下从 'hash-worker' 中导入 65 | > 66 | > 在 Node 环境下从 'hash-worker/node' 中导入 67 | 68 | ``` ts 69 | import { getFileHashChunks, destroyWorkerPool, HashWorkerResult, HashWorkerOptions } from 'hash-worker' 70 | 71 | function handleGetHash(file: File) { 72 | const param: HashWorkerOptions = { 73 | file: file, 74 | config: { 75 | workerCount: 8, 76 | strategy: Strategy.md5 77 | } 78 | } 79 | 80 | getFileHashChunks(param).then((data: HashWorkerResult) => { 81 | console.log('chunksHash', data.chunksHash) 82 | }) 83 | } 84 | 85 | /** 86 | * Destroy Worker Thread 87 | */ 88 | function handleDestroyWorkerPool() { 89 | destroyWorkerPool() 90 | } 91 | ``` 92 | 93 | ## Options 94 | 95 | **HashWorkerOptions** 96 | 97 | HashWorkerOptions 是用于配置计算哈希值所需的参数。 98 | 99 | | filed | type | default | description | 100 | |----------|--------|---------|-----------------------------| 101 | | file | File | / | 需要计算 Hash 的文件（浏览器环境下必填） | 102 | | filePath | string | / | 需要计算 Hash 的文件路径 （Node环境下必填） | 103 | | config | Config | Config | 计算 Hash 时的参数 | 104 | 105 | **Config** 106 | 107 | | filed | type | default | description | 108 | |--------------------------|----------|----------------|---------------------------| 109 | | chunkSize | number | 10 (MB) | 文件分片的大小 | 110 | | workerCount | number | 8 | 计算 Hash 时同时开启的 worker 数量 | 111 | | strategy | Strategy | Strategy.xxHash128 | hash 计算策略 | 112 | | isCloseWorkerImmediately | boolean | true | 当计算完成时, 是否立即销毁 Worker 线程 | 113 | | isShowLog | boolean | false | 当计算完成时, 是否在控制台显示 log | 114 | | hashFn | HashFn | async (hLeft, hRight?) => (hRight ? md5(hLeft + hRight) : hLeft)| 构建 MerkleTree 时的 hash 方法 | 115 | 116 | ```ts 117 | enum Strategy { 118 | md5 = 'md5', 119 | xxHash64 = 'xxHash64', 120 | xxHash128 = 'xxHash128', 121 | } 122 | 123 | type HashFn = (hLeft: string, hRight?: string) => Promise 124 | ``` 125 | 126 | 127 | 128 | **HashWorkerResult** 129 | 130 | HashWorkerResult 是计算哈希值之后的返回结果。 131 | 132 | | filed | type | description | 133 | |------------|--------------|--------------------------| 134 | | chunksBlob | Blob[] | 仅在浏览器环境下，会返回文件分片的 Blob[] | 135 | | chunksHash | string[] | 文件分片的 Hash[] | 136 | | merkleHash | string | 文件的 merkleHash | 137 | | metadata | FileMetaInfo | 文件的 metadata | 138 | 139 | **FileMetaInfo** 140 | 141 | | filed | type | description | 142 | |--------------|--------|----------------| 143 | | name | string | 用于计算 hash 的文件名 | 144 | | size | number | 文件大小，单位：KB | 145 | | lastModified | number | 文件最后一次修改的时间戳 | 146 | | type | string | 文件后缀名 | 147 | 148 | ### [Benchmark (MD5)](./packages/benchmark/README-zh.md) 149 | 150 | | Worker Count | Speed | 151 | |--------------|-----------| 152 | | 1 | 229 MB/s | 153 | | 4 | 632 MB/s | 154 | | 8 | 886 MB/s | 155 | | 12 | 1037 MB/s | 156 | 157 | * 以上数据是运行在 `Chrome v131` 和 `AMD Ryzen9 5950X` CPU 下, 通过使用 md5 来计算 hash 得到的。 158 | 159 | ## LICENSE 160 | 161 | [MIT](./LICENSE) 162 | 163 | ## Contributions 164 | 165 | 欢迎贡献代码！如果你发现了一个 bug 或者想添加一个新功能，请提交一个 issue 或 pull request。 166 | 167 | ## Author and contributors 168 | 169 |

170 | 171 | Tkunl 172 | 173 | 174 | Kanno 175 | 176 | 177 | Eternal-could 178 | 179 |

180 | 181 | 182 | 183 | 184 | 185 | 186 | -------------------------------------------------------------------------------- /packages/core/README-zh.md: -------------------------------------------------------------------------------- 1 | # Hash Worker [![npm package](https://img.shields.io/npm/v/hash-worker.svg)](https://www.npmjs.com/package/hash-worker) [![Bundle size](https://badgen.net/bundlephobia/minzip/hash-worker)](https://bundlephobia.com/result?p=hash-worker) [![codecov](https://codecov.io/gh/Tkunl/hash-worker/graph/badge.svg?token=G7GYAPEPYS)](https://codecov.io/gh/Tkunl/hash-worker) ![GitHub License](https://img.shields.io/github/license/Tkunl/hash-worker) 2 | 3 |

4 | 5 |

6 | 7 | ## Introduce 8 | 9 | [English Document](./README.md) 10 | 11 | **Hash-worker** 是一个用于快速计算文件哈希值的库。 12 | 13 | 它基于 hash-wasm 且利用了 WebWorker 进行并行计算，从而加快了计算文件分片的计算速度。 14 | 15 | Hash-worker 支持两种哈希计算算法：`md5`, `xxHash64`, `xxHash128`。 16 | 17 | 同时支持 `浏览器` 和 `Node.js` 环境。 18 | 19 | > [!WARNING] 20 | > Hash-worker 计算出的 MerkleHash 是基于文件块哈希值构建的 MerkleTree 的根哈希值。请注意，这并不直接等同于文件本身的哈希值。 21 | 22 | ## Install 23 | 24 | ```bash 25 | $ pnpm install hash-worker 26 | ``` 27 | 28 | ## Usage 29 | 30 | > [!WARNING] 31 | > 如果你在使用 `Vite` 作为构建工具, 需要在 `Vite` 的配置文件中, 添加如下配置, 用于将 hash-worker 排除 vite 的预构建行为 32 | 33 | ```js 34 | // vite.config.js 35 | import { defineConfig } from 'vite' 36 | import vue from '@vitejs/plugin-vue' 37 | 38 | export default defineConfig({ 39 | plugins: [vue()], 40 | // other configurations ... 41 | optimizeDeps: { 42 | exclude: ['hash-worker'] // new added.. 43 | } 44 | }) 45 | ``` 46 | 47 | ### Global 48 | 49 | ```html 50 | 51 | 52 | 55 | ``` 56 | 57 | 其中 `global.js` 和 `browser.worker.mjs` 是执行 `package.json` 中的 `build:core` 后的打包产物 58 | 59 | 打包产物位于 `packages/core/dist` 目录 60 | 61 | ### ESM 62 | 63 | > [!WARNING] 64 | > 在浏览器环境下从 'hash-worker' 中导入 65 | > 66 | > 在 Node 环境下从 'hash-worker/node' 中导入 67 | 68 | ``` ts 69 | import { getFileHashChunks, destroyWorkerPool, HashWorkerResult, HashWorkerOptions } from 'hash-worker' 70 | 71 | function handleGetHash(file: File) { 72 | const param: HashWorkerOptions = { 73 | file: file, 74 | config: { 75 | workerCount: 8, 76 | strategy: Strategy.md5 77 | } 78 | } 79 | 80 | getFileHashChunks(param).then((data: HashWorkerResult) => { 81 | console.log('chunksHash', data.chunksHash) 82 | }) 83 | } 84 | 85 | /** 86 | * Destroy Worker Thread 87 | */ 88 | function handleDestroyWorkerPool() { 89 | destroyWorkerPool() 90 | } 91 | ``` 92 | 93 | ## Options 94 | 95 | **HashWorkerOptions** 96 | 97 | HashWorkerOptions 是用于配置计算哈希值所需的参数。 98 | 99 | | filed | type | default | description | 100 | |----------|--------|---------|-----------------------------| 101 | | file | File | / | 需要计算 Hash 的文件（浏览器环境下必填） | 102 | | filePath | string | / | 需要计算 Hash 的文件路径 （Node环境下必填） | 103 | | config | Config | Config | 计算 Hash 时的参数 | 104 | 105 | **Config** 106 | 107 | | filed | type | default | description | 108 | |--------------------------|----------|----------------|---------------------------| 109 | | chunkSize | number | 10 (MB) | 文件分片的大小 | 110 | | workerCount | number | 8 | 计算 Hash 时同时开启的 worker 数量 | 111 | | strategy | Strategy | Strategy.xxHash128 | hash 计算策略 | 112 | | isCloseWorkerImmediately | boolean | true | 当计算完成时, 是否立即销毁 Worker 线程 | 113 | | isShowLog | boolean | false | 当计算完成时, 是否在控制台显示 log | 114 | | hashFn | HashFn | async (hLeft, hRight?) => (hRight ? md5(hLeft + hRight) : hLeft)| 构建 MerkleTree 时的 hash 方法 | 115 | 116 | ```ts 117 | enum Strategy { 118 | md5 = 'md5', 119 | xxHash64 = 'xxHash64', 120 | xxHash128 = 'xxHash128', 121 | } 122 | 123 | type HashFn = (hLeft: string, hRight?: string) => Promise 124 | ``` 125 | 126 | 127 | 128 | **HashWorkerResult** 129 | 130 | HashWorkerResult 是计算哈希值之后的返回结果。 131 | 132 | | filed | type | description | 133 | |------------|--------------|--------------------------| 134 | | chunksBlob | Blob[] | 仅在浏览器环境下，会返回文件分片的 Blob[] | 135 | | chunksHash | string[] | 文件分片的 Hash[] | 136 | | merkleHash | string | 文件的 merkleHash | 137 | | metadata | FileMetaInfo | 文件的 metadata | 138 | 139 | **FileMetaInfo** 140 | 141 | | filed | type | description | 142 | |--------------|--------|----------------| 143 | | name | string | 用于计算 hash 的文件名 | 144 | | size | number | 文件大小，单位：KB | 145 | | lastModified | number | 文件最后一次修改的时间戳 | 146 | | type | string | 文件后缀名 | 147 | 148 | ### [Benchmark (MD5)](./packages/benchmark/README-zh.md) 149 | 150 | | Worker Count | Speed | 151 | |--------------|-----------| 152 | | 1 | 229 MB/s | 153 | | 4 | 632 MB/s | 154 | | 8 | 886 MB/s | 155 | | 12 | 1037 MB/s | 156 | 157 | * 以上数据是运行在 `Chrome v131` 和 `AMD Ryzen9 5950X` CPU 下, 通过使用 md5 来计算 hash 得到的。 158 | 159 | ## LICENSE 160 | 161 | [MIT](./LICENSE) 162 | 163 | ## Contributions 164 | 165 | 欢迎贡献代码！如果你发现了一个 bug 或者想添加一个新功能，请提交一个 issue 或 pull request。 166 | 167 | ## Author and contributors 168 | 169 |

170 | 171 | Tkunl 172 | 173 | 174 | Kanno 175 | 176 | 177 | Eternal-could 178 | 179 |

180 | 181 | 182 | 183 | 184 | 185 | 186 | -------------------------------------------------------------------------------- /packages/core/src/shared/baseWorkerPool.ts: -------------------------------------------------------------------------------- 1 | import { BaseWorkerWrapper } from './baseWorkerWrapper' 2 | import { generateUUID } from './utils' 3 | import { WorkerReq, WorkerStatusEnum, TaskResult, TaskConfig } from '../types' 4 | 5 | interface QueuedTask { 6 | id: string 7 | param: WorkerReq 8 | index: number 9 | resolve: (value: TaskResult) => void 10 | config?: TaskConfig 11 | } 12 | 13 | /** 14 | * 基础工作池抽象类 15 | * @template TWorkerWrapper - 工作包装器类型，必须继承自BaseWorkerWrapper 16 | * 17 | * 注意：这里使用 是必要的，因为： 18 | * - 浏览器环境：BaseWorkerWrapper 19 | * - Node.js环境：BaseWorkerWrapper 20 | * 两种环境的泛型参数完全不同，使用any可以同时支持两种环境 21 | * 22 | * 类型安全性在具体的 BrowserWorkerWrapper 和 NodeWorkerWrapper 中得到保障 23 | */ 24 | export abstract class BaseWorkerPool< 25 | TWorkerWrapper extends BaseWorkerWrapper = BaseWorkerWrapper, 26 | > { 27 | pool: TWorkerWrapper[] = [] 28 | maxWorkerCount: number 29 | private taskQueue: QueuedTask[] = [] 30 | private isProcessing = false 31 | 32 | constructor(maxWorkers: number) { 33 | this.maxWorkerCount = maxWorkers 34 | this.pool = Array.from({ length: maxWorkers }).map(() => this.createWorker()) 35 | } 36 | 37 | abstract createWorker(): TWorkerWrapper 38 | 39 | async exec(params: WorkerReq[], config?: TaskConfig): Promise[]> { 40 | if (params.length === 0) { 41 | return [] 42 | } 43 | 44 | return new Promise[]>((resolve) => { 45 | const results: TaskResult[] = new Array(params.length) 46 | let completedCount = 0 47 | 48 | // 将所有任务加入队列 49 | params.forEach((param, index) => { 50 | const taskId = generateUUID() 51 | const queuedTask: QueuedTask = { 52 | id: taskId, 53 | param, 54 | index, 55 | config, 56 | resolve: (result: TaskResult) => { 57 | results[index] = result 58 | completedCount++ 59 | 60 | // 当所有任务完成时，返回结果 61 | if (completedCount === params.length) { 62 | resolve(results) 63 | } 64 | }, 65 | } 66 | this.taskQueue.push(queuedTask) 67 | }) 68 | 69 | // 开始处理队列 70 | this.processQueue() 71 | }) 72 | } 73 | 74 | private async processQueue(): Promise { 75 | if (this.isProcessing) { 76 | return 77 | } 78 | 79 | this.isProcessing = true 80 | 81 | try { 82 | while (this.taskQueue.length > 0) { 83 | const availableWorkers = this.getAvailableWorkers() 84 | 85 | if (availableWorkers.length === 0) { 86 | // 没有可用的worker，等待一段时间后重试 87 | await this.waitForAvailableWorker() 88 | continue 89 | } 90 | 91 | // 分配任务给可用的workers 92 | const tasksToProcess = Math.min(availableWorkers.length, this.taskQueue.length) 93 | const currentTasks = this.taskQueue.splice(0, tasksToProcess) 94 | 95 | // 并行执行任务 96 | const promises = currentTasks.map((task, i) => this.executeTask(availableWorkers[i], task)) 97 | 98 | // 等待当前这批任务完成 99 | await Promise.allSettled(promises) 100 | } 101 | } finally { 102 | this.isProcessing = false 103 | } 104 | } 105 | 106 | private async executeTask(worker: TWorkerWrapper, task: QueuedTask): Promise { 107 | const { param, index, resolve, config } = task 108 | 109 | try { 110 | const result = await worker.run(param, index, config) 111 | resolve({ 112 | success: true, 113 | data: result, 114 | index, 115 | }) 116 | } catch (error) { 117 | resolve({ 118 | success: false, 119 | error: error instanceof Error ? error : new Error('Unknown error'), 120 | index, 121 | }) 122 | } 123 | } 124 | 125 | private getAvailableWorkers(): TWorkerWrapper[] { 126 | return this.pool.filter((worker) => worker.status === WorkerStatusEnum.WAITING) 127 | } 128 | 129 | private getRunningWorkers(): TWorkerWrapper[] { 130 | return this.pool.filter((worker) => worker.status === WorkerStatusEnum.RUNNING) 131 | } 132 | 133 | private async waitForAvailableWorker(): Promise { 134 | return new Promise((resolve) => { 135 | const checkWorkers = () => { 136 | const availableWorkers = this.getAvailableWorkers() 137 | if (availableWorkers.length > 0) { 138 | resolve() 139 | } else { 140 | // 50ms后重新检查 141 | setTimeout(checkWorkers, 50) 142 | } 143 | } 144 | checkWorkers() 145 | }) 146 | } 147 | 148 | adjustPool(workerCount: number): void { 149 | const curCount = this.pool.length 150 | const diff = workerCount - curCount 151 | 152 | if (diff > 0) { 153 | // 增加workers 154 | Array.from({ length: diff }).forEach(() => { 155 | this.pool.push(this.createWorker()) 156 | }) 157 | } else if (diff < 0) { 158 | // 减少workers 159 | let count = Math.abs(diff) 160 | for (let i = this.pool.length - 1; i >= 0 && count > 0; i--) { 161 | const workerWrapper = this.pool[i] 162 | if (workerWrapper.status === WorkerStatusEnum.WAITING) { 163 | workerWrapper.terminate() 164 | this.pool.splice(i, 1) 165 | count-- 166 | } 167 | } 168 | } 169 | 170 | this.maxWorkerCount = workerCount 171 | } 172 | 173 | terminate(): void { 174 | // 清空任务队列 175 | this.taskQueue.length = 0 176 | 177 | // 终止所有workers 178 | this.pool.forEach((workerWrapper) => workerWrapper.terminate()) 179 | this.pool.length = 0 180 | 181 | // 重置状态 182 | this.isProcessing = false 183 | } 184 | 185 | // 获取池状态信息，用于调试和监控 186 | getPoolStatus() { 187 | const runningWorkers = this.getRunningWorkers() 188 | const waitingWorkers = this.getAvailableWorkers() 189 | const errorWorkers = this.pool.filter((w) => w.status === WorkerStatusEnum.ERROR) 190 | 191 | return { 192 | totalWorkers: this.pool.length, 193 | runningWorkers: runningWorkers.length, 194 | waitingWorkers: waitingWorkers.length, 195 | errorWorkers: errorWorkers.length, 196 | queuedTasks: this.taskQueue.length, 197 | isProcessing: this.isProcessing, 198 | // 添加更详细的运行时信息 199 | runningTasksInfo: runningWorkers.map((w, i) => ({ workerId: i, status: w.status })), 200 | } 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /packages/core/__tests__/node/shared/utils.spec.ts: -------------------------------------------------------------------------------- 1 | import { getArrParts, runAsyncFuncSerialized, generateUUID } from '../../../src/shared/utils' 2 | 3 | describe('utils', () => { 4 | describe('getArrParts', () => { 5 | it('应该正确分割数组', () => { 6 | const arr = [1, 2, 3, 4, 5, 6] 7 | expect(getArrParts(arr, 2)).toEqual([ 8 | [1, 2], 9 | [3, 4], 10 | [5, 6], 11 | ]) 12 | expect(getArrParts(arr, 3)).toEqual([ 13 | [1, 2, 3], 14 | [4, 5, 6], 15 | ]) 16 | expect(getArrParts(arr, 4)).toEqual([ 17 | [1, 2, 3, 4], 18 | [5, 6], 19 | ]) 20 | }) 21 | 22 | it('当 size 等于数组长度时，应该返回原数组', () => { 23 | const arr = [1, 2, 3] 24 | expect(getArrParts(arr, 3)).toEqual([[1, 2, 3]]) 25 | }) 26 | 27 | it('当 size 大于数组长度时，应该返回原数组', () => { 28 | const arr = [1, 2, 3] 29 | expect(getArrParts(arr, 5)).toEqual([[1, 2, 3]]) 30 | }) 31 | 32 | it('当 size 为 1 时，应该将每个元素分割成单独的数组', () => { 33 | const arr = [1, 2, 3] 34 | expect(getArrParts(arr, 1)).toEqual([[1], [2], [3]]) 35 | }) 36 | 37 | it('当 size 为 0 或负数时，应该抛出 RangeError', () => { 38 | const arr = [1, 2, 3] 39 | expect(() => getArrParts(arr, 0)).toThrow(RangeError) 40 | expect(() => getArrParts(arr, 0)).toThrow('size must be a positive integer') 41 | expect(() => getArrParts(arr, -1)).toThrow(RangeError) 42 | expect(() => getArrParts(arr, -1)).toThrow('size must be a positive integer') 43 | expect(() => getArrParts(arr, -5)).toThrow(RangeError) 44 | expect(() => getArrParts(arr, -5)).toThrow('size must be a positive integer') 45 | }) 46 | 47 | it('当 size 不是整数时，应该抛出 RangeError', () => { 48 | const arr = [1, 2, 3] 49 | expect(() => getArrParts(arr, 1.5)).toThrow(RangeError) 50 | expect(() => getArrParts(arr, 1.5)).toThrow('size must be a positive integer') 51 | expect(() => getArrParts(arr, 2.7)).toThrow(RangeError) 52 | expect(() => getArrParts(arr, 2.7)).toThrow('size must be a positive integer') 53 | }) 54 | 55 | it('当输入空数组时，应该返回空数组', () => { 56 | expect(getArrParts([], 2)).toEqual([]) 57 | }) 58 | 59 | it('应该处理字符串数组', () => { 60 | const arr = ['a', 'b', 'c', 'd'] 61 | expect(getArrParts(arr, 2)).toEqual([ 62 | ['a', 'b'], 63 | ['c', 'd'], 64 | ]) 65 | }) 66 | 67 | it('应该处理对象数组', () => { 68 | const arr = [{ id: 1 }, { id: 2 }, { id: 3 }, { id: 4 }] 69 | expect(getArrParts(arr, 2)).toEqual([ 70 | [{ id: 1 }, { id: 2 }], 71 | [{ id: 3 }, { id: 4 }], 72 | ]) 73 | }) 74 | }) 75 | 76 | describe('runAsyncFuncSerialized', () => { 77 | it('应该按顺序串行执行异步函数并合并结果', async () => { 78 | const tasks = [ 79 | () => Promise.resolve([1, 2]), 80 | () => Promise.resolve([3, 4]), 81 | () => Promise.resolve([5, 6]), 82 | ] 83 | 84 | const result = await runAsyncFuncSerialized(tasks) 85 | expect(result).toEqual([1, 2, 3, 4, 5, 6]) 86 | }) 87 | 88 | it('应该处理返回空数组的异步函数', async () => { 89 | const tasks = [ 90 | () => Promise.resolve([1, 2]), 91 | () => Promise.resolve([]), 92 | () => Promise.resolve([3, 4]), 93 | ] 94 | 95 | const result = await runAsyncFuncSerialized(tasks) 96 | expect(result).toEqual([1, 2, 3, 4]) 97 | }) 98 | 99 | it('当任务数组为空时，应该返回空数组', async () => { 100 | const result = await runAsyncFuncSerialized([]) 101 | expect(result).toEqual([]) 102 | }) 103 | 104 | it('应该处理异步函数抛出错误的情况', async () => { 105 | const tasks = [ 106 | () => Promise.resolve([1, 2]), 107 | () => Promise.reject(new Error('测试错误')), 108 | () => Promise.resolve([3, 4]), 109 | ] 110 | 111 | await expect(runAsyncFuncSerialized(tasks)).rejects.toThrow('测试错误') 112 | }) 113 | 114 | it('应该处理包含复杂对象的数组', async () => { 115 | const tasks = [ 116 | () => Promise.resolve([{ id: 1, name: 'Alice' }]), 117 | () => 118 | Promise.resolve([ 119 | { id: 2, name: 'Bob' }, 120 | { id: 3, name: 'Charlie' }, 121 | ]), 122 | ] 123 | 124 | const result = await runAsyncFuncSerialized(tasks) 125 | expect(result).toEqual([ 126 | { id: 1, name: 'Alice' }, 127 | { id: 2, name: 'Bob' }, 128 | { id: 3, name: 'Charlie' }, 129 | ]) 130 | }) 131 | 132 | it('应该确保函数按顺序执行', async () => { 133 | const executionOrder: number[] = [] 134 | 135 | const tasks = [ 136 | async () => { 137 | executionOrder.push(1) 138 | await new Promise((resolve) => setTimeout(resolve, 10)) 139 | return [1] 140 | }, 141 | async () => { 142 | executionOrder.push(2) 143 | await new Promise((resolve) => setTimeout(resolve, 5)) 144 | return [2] 145 | }, 146 | async () => { 147 | executionOrder.push(3) 148 | return [3] 149 | }, 150 | ] 151 | 152 | const result = await runAsyncFuncSerialized(tasks) 153 | expect(result).toEqual([1, 2, 3]) 154 | expect(executionOrder).toEqual([1, 2, 3]) 155 | }) 156 | }) 157 | 158 | describe('generateUUID', () => { 159 | it('应该生成有效的 UUID v4 格式字符串', () => { 160 | const uuid = generateUUID() 161 | // UUID v4 格式：xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx 162 | expect(uuid).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/) 163 | }) 164 | 165 | it('应该生成不同的 UUID', () => { 166 | const uuid1 = generateUUID() 167 | const uuid2 = generateUUID() 168 | const uuid3 = generateUUID() 169 | 170 | expect(uuid1).not.toBe(uuid2) 171 | expect(uuid1).not.toBe(uuid3) 172 | expect(uuid2).not.toBe(uuid3) 173 | }) 174 | 175 | it('应该生成正确长度的字符串', () => { 176 | const uuid = generateUUID() 177 | expect(uuid).toHaveLength(36) // UUID v4 标准长度包含连字符 178 | }) 179 | 180 | it('应该只包含十六进制字符和连字符', () => { 181 | const uuid = generateUUID() 182 | expect(uuid).toMatch(/^[0-9a-f-]+$/) 183 | }) 184 | 185 | it('应该符合 UUID v4 格式', () => { 186 | const uuid = generateUUID() 187 | // 检查第13位是否为4 (根据 UUID v4 标准) 188 | expect(uuid[14]).toBe('4') // 第14位是版本号位置（0-based index） 189 | // 检查第17位是否为8, 9, a, 或 b (根据 UUID v4 标准) 190 | expect(['8', '9', 'a', 'b']).toContain(uuid[19]) // 第19位是变体位置（0-based index） 191 | }) 192 | }) 193 | }) 194 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Hash Worker [![npm package](https://img.shields.io/npm/v/hash-worker.svg)](https://www.npmjs.com/package/hash-worker) [![Bundle size](https://badgen.net/bundlephobia/minzip/hash-worker)](https://bundlephobia.com/result?p=hash-worker) [![codecov](https://codecov.io/gh/Tkunl/hash-worker/graph/badge.svg?token=G7GYAPEPYS)](https://codecov.io/gh/Tkunl/hash-worker) ![GitHub License](https://img.shields.io/github/license/Tkunl/hash-worker) 2 | 3 |

4 | 5 |

6 | 7 | ## Introduce 8 | 9 | [中文文档](./README-zh.md) 10 | 11 | **Hash-worker** is a library for fast calculation of file chunk hashes. 12 | 13 | It is based on `hash-wasm` and utilizes `WebWorkers` for parallel computation, which speeds up computation when 14 | processing file blocks. 15 | 16 | Hash-worker supports two hash computation algorithms: `md5`, `xxHash64`, `xxHash128`. 17 | 18 | Both `browser` and `Node.js` are supported. 19 | 20 | > [!WARNING] 21 | > The merkleHash computed by the Hash-worker is the root hash of a MerkleTree constructed based on file block hashes. Note that this is not directly equivalent to a hash of the file itself. 22 | 23 | ## Install 24 | 25 | ```bash 26 | $ pnpm install hash-worker 27 | ``` 28 | 29 | ## Usage 30 | 31 | > [!WARNING] 32 | If you are using `Vite` as your build tool, you need to add the following configuration to your `Vite` config file to exclude hash-worker from Vite's pre-bundling process. 33 | 34 | ```js 35 | // vite.config.js 36 | import { defineConfig } from 'vite' 37 | import vue from '@vitejs/plugin-vue' 38 | 39 | export default defineConfig({ 40 | plugins: [vue()], 41 | // other configurations ... 42 | optimizeDeps: { 43 | exclude: ['hash-worker'] // new added.. 44 | } 45 | }) 46 | ``` 47 | 48 | ### Global 49 | 50 | ```html 51 | 52 | 53 | 56 | ``` 57 | 58 | The `global.js` and `browser.worker.mjs` are the build artifacts resulting from executing `build:core` in `package.json`. 59 | 60 | The build artifacts are located in the `packages/core/dist` directory. 61 | 62 | ### ESM 63 | 64 | > [!WARNING] 65 | > Import from 'hash-worker' in the browser environment 66 | > 67 | > Import from 'hash-worker/node' in the browser environment 68 | 69 | ``` ts 70 | import { getFileHashChunks, destroyWorkerPool, HashWorkerResult, HashWorkerOptions } from 'hash-worker' 71 | 72 | function handleGetHash(file: File) { 73 | const param: HashWorkerOptions = { 74 | file: file, 75 | config: { 76 | workerCount: 8, 77 | strategy: Strategy.md5 78 | } 79 | } 80 | 81 | getFileHashChunks(param).then((data: HashWorkerResult) => { 82 | console.log('chunksHash', data.chunksHash) 83 | }) 84 | } 85 | 86 | /** 87 | * Destroy Worker Thread 88 | */ 89 | function handleDestroyWorkerPool() { 90 | destroyWorkerPool() 91 | } 92 | ``` 93 | 94 | ## Options 95 | 96 | **HashWorkerOptions** 97 | 98 | HashWorkerOptions is used to configure the parameters needed to calculate the hash. 99 | 100 | | filed | type | default | description | 101 | |----------|--------|---------|--------------------------------------------------------------------------------------| 102 | | file | File | / | Files that need to calculate the hash (required for browser environments) | 103 | | filePath | string | / | Path to the file where the hash is to be calculated (required for Node environments) | 104 | | config | Config | Config | Parameters for calculating the Hash | 105 | 106 | **Config** 107 | 108 | | filed | type | default | description | 109 | |--------------------------|----------|----------------|-----------------------------------------------------------------------------------| 110 | | chunkSize | number | 10 (MB) | Size of the file slice | 111 | | workerCount | number | 8 | Number of workers turned on at the same time as the hash is calculated | 112 | | strategy | Strategy | Strategy.xxHash128 | Hash computation strategy | 113 | | isCloseWorkerImmediately | boolean | true | Whether to destroy the worker thread immediately when the calculation is complete | 114 | | isShowLog | boolean | false | Whether to show log in console when he calculation is complete | 115 | | hashFn | HashFn | async (hLeft, hRight?) => (hRight ? md5(hLeft + hRight) : hLeft)| The hash method for build MerkleTree | 116 | 117 | ```ts 118 | // strategy.ts 119 | enum Strategy { 120 | md5 = 'md5', 121 | xxHash64 = 'xxHash64', 122 | xxHash128 = 'xxHash128', 123 | } 124 | 125 | type HashFn = (hLeft: string, hRight?: string) => Promise 126 | ``` 127 | 128 | 129 | 130 | **HashWorkerResult** 131 | 132 | HashWorkerResult is the returned result after calculating the hash value. 133 | 134 | | filed | type | description | 135 | |------------|--------------|-------------------------------------------------------------------------| 136 | | chunksBlob | Blob[] | In a browser environment only, the Blob[] of the file slice is returned | 137 | | chunksHash | string[] | Hash[] for file slicing | 138 | | merkleHash | string | The merkleHash of the file | 139 | | metadata | FileMetaInfo | The metadata of the file | 140 | 141 | **FileMetaInfo** 142 | 143 | | filed | type | description | 144 | |--------------|--------|-------------------------------------------------| 145 | | name | string | The name of the file used to calculate the hash | 146 | | size | number | File size in KB | 147 | | lastModified | number | Timestamp of the last modification of the file | 148 | | type | string | file extension | 149 | 150 | ### [Benchmark (MD5)](./packages/benchmark/README.md) 151 | 152 | | Worker Count | Speed | 153 | |--------------|-----------| 154 | | 1 | 229 MB/s | 155 | | 4 | 632 MB/s | 156 | | 8 | 886 MB/s | 157 | | 12 | 1037 MB/s | 158 | 159 | The above data is run on the `Chrome v131` and `AMD Ryzen9 5950X` CPU, by using md5 to calculate hash. 160 | 161 | ## LICENSE 162 | 163 | [MIT](./LICENSE) 164 | 165 | ## Contributions 166 | 167 | Contributions are welcome! If you find a bug or want to add a new feature, please open an issue or submit a pull 168 | request. 169 | 170 | ## Author and contributors 171 | 172 |

173 | 174 | Tkunl 175 | 176 | 177 | Eternal-could 178 | 179 | 180 | Kanno 181 | 182 |

183 | -------------------------------------------------------------------------------- /packages/core/README.md: -------------------------------------------------------------------------------- 1 | # Hash Worker [![npm package](https://img.shields.io/npm/v/hash-worker.svg)](https://www.npmjs.com/package/hash-worker) [![Bundle size](https://badgen.net/bundlephobia/minzip/hash-worker)](https://bundlephobia.com/result?p=hash-worker) [![codecov](https://codecov.io/gh/Tkunl/hash-worker/graph/badge.svg?token=G7GYAPEPYS)](https://codecov.io/gh/Tkunl/hash-worker) ![GitHub License](https://img.shields.io/github/license/Tkunl/hash-worker) 2 | 3 |

4 | 5 |

6 | 7 | ## Introduce 8 | 9 | [中文文档](./README-zh.md) 10 | 11 | **Hash-worker** is a library for fast calculation of file chunk hashes. 12 | 13 | It is based on `hash-wasm` and utilizes `WebWorkers` for parallel computation, which speeds up computation when 14 | processing file blocks. 15 | 16 | Hash-worker supports two hash computation algorithms: `md5`, `xxHash64`, `xxHash128`. 17 | 18 | Both `browser` and `Node.js` are supported. 19 | 20 | > [!WARNING] 21 | > The merkleHash computed by the Hash-worker is the root hash of a MerkleTree constructed based on file block hashes. Note that this is not directly equivalent to a hash of the file itself. 22 | 23 | ## Install 24 | 25 | ```bash 26 | $ pnpm install hash-worker 27 | ``` 28 | 29 | ## Usage 30 | 31 | > [!WARNING] 32 | If you are using `Vite` as your build tool, you need to add the following configuration to your `Vite` config file to exclude hash-worker from Vite's pre-bundling process. 33 | 34 | ```js 35 | // vite.config.js 36 | import { defineConfig } from 'vite' 37 | import vue from '@vitejs/plugin-vue' 38 | 39 | export default defineConfig({ 40 | plugins: [vue()], 41 | // other configurations ... 42 | optimizeDeps: { 43 | exclude: ['hash-worker'] // new added.. 44 | } 45 | }) 46 | ``` 47 | 48 | ### Global 49 | 50 | ```html 51 | 52 | 53 | 56 | ``` 57 | 58 | The `global.js` and `browser.worker.mjs` are the build artifacts resulting from executing `build:core` in `package.json`. 59 | 60 | The build artifacts are located in the `packages/core/dist` directory. 61 | 62 | ### ESM 63 | 64 | > [!WARNING] 65 | > Import from 'hash-worker' in the browser environment 66 | > 67 | > Import from 'hash-worker/node' in the browser environment 68 | 69 | ``` ts 70 | import { getFileHashChunks, destroyWorkerPool, HashWorkerResult, HashWorkerOptions } from 'hash-worker' 71 | 72 | function handleGetHash(file: File) { 73 | const param: HashWorkerOptions = { 74 | file: file, 75 | config: { 76 | workerCount: 8, 77 | strategy: Strategy.md5 78 | } 79 | } 80 | 81 | getFileHashChunks(param).then((data: HashWorkerResult) => { 82 | console.log('chunksHash', data.chunksHash) 83 | }) 84 | } 85 | 86 | /** 87 | * Destroy Worker Thread 88 | */ 89 | function handleDestroyWorkerPool() { 90 | destroyWorkerPool() 91 | } 92 | ``` 93 | 94 | ## Options 95 | 96 | **HashWorkerOptions** 97 | 98 | HashWorkerOptions is used to configure the parameters needed to calculate the hash. 99 | 100 | | filed | type | default | description | 101 | |----------|--------|---------|--------------------------------------------------------------------------------------| 102 | | file | File | / | Files that need to calculate the hash (required for browser environments) | 103 | | filePath | string | / | Path to the file where the hash is to be calculated (required for Node environments) | 104 | | config | Config | Config | Parameters for calculating the Hash | 105 | 106 | **Config** 107 | 108 | | filed | type | default | description | 109 | |--------------------------|----------|----------------|-----------------------------------------------------------------------------------| 110 | | chunkSize | number | 10 (MB) | Size of the file slice | 111 | | workerCount | number | 8 | Number of workers turned on at the same time as the hash is calculated | 112 | | strategy | Strategy | Strategy.xxHash128 | Hash computation strategy | 113 | | isCloseWorkerImmediately | boolean | true | Whether to destroy the worker thread immediately when the calculation is complete | 114 | | isShowLog | boolean | false | Whether to show log in console when he calculation is complete | 115 | | hashFn | HashFn | async (hLeft, hRight?) => (hRight ? md5(hLeft + hRight) : hLeft)| The hash method for build MerkleTree | 116 | 117 | ```ts 118 | // strategy.ts 119 | enum Strategy { 120 | md5 = 'md5', 121 | xxHash64 = 'xxHash64', 122 | xxHash128 = 'xxHash128', 123 | } 124 | 125 | type HashFn = (hLeft: string, hRight?: string) => Promise 126 | ``` 127 | 128 | 129 | 130 | **HashWorkerResult** 131 | 132 | HashWorkerResult is the returned result after calculating the hash value. 133 | 134 | | filed | type | description | 135 | |------------|--------------|-------------------------------------------------------------------------| 136 | | chunksBlob | Blob[] | In a browser environment only, the Blob[] of the file slice is returned | 137 | | chunksHash | string[] | Hash[] for file slicing | 138 | | merkleHash | string | The merkleHash of the file | 139 | | metadata | FileMetaInfo | The metadata of the file | 140 | 141 | **FileMetaInfo** 142 | 143 | | filed | type | description | 144 | |--------------|--------|-------------------------------------------------| 145 | | name | string | The name of the file used to calculate the hash | 146 | | size | number | File size in KB | 147 | | lastModified | number | Timestamp of the last modification of the file | 148 | | type | string | file extension | 149 | 150 | ### [Benchmark (MD5)](./packages/benchmark/README.md) 151 | 152 | | Worker Count | Speed | 153 | |--------------|-----------| 154 | | 1 | 229 MB/s | 155 | | 4 | 632 MB/s | 156 | | 8 | 886 MB/s | 157 | | 12 | 1037 MB/s | 158 | 159 | The above data is run on the `Chrome v131` and `AMD Ryzen9 5950X` CPU, by using md5 to calculate hash. 160 | 161 | ## LICENSE 162 | 163 | [MIT](./LICENSE) 164 | 165 | ## Contributions 166 | 167 | Contributions are welcome! If you find a bug or want to add a new feature, please open an issue or submit a pull 168 | request. 169 | 170 | ## Author and contributors 171 | 172 |

173 | 174 | Tkunl 175 | 176 | 177 | Eternal-could 178 | 179 | 180 | Kanno 181 | 182 |

183 | -------------------------------------------------------------------------------- /packages/core/__tests__/node/shared/arrayBufferService.spec.ts: -------------------------------------------------------------------------------- 1 | import { initBufService, obtainBuf, restoreBuf } from '../../../src/shared/arrayBufferService' 2 | import { WorkerReq } from '../../../src/types' 3 | import { Strategy } from '../../../src/types' 4 | 5 | describe('ArrayBufferService', () => { 6 | let mockArrayBuffers: ArrayBuffer[] 7 | let mockWorkerReq: WorkerReq 8 | 9 | beforeEach(() => { 10 | // 创建测试用的 ArrayBuffer 11 | mockArrayBuffers = [new ArrayBuffer(8), new ArrayBuffer(16), new ArrayBuffer(32)] 12 | 13 | // 创建测试用的 WorkerReq 14 | mockWorkerReq = { 15 | chunk: new ArrayBuffer(8), 16 | strategy: Strategy.md5, 17 | } 18 | }) 19 | 20 | describe('initBufService', () => { 21 | it('应该正确初始化 ArrayBuffer 数组', () => { 22 | // 初始化服务 23 | initBufService(mockArrayBuffers) 24 | 25 | // 验证初始化后的行为 26 | const result = obtainBuf(mockWorkerReq) 27 | expect(result).toBe(mockWorkerReq.chunk) 28 | }) 29 | 30 | it('应该能够处理空数组', () => { 31 | // 初始化空数组 32 | initBufService([]) 33 | 34 | // 验证仍然可以正常工作 35 | const result = obtainBuf(mockWorkerReq) 36 | expect(result).toBe(mockWorkerReq.chunk) 37 | }) 38 | 39 | it('应该能够处理单个 ArrayBuffer', () => { 40 | const singleBuffer = [new ArrayBuffer(8)] 41 | initBufService(singleBuffer) 42 | 43 | const result = obtainBuf(mockWorkerReq) 44 | expect(result).toBe(mockWorkerReq.chunk) 45 | }) 46 | }) 47 | 48 | describe('obtainBuf', () => { 49 | beforeEach(() => { 50 | initBufService(mockArrayBuffers) 51 | }) 52 | 53 | it('应该返回请求中的 chunk', () => { 54 | const result = obtainBuf(mockWorkerReq) 55 | expect(result).toBe(mockWorkerReq.chunk) 56 | }) 57 | 58 | it('应该返回不同大小的 ArrayBuffer', () => { 59 | const largeChunk = new ArrayBuffer(1024) 60 | const largeReq: WorkerReq = { 61 | chunk: largeChunk, 62 | strategy: Strategy.md5, 63 | } 64 | 65 | const result = obtainBuf(largeReq) 66 | expect(result).toBe(largeChunk) 67 | expect(result.byteLength).toBe(1024) 68 | }) 69 | 70 | it('应该处理不同的策略类型', () => { 71 | const reqWithDifferentStrategy: WorkerReq = { 72 | chunk: new ArrayBuffer(16), 73 | strategy: Strategy.xxHash128, 74 | } 75 | 76 | const result = obtainBuf(reqWithDifferentStrategy) 77 | expect(result).toBe(reqWithDifferentStrategy.chunk) 78 | }) 79 | }) 80 | 81 | describe('restoreBuf', () => { 82 | beforeEach(() => { 83 | initBufService(mockArrayBuffers) 84 | }) 85 | 86 | it('应该正确恢复指定索引的 ArrayBuffer', () => { 87 | const newBuffer = new ArrayBuffer(64) 88 | const index = 1 89 | 90 | // 恢复指定索引的 buffer 91 | restoreBuf({ buf: newBuffer, index }) 92 | 93 | // 验证恢复是否成功（通过间接方式验证，因为内部状态是私有的） 94 | // 这里我们主要测试函数调用不会抛出错误 95 | expect(() => { 96 | restoreBuf({ buf: newBuffer, index }) 97 | }).not.toThrow() 98 | }) 99 | 100 | it('应该能够恢复第一个位置的 ArrayBuffer', () => { 101 | const newBuffer = new ArrayBuffer(128) 102 | const index = 0 103 | 104 | expect(() => { 105 | restoreBuf({ buf: newBuffer, index }) 106 | }).not.toThrow() 107 | }) 108 | 109 | it('应该能够恢复最后一个位置的 ArrayBuffer', () => { 110 | const newBuffer = new ArrayBuffer(256) 111 | const index = mockArrayBuffers.length - 1 112 | 113 | expect(() => { 114 | restoreBuf({ buf: newBuffer, index }) 115 | }).not.toThrow() 116 | }) 117 | 118 | it('应该抛出错误当索引超出原数组长度', () => { 119 | const newBuffer = new ArrayBuffer(512) 120 | const index = 999 121 | 122 | expect(() => { 123 | restoreBuf({ buf: newBuffer, index }) 124 | }).toThrow('Index 999 is out of bounds (array length: 3)') 125 | }) 126 | 127 | it('应该能够处理空 ArrayBuffer', () => { 128 | const emptyBuffer = new ArrayBuffer(0) 129 | const index = 0 130 | 131 | expect(() => { 132 | restoreBuf({ buf: emptyBuffer, index }) 133 | }).not.toThrow() 134 | }) 135 | }) 136 | 137 | describe('集成测试', () => { 138 | it('应该能够完整地初始化、获取和恢复 ArrayBuffer', () => { 139 | // 1. 初始化 140 | const testBuffers = [new ArrayBuffer(8), new ArrayBuffer(16), new ArrayBuffer(32)] 141 | initBufService(testBuffers) 142 | 143 | // 2. 获取 buffer 144 | const testReq: WorkerReq = { 145 | chunk: new ArrayBuffer(24), 146 | strategy: Strategy.md5, 147 | } 148 | const obtainedBuffer = obtainBuf(testReq) 149 | expect(obtainedBuffer).toBe(testReq.chunk) 150 | 151 | // 3. 恢复 buffer 152 | const newBuffer = new ArrayBuffer(48) 153 | expect(() => { 154 | restoreBuf({ buf: newBuffer, index: 1 }) 155 | }).not.toThrow() 156 | }) 157 | 158 | it('应该能够处理多次连续操作', () => { 159 | // 初始化 160 | initBufService([new ArrayBuffer(8), new ArrayBuffer(16)]) 161 | 162 | // 多次获取 163 | for (let i = 0; i < 5; i++) { 164 | const req: WorkerReq = { 165 | chunk: new ArrayBuffer(i * 8), 166 | strategy: Strategy.md5, 167 | } 168 | const result = obtainBuf(req) 169 | expect(result).toBe(req.chunk) 170 | } 171 | 172 | // 多次恢复 - 只使用有效的索引 173 | for (let i = 0; i < 2; i++) { 174 | expect(() => { 175 | restoreBuf({ buf: new ArrayBuffer(i * 16), index: i }) 176 | }).not.toThrow() 177 | } 178 | }) 179 | }) 180 | 181 | describe('边界情况测试', () => { 182 | it('应该能够处理非常大的 ArrayBuffer', () => { 183 | const largeBuffer = new ArrayBuffer(1024 * 1024) // 1MB 184 | initBufService([largeBuffer]) 185 | 186 | const largeReq: WorkerReq = { 187 | chunk: largeBuffer, 188 | strategy: Strategy.md5, 189 | } 190 | 191 | const result = obtainBuf(largeReq) 192 | expect(result).toBe(largeBuffer) 193 | expect(result.byteLength).toBe(1024 * 1024) 194 | }) 195 | 196 | it('应该抛出错误当使用负数索引', () => { 197 | initBufService(mockArrayBuffers) 198 | 199 | expect(() => { 200 | restoreBuf({ buf: new ArrayBuffer(8), index: -1 }) 201 | }).toThrow('Invalid index: must be a non-negative integer') 202 | }) 203 | 204 | it('应该抛出错误当使用浮点数索引', () => { 205 | initBufService(mockArrayBuffers) 206 | 207 | expect(() => { 208 | restoreBuf({ buf: new ArrayBuffer(8), index: 1.5 }) 209 | }).toThrow('Invalid index: must be a non-negative integer') 210 | }) 211 | }) 212 | 213 | describe('错误处理', () => { 214 | it('应该抛出错误当 obtainBuf 接收到无效的 WorkerReq', () => { 215 | initBufService(mockArrayBuffers) 216 | 217 | expect(() => { 218 | obtainBuf(null as any) 219 | }).toThrow('Invalid WorkerReq: chunk is required') 220 | 221 | expect(() => { 222 | obtainBuf({ chunk: null, strategy: Strategy.md5 } as any) 223 | }).toThrow('Invalid WorkerReq: chunk is required') 224 | 225 | expect(() => { 226 | obtainBuf({ chunk: 'not an ArrayBuffer', strategy: Strategy.md5 } as any) 227 | }).toThrow('Invalid chunk: must be an ArrayBuffer') 228 | }) 229 | 230 | it('应该抛出错误当 restoreBuf 接收到无效的 buffer', () => { 231 | initBufService(mockArrayBuffers) 232 | 233 | expect(() => { 234 | restoreBuf({ buf: null as any, index: 0 }) 235 | }).toThrow('Invalid buffer: must be an ArrayBuffer') 236 | 237 | expect(() => { 238 | restoreBuf({ buf: 'not an ArrayBuffer' as any, index: 0 }) 239 | }).toThrow('Invalid buffer: must be an ArrayBuffer') 240 | }) 241 | 242 | it('应该抛出错误当 initBufService 接收到非数组参数', () => { 243 | expect(() => { 244 | initBufService('not an array' as any) 245 | }).toThrow('Buffers must be an array') 246 | 247 | expect(() => { 248 | initBufService(null as any) 249 | }).toThrow('Buffers must be an array') 250 | }) 251 | }) 252 | }) 253 | -------------------------------------------------------------------------------- /packages/core/__tests__/node/shared/workerService.spec.ts: -------------------------------------------------------------------------------- 1 | // Mock the arrayBufferService module 2 | jest.mock('../../../src/shared/arrayBufferService', () => ({ 3 | initBufService: jest.fn(), 4 | clearBufService: jest.fn(), 5 | })) 6 | 7 | import { WorkerService } from '../../../src/shared/workerService' 8 | import { Strategy } from '../../../src/types' 9 | import * as arrayBufferService from '../../../src/shared/arrayBufferService' 10 | 11 | // Mock BaseWorkerPool 12 | class MockWorkerPool { 13 | exec = jest.fn() 14 | adjustPool = jest.fn() 15 | terminate = jest.fn() 16 | pool = [] 17 | maxWorkerCount = 4 18 | taskQueue = [] 19 | isProcessing = false 20 | getPoolStatus = jest.fn() 21 | } 22 | 23 | describe('WorkerService', () => { 24 | let workerService: any 25 | let mockPool: MockWorkerPool 26 | 27 | beforeEach(() => { 28 | mockPool = new MockWorkerPool() 29 | workerService = new WorkerService(mockPool as any) 30 | jest.clearAllMocks() 31 | }) 32 | 33 | describe('constructor', () => { 34 | it('应该正确初始化 WorkerService', () => { 35 | const pool = new MockWorkerPool() 36 | const service = new WorkerService(pool as any) 37 | 38 | expect(service).toBeInstanceOf(WorkerService) 39 | expect(service.getHashForFiles).toBeDefined() 40 | expect(service.adjustWorkerPoolSize).toBeDefined() 41 | expect(service.terminate).toBeDefined() 42 | expect(service.isActive()).toBe(true) 43 | }) 44 | }) 45 | 46 | describe('getHashForFiles', () => { 47 | it('应该正确调用 pool.exec 并返回结果', async () => { 48 | const chunks = [new ArrayBuffer(8), new ArrayBuffer(16)] 49 | const strategy = Strategy.md5 50 | const expectedResult = ['hash1', 'hash2'] 51 | 52 | mockPool.exec.mockResolvedValue([ 53 | { success: true, data: 'hash1', index: 0 }, 54 | { success: true, data: 'hash2', index: 1 }, 55 | ]) 56 | 57 | const result = await workerService.getHashForFiles(chunks, strategy) 58 | 59 | expect(arrayBufferService.initBufService).toHaveBeenCalledWith(chunks) 60 | expect(mockPool.exec).toHaveBeenCalledWith( 61 | [ 62 | { chunk: chunks[0], strategy }, 63 | { chunk: chunks[1], strategy }, 64 | ], 65 | undefined, 66 | ) 67 | expect(result).toEqual(expectedResult) 68 | }) 69 | 70 | it('应该处理空数组的情况', async () => { 71 | const chunks: ArrayBuffer[] = [] 72 | const strategy = Strategy.md5 73 | const expectedResult: string[] = [] 74 | 75 | const result = await workerService.getHashForFiles(chunks, strategy) 76 | 77 | expect(result).toEqual(expectedResult) 78 | // 空数组情况下不会调用 initBufService 和 pool.exec 79 | expect(arrayBufferService.initBufService).not.toHaveBeenCalled() 80 | expect(mockPool.exec).not.toHaveBeenCalled() 81 | }) 82 | 83 | it('应该处理单个 chunk 的情况', async () => { 84 | const chunks = [new ArrayBuffer(32)] 85 | const strategy = Strategy.xxHash128 86 | const expectedResult = ['single-hash'] 87 | 88 | mockPool.exec.mockResolvedValue([{ success: true, data: 'single-hash', index: 0 }]) 89 | 90 | const result = await workerService.getHashForFiles(chunks, strategy) 91 | 92 | expect(arrayBufferService.initBufService).toHaveBeenCalledWith(chunks) 93 | expect(mockPool.exec).toHaveBeenCalledWith([{ chunk: chunks[0], strategy }], undefined) 94 | expect(result).toEqual(expectedResult) 95 | }) 96 | 97 | it('应该正确处理 pool.exec 抛出的错误', async () => { 98 | const chunks = [new ArrayBuffer(8)] 99 | const strategy = Strategy.md5 100 | const error = new Error('Worker execution failed') 101 | 102 | mockPool.exec.mockRejectedValue(error) 103 | 104 | await expect(workerService.getHashForFiles(chunks, strategy)).rejects.toThrow( 105 | 'Worker execution failed', 106 | ) 107 | expect(arrayBufferService.initBufService).toHaveBeenCalledWith(chunks) 108 | expect(mockPool.exec).toHaveBeenCalledWith([{ chunk: chunks[0], strategy }], undefined) 109 | }) 110 | 111 | it('应该正确处理部分任务失败的情况', async () => { 112 | const chunks = [new ArrayBuffer(8), new ArrayBuffer(16)] 113 | const strategy = Strategy.md5 114 | 115 | mockPool.exec.mockResolvedValue([ 116 | { success: true, data: 'hash1', index: 0 }, 117 | { success: false, error: new Error('Task failed'), index: 1 }, 118 | ]) 119 | 120 | await expect(workerService.getHashForFiles(chunks, strategy)).rejects.toThrow( 121 | 'Hash calculation failed for 1 chunks: Chunk 1: Task failed', 122 | ) 123 | expect(arrayBufferService.initBufService).toHaveBeenCalledWith(chunks) 124 | expect(mockPool.exec).toHaveBeenCalledWith( 125 | [ 126 | { chunk: chunks[0], strategy }, 127 | { chunk: chunks[1], strategy }, 128 | ], 129 | undefined, 130 | ) 131 | }) 132 | }) 133 | 134 | describe('adjustWorkerPoolSize', () => { 135 | it('应该正确调用 pool.adjustPool', () => { 136 | const workerCount = 6 137 | 138 | workerService.adjustWorkerPoolSize(workerCount) 139 | 140 | expect(mockPool.adjustPool).toHaveBeenCalledWith(workerCount) 141 | }) 142 | 143 | it('应该处理增加 worker 数量的情况', () => { 144 | const newWorkerCount = 10 145 | 146 | workerService.adjustWorkerPoolSize(newWorkerCount) 147 | 148 | expect(mockPool.adjustPool).toHaveBeenCalledWith(newWorkerCount) 149 | }) 150 | 151 | it('应该处理减少 worker 数量的情况', () => { 152 | const newWorkerCount = 2 153 | 154 | workerService.adjustWorkerPoolSize(newWorkerCount) 155 | 156 | expect(mockPool.adjustPool).toHaveBeenCalledWith(newWorkerCount) 157 | }) 158 | 159 | it('应该处理 worker 数量为 0 的情况', () => { 160 | const newWorkerCount = 0 161 | 162 | expect(() => workerService.adjustWorkerPoolSize(newWorkerCount)).toThrow( 163 | 'Worker count must be at least 1', 164 | ) 165 | }) 166 | }) 167 | 168 | describe('terminate', () => { 169 | it('应该正确调用 pool.terminate 并设置 pool 为 null', () => { 170 | workerService.terminate() 171 | 172 | expect(mockPool.terminate).toHaveBeenCalled() 173 | // 由于 pool 是 protected 属性，我们通过其他方式验证 174 | // 这里我们验证 terminate 方法被正确调用 175 | }) 176 | 177 | it('应该处理 pool 为 null 的情况', () => { 178 | // 先终止一次 179 | workerService.terminate() 180 | expect(mockPool.terminate).toHaveBeenCalledTimes(1) 181 | 182 | // 再次终止，应该不会出错 183 | workerService.terminate() 184 | expect(mockPool.terminate).toHaveBeenCalledTimes(1) // 仍然只调用一次，因为第二次时 pool 已经是 null 185 | }) 186 | }) 187 | 188 | describe('集成测试', () => { 189 | it('应该能够完整地执行工作流程', async () => { 190 | const chunks = [new ArrayBuffer(8), new ArrayBuffer(16)] 191 | const strategy = Strategy.md5 192 | const expectedResult = ['hash1', 'hash2'] 193 | 194 | mockPool.exec.mockResolvedValue([ 195 | { success: true, data: 'hash1', index: 0 }, 196 | { success: true, data: 'hash2', index: 1 }, 197 | ]) 198 | 199 | // 执行 hash 计算 200 | const result = await workerService.getHashForFiles(chunks, strategy) 201 | expect(result).toEqual(expectedResult) 202 | 203 | // 调整 worker 池大小 204 | workerService.adjustWorkerPoolSize(6) 205 | expect(mockPool.adjustPool).toHaveBeenCalledWith(6) 206 | 207 | // 终止服务 208 | workerService.terminate() 209 | expect(mockPool.terminate).toHaveBeenCalled() 210 | }) 211 | 212 | it('应该能够处理不同策略的 hash 计算', async () => { 213 | const chunks = [new ArrayBuffer(32)] 214 | const strategies = [Strategy.md5, Strategy.xxHash128] 215 | 216 | for (const strategy of strategies) { 217 | const expectedResult = [`${strategy}-hash`] 218 | mockPool.exec.mockResolvedValue([{ success: true, data: `${strategy}-hash`, index: 0 }]) 219 | 220 | const result = await workerService.getHashForFiles(chunks, strategy) 221 | expect(result).toEqual(expectedResult) 222 | expect(mockPool.exec).toHaveBeenCalledWith([{ chunk: chunks[0], strategy }], undefined) 223 | } 224 | }) 225 | }) 226 | }) 227 | -------------------------------------------------------------------------------- /packages/core/__tests__/node/node/nodeUtils.spec.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs' 2 | import fsp from 'fs/promises' 3 | import { 4 | readFileAsArrayBuffer, 5 | getFileSliceLocations, 6 | getFileMetadata, 7 | } from '../../../src/node/nodeUtils' 8 | 9 | // Mock fs 模块 10 | jest.mock('fs') 11 | jest.mock('fs/promises') 12 | 13 | const mockFs = fs as jest.Mocked 14 | const mockFsp = fsp as jest.Mocked 15 | 16 | describe('nodeUtils', () => { 17 | beforeEach(() => { 18 | jest.clearAllMocks() 19 | }) 20 | 21 | describe('readFileAsArrayBuffer', () => { 22 | it('应该正确读取文件并返回 ArrayBuffer', async () => { 23 | const mockPath = '/test/file.txt' 24 | const mockData = Buffer.from('Hello World') 25 | const mockReadStream = { 26 | on: jest.fn(), 27 | } as any 28 | 29 | mockFs.createReadStream.mockReturnValue(mockReadStream) 30 | 31 | const promise = readFileAsArrayBuffer(mockPath, 0, 10) 32 | 33 | // 模拟数据事件 34 | const dataCallback = mockReadStream.on.mock.calls.find((call: any) => call[0] === 'data')[1] 35 | dataCallback(mockData) 36 | 37 | // 模拟结束事件 38 | const endCallback = mockReadStream.on.mock.calls.find((call: any) => call[0] === 'end')[1] 39 | endCallback() 40 | 41 | const result = await promise 42 | 43 | expect(mockFs.createReadStream).toHaveBeenCalledWith(mockPath, { start: 0, end: 10 }) 44 | expect(result).toBeInstanceOf(ArrayBuffer) 45 | expect(result.byteLength).toBe(mockData.length) 46 | }) 47 | 48 | it('应该在读取错误时抛出异常', async () => { 49 | const mockPath = '/test/file.txt' 50 | const mockError = new Error('File not found') 51 | const mockReadStream = { 52 | on: jest.fn(), 53 | } as any 54 | 55 | mockFs.createReadStream.mockReturnValue(mockReadStream) 56 | 57 | const promise = readFileAsArrayBuffer(mockPath, 0, 10) 58 | 59 | // 模拟错误事件 60 | const errorCallback = mockReadStream.on.mock.calls.find((call: any) => call[0] === 'error')[1] 61 | errorCallback(mockError) 62 | 63 | await expect(promise).rejects.toThrow('File not found') 64 | }) 65 | 66 | it('应该处理多个数据块', async () => { 67 | const mockPath = '/test/file.txt' 68 | const mockData1 = Buffer.from('Hello ') 69 | const mockData2 = Buffer.from('World') 70 | const mockReadStream = { 71 | on: jest.fn(), 72 | } as any 73 | 74 | mockFs.createReadStream.mockReturnValue(mockReadStream) 75 | 76 | const promise = readFileAsArrayBuffer(mockPath, 0, 10) 77 | 78 | // 模拟多个数据事件 79 | const dataCallback = mockReadStream.on.mock.calls.find((call: any) => call[0] === 'data')[1] 80 | dataCallback(mockData1) 81 | dataCallback(mockData2) 82 | 83 | // 模拟结束事件 84 | const endCallback = mockReadStream.on.mock.calls.find((call: any) => call[0] === 'end')[1] 85 | endCallback() 86 | 87 | const result = await promise 88 | 89 | expect(result).toBeInstanceOf(ArrayBuffer) 90 | expect(result.byteLength).toBe(mockData1.length + mockData2.length) 91 | }) 92 | }) 93 | 94 | describe('getFileSliceLocations', () => { 95 | it('应该正确计算文件分片位置', async () => { 96 | const mockPath = '/test/file.txt' 97 | const mockStats = { 98 | size: 2097152, // 2MB 99 | } 100 | 101 | mockFsp.stat.mockResolvedValue(mockStats as any) 102 | 103 | const result = await getFileSliceLocations(mockPath, 1) 104 | 105 | expect(mockFsp.stat).toHaveBeenCalledWith(mockPath) 106 | expect(result.sliceLocation).toHaveLength(2) 107 | expect(result.sliceLocation[0]).toEqual([0, 1048575]) // 0 到 1MB-1 108 | expect(result.sliceLocation[1]).toEqual([1048576, 2097151]) // 1MB 到 2MB-1 109 | expect(result.endLocation).toBe(2097152) 110 | }) 111 | 112 | it('应该使用自定义分块大小', async () => { 113 | const mockPath = '/test/file.txt' 114 | const mockStats = { 115 | size: 3145728, // 3MB 116 | } 117 | 118 | mockFsp.stat.mockResolvedValue(mockStats as any) 119 | 120 | const result = await getFileSliceLocations(mockPath, 2) // 2MB 分块 121 | 122 | expect(result.sliceLocation).toHaveLength(2) 123 | expect(result.sliceLocation[0]).toEqual([0, 2097151]) // 0 到 2MB-1 124 | expect(result.sliceLocation[1]).toEqual([2097152, 3145727]) // 2MB 到文件末尾 (3MB-1) 125 | }) 126 | 127 | it('应该处理小于分块大小的文件', async () => { 128 | const mockPath = '/test/small.txt' 129 | const mockStats = { 130 | size: 512000, // 500KB 131 | } 132 | 133 | mockFsp.stat.mockResolvedValue(mockStats as any) 134 | 135 | const result = await getFileSliceLocations(mockPath, 1) 136 | 137 | expect(result.sliceLocation).toHaveLength(1) 138 | expect(result.sliceLocation[0]).toEqual([0, 511999]) // 0 到文件末尾 (500KB-1) 139 | expect(result.endLocation).toBe(512000) 140 | }) 141 | 142 | it('应该在 baseSize 小于等于 0 时抛出错误', async () => { 143 | const mockPath = '/test/file.txt' 144 | 145 | await expect(getFileSliceLocations(mockPath, 0)).rejects.toThrow( 146 | 'baseSize must be greater than 0', 147 | ) 148 | await expect(getFileSliceLocations(mockPath, -1)).rejects.toThrow( 149 | 'baseSize must be greater than 0', 150 | ) 151 | }) 152 | 153 | it('应该处理空文件', async () => { 154 | const mockPath = '/test/empty.txt' 155 | const mockStats = { 156 | size: 0, 157 | } 158 | 159 | mockFsp.stat.mockResolvedValue(mockStats as any) 160 | 161 | const result = await getFileSliceLocations(mockPath, 1) 162 | 163 | expect(result.sliceLocation).toHaveLength(0) 164 | expect(result.endLocation).toBe(0) 165 | }) 166 | }) 167 | 168 | describe('getFileMetadata', () => { 169 | it('应该正确获取文件元数据', async () => { 170 | const mockPath = '/test/example.txt' 171 | const mockTime = new Date('2023-01-01T00:00:00Z') 172 | const mockStats = { 173 | size: 2048, // 2KB 174 | mtime: mockTime, 175 | } 176 | 177 | mockFsp.stat.mockResolvedValue(mockStats as any) 178 | 179 | const result = await getFileMetadata(mockPath) 180 | 181 | expect(mockFsp.stat).toHaveBeenCalledWith(mockPath) 182 | expect(result).toEqual({ 183 | name: 'example.txt', 184 | size: 2, // 2KB / 1024 = 2 185 | lastModified: mockTime.getTime(), 186 | type: '.txt', 187 | }) 188 | }) 189 | 190 | it('应该处理没有扩展名的文件', async () => { 191 | const mockPath = '/test/README' 192 | const mockTime = new Date('2023-01-01T00:00:00Z') 193 | const mockStats = { 194 | size: 1024, // 1KB 195 | mtime: mockTime, 196 | } 197 | 198 | mockFsp.stat.mockResolvedValue(mockStats as any) 199 | 200 | const result = await getFileMetadata(mockPath) 201 | 202 | expect(result).toEqual({ 203 | name: 'README', 204 | size: 1, // 1KB / 1024 = 1 205 | lastModified: mockTime.getTime(), 206 | type: '', // 没有扩展名 207 | }) 208 | }) 209 | 210 | it('应该处理大文件', async () => { 211 | const mockPath = '/test/large.zip' 212 | const mockTime = new Date('2023-01-01T00:00:00Z') 213 | const mockStats = { 214 | size: 5242880, // 5MB 215 | mtime: mockTime, 216 | } 217 | 218 | mockFsp.stat.mockResolvedValue(mockStats as any) 219 | 220 | const result = await getFileMetadata(mockPath) 221 | 222 | expect(result).toEqual({ 223 | name: 'large.zip', 224 | size: 5120, // 5MB / 1024 = 5120 225 | lastModified: mockTime.getTime(), 226 | type: '.zip', 227 | }) 228 | }) 229 | 230 | it('应该处理复杂路径', async () => { 231 | const mockPath = '/path/to/deep/nested/file.js' 232 | const mockTime = new Date('2023-01-01T00:00:00Z') 233 | const mockStats = { 234 | size: 3072, // 3KB 235 | mtime: mockTime, 236 | } 237 | 238 | mockFsp.stat.mockResolvedValue(mockStats as any) 239 | 240 | const result = await getFileMetadata(mockPath) 241 | 242 | expect(result).toEqual({ 243 | name: 'file.js', 244 | size: 3, // 3KB / 1024 = 3 245 | lastModified: mockTime.getTime(), 246 | type: '.js', 247 | }) 248 | }) 249 | }) 250 | }) 251 | -------------------------------------------------------------------------------- /packages/core/__tests__/node/browser/browserUtils.spec.ts: -------------------------------------------------------------------------------- 1 | import { 2 | getArrayBufFromBlobs, 3 | sliceFile, 4 | getFileMetadataInBrowser, 5 | } from '../../../src/browser/browserUtils' 6 | 7 | // 注入 TextDecoder 以兼容 jsdom 环境 8 | if (typeof global.TextDecoder === 'undefined') { 9 | global.TextDecoder = require('util').TextDecoder 10 | } 11 | 12 | // Mock Blob.arrayBuffer method 13 | const originalArrayBuffer = Blob.prototype.arrayBuffer 14 | beforeAll(() => { 15 | if (!Blob.prototype.arrayBuffer) { 16 | Blob.prototype.arrayBuffer = function () { 17 | return new Promise((resolve) => { 18 | const reader = new FileReader() 19 | reader.onload = () => resolve(reader.result as ArrayBuffer) 20 | reader.readAsArrayBuffer(this) 21 | }) 22 | } 23 | } 24 | }) 25 | 26 | afterAll(() => { 27 | if (originalArrayBuffer) { 28 | Blob.prototype.arrayBuffer = originalArrayBuffer 29 | } 30 | }) 31 | 32 | describe('browserUtils', () => { 33 | describe('getArrayBufFromBlobs', () => { 34 | it('应该将 Blob 数组转换为 ArrayBuffer 数组', async () => { 35 | const text1 = 'Hello World' 36 | const text2 = 'Test Data' 37 | const blob1 = new Blob([text1], { type: 'text/plain' }) 38 | const blob2 = new Blob([text2], { type: 'text/plain' }) 39 | 40 | const result = await getArrayBufFromBlobs([blob1, blob2]) 41 | 42 | expect(result).toHaveLength(2) 43 | expect(result[0]).toBeInstanceOf(ArrayBuffer) 44 | expect(result[1]).toBeInstanceOf(ArrayBuffer) 45 | 46 | // 验证内容 47 | const decoder = new TextDecoder() 48 | expect(decoder.decode(result[0])).toBe(text1) 49 | expect(decoder.decode(result[1])).toBe(text2) 50 | }) 51 | 52 | it('应该处理空数组', async () => { 53 | const result = await getArrayBufFromBlobs([]) 54 | expect(result).toEqual([]) 55 | }) 56 | 57 | it('应该处理单个 Blob', async () => { 58 | const text = 'Single Blob Test' 59 | const blob = new Blob([text], { type: 'text/plain' }) 60 | 61 | const result = await getArrayBufFromBlobs([blob]) 62 | 63 | expect(result).toHaveLength(1) 64 | expect(result[0]).toBeInstanceOf(ArrayBuffer) 65 | 66 | const decoder = new TextDecoder() 67 | expect(decoder.decode(result[0])).toBe(text) 68 | }) 69 | }) 70 | 71 | describe('sliceFile', () => { 72 | it('应该按指定大小分割文件', () => { 73 | const fileContent = 'A'.repeat(3 * 1024 * 1024) // 3MB 74 | const file = new File([fileContent], 'test.txt', { type: 'text/plain' }) 75 | 76 | const chunks = sliceFile(file, 1) // 1MB chunks 77 | 78 | expect(chunks).toHaveLength(3) 79 | chunks.forEach((chunk: Blob) => { 80 | expect(chunk.size).toBeLessThanOrEqual(1024 * 1024) 81 | }) 82 | }) 83 | 84 | it('应该使用默认分块大小', () => { 85 | const fileContent = 'A'.repeat(2 * 1024 * 1024) // 2MB 86 | const file = new File([fileContent], 'test.txt', { type: 'text/plain' }) 87 | 88 | const chunks = sliceFile(file) // 默认 1MB 89 | 90 | expect(chunks).toHaveLength(2) 91 | chunks.forEach((chunk: Blob) => { 92 | expect(chunk.size).toBeLessThanOrEqual(1024 * 1024) 93 | }) 94 | }) 95 | 96 | it('应该处理小于分块大小的文件', () => { 97 | const fileContent = 'Small file content' 98 | const file = new File([fileContent], 'small.txt', { type: 'text/plain' }) 99 | 100 | const chunks = sliceFile(file, 1) 101 | 102 | expect(chunks).toHaveLength(1) 103 | expect(chunks[0].size).toBe(fileContent.length) 104 | }) 105 | 106 | it('应该处理自定义分块大小', () => { 107 | const fileContent = 'A'.repeat(5 * 1024 * 1024) // 5MB 108 | const file = new File([fileContent], 'test.txt', { type: 'text/plain' }) 109 | 110 | const chunks = sliceFile(file, 2) // 2MB chunks 111 | 112 | expect(chunks).toHaveLength(3) // 2MB + 2MB + 1MB 113 | expect(chunks[0].size).toBe(2 * 1024 * 1024) 114 | expect(chunks[1].size).toBe(2 * 1024 * 1024) 115 | expect(chunks[2].size).toBe(1 * 1024 * 1024) 116 | }) 117 | 118 | it('应该处理小于 1MB 的分块大小', () => { 119 | const fileContent = 'A'.repeat(1024 * 1024) // 1MB 120 | const file = new File([fileContent], 'test.txt', { type: 'text/plain' }) 121 | 122 | const chunks = sliceFile(file, 0.5) // 0.5MB chunks 123 | 124 | expect(chunks).toHaveLength(2) 125 | chunks.forEach((chunk: Blob) => { 126 | expect(chunk.size).toBeLessThanOrEqual(0.5 * 1024 * 1024) 127 | }) 128 | }) 129 | 130 | it('应该抛出错误当 baseSize 小于等于 0', () => { 131 | const fileContent = 'Test content' 132 | const file = new File([fileContent], 'test.txt', { type: 'text/plain' }) 133 | 134 | expect(() => sliceFile(file, 0)).toThrow('baseSize must be greater than 0') 135 | expect(() => sliceFile(file, -1)).toThrow('baseSize must be greater than 0') 136 | }) 137 | 138 | it('应该处理空文件', () => { 139 | const file = new File([], 'empty.txt', { type: 'text/plain' }) 140 | 141 | const chunks = sliceFile(file, 1) 142 | 143 | // 空文件应该返回一个空的分块 144 | expect(chunks).toHaveLength(1) 145 | expect(chunks[0].size).toBe(0) 146 | }) 147 | }) 148 | 149 | describe('getFileMetadataInBrowser', () => { 150 | it('应该正确提取文件元数据', async () => { 151 | const fileContent = 'Test file content' 152 | const fileName = 'test-document.pdf' 153 | const fileSize = fileContent.length 154 | const lastModified = Date.now() 155 | 156 | const file = new File([fileContent], fileName, { 157 | type: 'application/pdf', 158 | lastModified, 159 | }) 160 | 161 | const metadata = await getFileMetadataInBrowser(file) 162 | 163 | expect(metadata).toEqual({ 164 | name: fileName, 165 | size: fileSize / 1024, // 转换为 KB 166 | lastModified, 167 | type: '.pdf', 168 | }) 169 | }) 170 | 171 | it('应该处理没有扩展名的文件', async () => { 172 | const fileContent = 'No extension content' 173 | const fileName = 'noextension' 174 | const file = new File([fileContent], fileName, { type: 'text/plain' }) 175 | 176 | const metadata = await getFileMetadataInBrowser(file) 177 | 178 | expect(metadata.type).toBe('') 179 | }) 180 | 181 | it('应该处理多个点的文件名', async () => { 182 | const fileContent = 'Multiple dots content' 183 | const fileName = 'file.name.with.dots.txt' 184 | const file = new File([fileContent], fileName, { type: 'text/plain' }) 185 | 186 | const metadata = await getFileMetadataInBrowser(file) 187 | 188 | expect(metadata.type).toBe('.txt') 189 | }) 190 | 191 | it('应该处理以点开头的文件名', async () => { 192 | const fileContent = 'Hidden file content' 193 | const fileName = '.hiddenfile' 194 | const file = new File([fileContent], fileName, { type: 'text/plain' }) 195 | 196 | const metadata = await getFileMetadataInBrowser(file) 197 | 198 | // 以点开头的文件名应该返回空字符串作为扩展名 199 | expect(metadata.type).toBe('') 200 | }) 201 | 202 | it('应该处理以点结尾的文件名', async () => { 203 | const fileContent = 'Ending with dot content' 204 | const fileName = 'file.' 205 | const file = new File([fileContent], fileName, { type: 'text/plain' }) 206 | 207 | const metadata = await getFileMetadataInBrowser(file) 208 | 209 | // 以点结尾的文件名应该返回空字符串作为扩展名 210 | expect(metadata.type).toBe('') 211 | }) 212 | 213 | it('应该处理空文件名', async () => { 214 | const fileContent = 'Empty name content' 215 | const fileName = '' 216 | const file = new File([fileContent], fileName, { type: 'text/plain' }) 217 | 218 | const metadata = await getFileMetadataInBrowser(file) 219 | 220 | expect(metadata.name).toBe('') 221 | expect(metadata.type).toBe('') 222 | }) 223 | 224 | it('应该处理大文件', async () => { 225 | const fileContent = 'A'.repeat(10 * 1024 * 1024) // 10MB 226 | const fileName = 'large-file.dat' 227 | const file = new File([fileContent], fileName, { type: 'application/octet-stream' }) 228 | 229 | const metadata = await getFileMetadataInBrowser(file) 230 | 231 | expect(metadata.size).toBe(10 * 1024) // 10MB in KB 232 | expect(metadata.type).toBe('.dat') 233 | }) 234 | 235 | it('应该处理特殊字符的文件名', async () => { 236 | const fileContent = 'Special chars content' 237 | const fileName = 'file-with-special-chars!@#$%^&*()_+-=[]{}|;:,.<>?.txt' 238 | const file = new File([fileContent], fileName, { type: 'text/plain' }) 239 | 240 | const metadata = await getFileMetadataInBrowser(file) 241 | 242 | expect(metadata.name).toBe(fileName) 243 | expect(metadata.type).toBe('.txt') 244 | }) 245 | }) 246 | }) 247 | -------------------------------------------------------------------------------- /packages/core/__tests__/node/shared/merkleTree.spec.ts: -------------------------------------------------------------------------------- 1 | import { MerkleTree, MerkleNode, HashFn } from '../../../src/shared/merkleTree' 2 | 3 | describe('MerkleTree', () => { 4 | describe('MerkleNode', () => { 5 | it('应该正确创建 MerkleNode', () => { 6 | const node = new MerkleNode('hash123') 7 | expect(node.hash).toBe('hash123') 8 | expect(node.left).toBeNull() 9 | expect(node.right).toBeNull() 10 | }) 11 | 12 | it('应该正确创建带有子节点的 MerkleNode', () => { 13 | const leftChild = new MerkleNode('leftHash') 14 | const rightChild = new MerkleNode('rightHash') 15 | const node = new MerkleNode('parentHash', leftChild, rightChild) 16 | 17 | expect(node.hash).toBe('parentHash') 18 | expect(node.left).toBe(leftChild) 19 | expect(node.right).toBe(rightChild) 20 | }) 21 | 22 | it('应该正确创建只有左子节点的 MerkleNode', () => { 23 | const leftChild = new MerkleNode('leftHash') 24 | const node = new MerkleNode('parentHash', leftChild) 25 | 26 | expect(node.hash).toBe('parentHash') 27 | expect(node.left).toBe(leftChild) 28 | expect(node.right).toBeNull() 29 | }) 30 | }) 31 | 32 | describe('构造函数', () => { 33 | it('应该使用默认哈希函数创建 MerkleTree', () => { 34 | const tree = new MerkleTree() 35 | expect(tree.leafs).toEqual([]) 36 | }) 37 | 38 | it('应该使用自定义哈希函数创建 MerkleTree', async () => { 39 | const customHashFn: HashFn = async (hLeft, hRight) => { 40 | return hRight ? `custom_${hLeft}_${hRight}` : hLeft 41 | } 42 | 43 | const tree = new MerkleTree(customHashFn) 44 | await tree.init(['hash1', 'hash2']) 45 | const result = tree.getRootHash() 46 | expect(result).toBe('custom_hash1_hash2') 47 | }) 48 | }) 49 | 50 | describe('init', () => { 51 | it('应该使用字符串数组初始化', async () => { 52 | const tree = new MerkleTree() 53 | const hashList = ['hash1', 'hash2', 'hash3', 'hash4'] 54 | 55 | await tree.init(hashList) 56 | 57 | expect(tree.leafs).toHaveLength(4) 58 | expect(tree.leafs[0].hash).toBe('hash1') 59 | expect(tree.leafs[1].hash).toBe('hash2') 60 | expect(tree.leafs[2].hash).toBe('hash3') 61 | expect(tree.leafs[3].hash).toBe('hash4') 62 | }) 63 | 64 | it('应该使用 MerkleNode 数组初始化', async () => { 65 | const tree = new MerkleTree() 66 | const nodes = [new MerkleNode('hash1'), new MerkleNode('hash2'), new MerkleNode('hash3')] 67 | 68 | await tree.init(nodes) 69 | 70 | expect(tree.leafs).toHaveLength(3) 71 | expect(tree.leafs[0].hash).toBe('hash1') 72 | expect(tree.leafs[1].hash).toBe('hash2') 73 | expect(tree.leafs[2].hash).toBe('hash3') 74 | }) 75 | 76 | it('当传入空数组时应该抛出错误', async () => { 77 | const tree = new MerkleTree() 78 | 79 | await expect(tree.init([])).rejects.toThrow('无法使用空输入创建 Merkle 树') 80 | }) 81 | 82 | it('当传入空字符串数组时应该抛出错误', async () => { 83 | const tree = new MerkleTree() 84 | 85 | await expect(tree.init([] as string[])).rejects.toThrow('无法使用空输入创建 Merkle 树') 86 | }) 87 | 88 | it('当传入空节点数组时应该抛出错误', async () => { 89 | const tree = new MerkleTree() 90 | 91 | await expect(tree.init([] as any[])).rejects.toThrow('无法使用空输入创建 Merkle 树') 92 | }) 93 | }) 94 | 95 | describe('树构建', () => { 96 | it('应该构建单个节点的树', async () => { 97 | const tree = new MerkleTree() 98 | await tree.init(['singleHash']) 99 | 100 | const rootHash = tree.getRootHash() 101 | expect(rootHash).toBe('singleHash') 102 | expect(tree.root.left).toBeNull() 103 | expect(tree.root.right).toBeNull() 104 | }) 105 | 106 | it('应该构建两个节点的树', async () => { 107 | const tree = new MerkleTree() 108 | await tree.init(['hash1', 'hash2']) 109 | 110 | const rootHash = tree.getRootHash() 111 | expect(rootHash).toBeDefined() 112 | expect(tree.root.left).toBe(tree.leafs[0]) 113 | expect(tree.root.right).toBe(tree.leafs[1]) 114 | }) 115 | 116 | it('应该构建三个节点的树（奇数个叶子节点）', async () => { 117 | const tree = new MerkleTree() 118 | await tree.init(['hash1', 'hash2', 'hash3']) 119 | 120 | const rootHash = tree.getRootHash() 121 | expect(rootHash).toBeDefined() 122 | expect(tree.root.left).toBeDefined() 123 | expect(tree.root.right).toBeDefined() 124 | }) 125 | 126 | it('应该构建四个节点的树', async () => { 127 | const tree = new MerkleTree() 128 | await tree.init(['hash1', 'hash2', 'hash3', 'hash4']) 129 | 130 | const rootHash = tree.getRootHash() 131 | expect(rootHash).toBeDefined() 132 | expect(tree.root.left).toBeDefined() 133 | expect(tree.root.right).toBeDefined() 134 | }) 135 | 136 | it('应该使用自定义哈希函数构建树', async () => { 137 | const customHashFn: HashFn = async (hLeft, hRight) => { 138 | return hRight ? `combined_${hLeft}_${hRight}` : hLeft 139 | } 140 | 141 | const tree = new MerkleTree(customHashFn) 142 | await tree.init(['hash1', 'hash2']) 143 | 144 | const rootHash = tree.getRootHash() 145 | expect(rootHash).toBe('combined_hash1_hash2') 146 | }) 147 | }) 148 | 149 | describe('getRootHash', () => { 150 | it('应该返回根节点的哈希值', async () => { 151 | const tree = new MerkleTree() 152 | await tree.init(['hash1', 'hash2']) 153 | 154 | const rootHash = tree.getRootHash() 155 | expect(rootHash).toBeDefined() 156 | expect(typeof rootHash).toBe('string') 157 | }) 158 | 159 | it('应该返回单个节点的哈希值', async () => { 160 | const tree = new MerkleTree() 161 | await tree.init(['singleHash']) 162 | 163 | const rootHash = tree.getRootHash() 164 | expect(rootHash).toBe('singleHash') 165 | }) 166 | }) 167 | 168 | describe('calculateHash', () => { 169 | it('应该使用默认哈希函数计算哈希', async () => { 170 | const tree = new MerkleTree() 171 | const result = await tree['calculateHash']({ leftHash: 'left', rightHash: 'right' }) 172 | expect(result).toBeDefined() 173 | expect(typeof result).toBe('string') 174 | }) 175 | 176 | it('应该处理只有左子节点的情况', async () => { 177 | const tree = new MerkleTree() 178 | const result = await tree['calculateHash']({ leftHash: 'left' }) 179 | expect(result).toBe('left') 180 | }) 181 | }) 182 | 183 | describe('集成测试', () => { 184 | it('应该完整构建和验证 Merkle 树', async () => { 185 | const tree = new MerkleTree() 186 | const hashList = ['a', 'b', 'c', 'd'] 187 | 188 | await tree.init(hashList) 189 | 190 | expect(tree.leafs).toHaveLength(4) 191 | expect(tree.getRootHash()).toBeDefined() 192 | expect(tree.root.left).toBeDefined() 193 | expect(tree.root.right).toBeDefined() 194 | }) 195 | 196 | it('应该处理大量节点的树构建', async () => { 197 | const tree = new MerkleTree() 198 | const hashList = Array.from({ length: 8 }, (_, i) => `hash${i}`) 199 | 200 | await tree.init(hashList) 201 | 202 | expect(tree.leafs).toHaveLength(8) 203 | expect(tree.getRootHash()).toBeDefined() 204 | }) 205 | 206 | it('应该使用自定义哈希函数进行完整测试', async () => { 207 | const customHashFn: HashFn = async (hLeft, hRight) => { 208 | return hRight ? `${hLeft}+${hRight}` : hLeft 209 | } 210 | 211 | const tree = new MerkleTree(customHashFn) 212 | await tree.init(['a', 'b', 'c']) 213 | 214 | const rootHash = tree.getRootHash() 215 | expect(rootHash).toBeDefined() 216 | expect(rootHash).toContain('+') 217 | }) 218 | 219 | it('应该验证树的层级结构', async () => { 220 | const tree = new MerkleTree() 221 | await tree.init(['hash1', 'hash2', 'hash3', 'hash4']) 222 | 223 | // 验证根节点有左右子节点 224 | expect(tree.root.left).toBeDefined() 225 | expect(tree.root.right).toBeDefined() 226 | 227 | // 验证叶子节点没有子节点 228 | expect(tree.leafs[0].left).toBeNull() 229 | expect(tree.leafs[0].right).toBeNull() 230 | expect(tree.leafs[1].left).toBeNull() 231 | expect(tree.leafs[1].right).toBeNull() 232 | }) 233 | }) 234 | 235 | describe('边界情况', () => { 236 | it('应该处理只有一个叶子节点的情况', async () => { 237 | const tree = new MerkleTree() 238 | await tree.init(['single']) 239 | 240 | expect(tree.leafs).toHaveLength(1) 241 | expect(tree.getRootHash()).toBe('single') 242 | }) 243 | 244 | it('应该处理奇数个叶子节点的情况', async () => { 245 | const tree = new MerkleTree() 246 | await tree.init(['a', 'b', 'c']) 247 | 248 | expect(tree.leafs).toHaveLength(3) 249 | expect(tree.getRootHash()).toBeDefined() 250 | }) 251 | 252 | it('应该处理大量奇数个叶子节点', async () => { 253 | const tree = new MerkleTree() 254 | const hashList = Array.from({ length: 7 }, (_, i) => `hash${i}`) 255 | 256 | await tree.init(hashList) 257 | 258 | expect(tree.leafs).toHaveLength(7) 259 | expect(tree.getRootHash()).toBeDefined() 260 | }) 261 | }) 262 | }) 263 | -------------------------------------------------------------------------------- /packages/core/__tests__/node/browser/browserWorkerPool.spec.ts: -------------------------------------------------------------------------------- 1 | import { BrowserWorkerPool } from '../../../src/browser/browserWorkerPool' 2 | import { BrowserWorkerWrapper } from '../../../src/browser/browserWorkerWrapper' 3 | import { Strategy, WorkerStatusEnum } from '../../../src/types' 4 | 5 | // Mock Worker 6 | class MockWorker { 7 | onmessage: ((event: MessageEvent) => void) | null = null 8 | onerror: ((event: ErrorEvent) => void) | null = null 9 | postMessage = jest.fn() 10 | terminate = jest.fn() 11 | } 12 | 13 | // Mock URL.createObjectURL 14 | const mockCreateObjectURL = jest.fn() 15 | const mockRevokeObjectURL = jest.fn() 16 | 17 | // Mock import.meta.url 18 | const mockImportMetaUrl = 'https://test.com/browserWorkerPool.spec.ts' 19 | 20 | // 全局 Mock 21 | global.Worker = jest.fn().mockImplementation(() => new MockWorker()) as any 22 | global.URL.createObjectURL = mockCreateObjectURL 23 | global.URL.revokeObjectURL = mockRevokeObjectURL 24 | 25 | // Mock import.meta 26 | Object.defineProperty(global, 'import', { 27 | value: { 28 | meta: { 29 | url: mockImportMetaUrl, 30 | }, 31 | }, 32 | writable: true, 33 | }) 34 | 35 | describe('BrowserWorkerPool', () => { 36 | let workerPool: BrowserWorkerPool 37 | 38 | beforeEach(() => { 39 | jest.clearAllMocks() 40 | workerPool = new BrowserWorkerPool(2) 41 | }) 42 | 43 | afterEach(() => { 44 | workerPool.terminate() 45 | }) 46 | 47 | describe('构造函数', () => { 48 | it('应该正确初始化 worker 池', () => { 49 | expect(workerPool.maxWorkerCount).toBe(2) 50 | expect(workerPool.pool).toHaveLength(2) 51 | }) 52 | 53 | it('应该创建指定数量的 BrowserWorkerWrapper 实例', () => { 54 | workerPool.pool.forEach((worker: BrowserWorkerWrapper) => { 55 | expect(worker).toBeInstanceOf(BrowserWorkerWrapper) 56 | }) 57 | }) 58 | 59 | it('应该使用不同的 maxWorkerCount 初始化', () => { 60 | const poolWith3Workers = new BrowserWorkerPool(3) 61 | expect(poolWith3Workers.maxWorkerCount).toBe(3) 62 | expect(poolWith3Workers.pool).toHaveLength(3) 63 | poolWith3Workers.terminate() 64 | }) 65 | }) 66 | 67 | describe('createWorker', () => { 68 | it('应该创建 BrowserWorkerWrapper 实例', () => { 69 | const worker = workerPool.createWorker() 70 | expect(worker).toBeInstanceOf(BrowserWorkerWrapper) 71 | }) 72 | 73 | it('应该使用正确的 Worker 构造函数参数', () => { 74 | const createWorkerSpy = jest.spyOn(workerPool, 'createWorker') 75 | workerPool.createWorker() 76 | 77 | expect(createWorkerSpy).toHaveBeenCalledTimes(1) 78 | // 验证 Worker 被正确创建 79 | expect(global.Worker).toHaveBeenCalled() 80 | }) 81 | 82 | it('应该为每个 worker 创建独立的 Worker 实例', () => { 83 | const worker1 = workerPool.createWorker() 84 | const worker2 = workerPool.createWorker() 85 | 86 | expect(worker1).not.toBe(worker2) 87 | // 通过调用 terminate 方法来验证它们是不同的实例 88 | const terminateSpy1 = jest.spyOn(worker1, 'terminate') 89 | const terminateSpy2 = jest.spyOn(worker2, 'terminate') 90 | 91 | worker1.terminate() 92 | worker2.terminate() 93 | 94 | expect(terminateSpy1).toHaveBeenCalledTimes(1) 95 | expect(terminateSpy2).toHaveBeenCalledTimes(1) 96 | }) 97 | }) 98 | 99 | describe('继承的方法', () => { 100 | describe('exec', () => { 101 | it('应该能够执行任务', async () => { 102 | const mockData = new ArrayBuffer(8) 103 | const params = [ 104 | { chunk: mockData, strategy: Strategy.md5 }, 105 | { chunk: mockData, strategy: Strategy.md5 }, 106 | ] 107 | 108 | // Mock worker 响应 109 | const mockResults = ['hash1', 'hash2'] 110 | 111 | workerPool.pool.forEach((worker: BrowserWorkerWrapper) => { 112 | jest.spyOn(worker, 'run').mockImplementation((param, taskIndex) => { 113 | return Promise.resolve(mockResults[taskIndex as number] as any) 114 | }) 115 | }) 116 | 117 | const results = await workerPool.exec(params) 118 | 119 | expect(results).toHaveLength(2) 120 | }) 121 | 122 | it('应该处理任务数量超过 worker 数量的情况', async () => { 123 | const mockData = new ArrayBuffer(8) 124 | const params = [ 125 | { chunk: mockData, strategy: Strategy.md5 }, 126 | { chunk: mockData, strategy: Strategy.xxHash128 }, 127 | ] 128 | 129 | const mockResults = ['hash1', 'hash2'] 130 | let callIndex = 0 131 | 132 | workerPool.pool.forEach((worker: BrowserWorkerWrapper) => { 133 | jest.spyOn(worker, 'run').mockImplementation(() => { 134 | return Promise.resolve(mockResults[callIndex++] as any) 135 | }) 136 | }) 137 | 138 | const results = await workerPool.exec(params) 139 | 140 | expect(results).toHaveLength(2) 141 | }) 142 | 143 | it('应该处理 worker 执行失败的情况', async () => { 144 | const mockData = new ArrayBuffer(8) 145 | const params = [ 146 | { chunk: mockData, strategy: Strategy.md5 }, 147 | { chunk: mockData, strategy: Strategy.md5 }, 148 | ] 149 | 150 | const error = new Error('Worker execution failed') 151 | workerPool.pool.forEach((worker: BrowserWorkerWrapper) => { 152 | jest.spyOn(worker, 'run').mockRejectedValue(error) 153 | }) 154 | 155 | const results = await workerPool.exec(params) 156 | 157 | expect(results).toHaveLength(2) 158 | }) 159 | }) 160 | 161 | describe('adjustPool', () => { 162 | it('应该增加 worker 数量', () => { 163 | const initialCount = workerPool.pool.length 164 | workerPool.adjustPool(4) 165 | 166 | expect(workerPool.pool.length).toBe(4) 167 | expect(workerPool.pool.length).toBeGreaterThan(initialCount) 168 | }) 169 | 170 | it('应该减少 worker 数量', () => { 171 | // 确保所有 worker 都是等待状态，这样才能被移除 172 | workerPool.pool.forEach((worker: BrowserWorkerWrapper) => { 173 | worker.status = WorkerStatusEnum.WAITING 174 | }) 175 | 176 | workerPool.adjustPool(1) 177 | 178 | expect(workerPool.pool.length).toBe(1) 179 | expect(workerPool.maxWorkerCount).toBe(1) 180 | }) 181 | 182 | it('应该保持 worker 数量不变', () => { 183 | const initialCount = workerPool.pool.length 184 | workerPool.adjustPool(initialCount) 185 | 186 | expect(workerPool.pool.length).toBe(initialCount) 187 | }) 188 | 189 | it('应该只终止等待状态的 worker', () => { 190 | // 设置一些 worker 为运行状态 191 | workerPool.pool[0].status = WorkerStatusEnum.RUNNING 192 | workerPool.pool[1].status = WorkerStatusEnum.WAITING 193 | 194 | const terminateSpy = jest.spyOn(workerPool.pool[1], 'terminate') 195 | 196 | workerPool.adjustPool(1) 197 | 198 | // 应该减少到1个worker，只移除等待状态的worker 199 | expect(workerPool.pool.length).toBe(1) 200 | expect(workerPool.maxWorkerCount).toBe(1) 201 | expect(terminateSpy).toHaveBeenCalled() 202 | }) 203 | }) 204 | 205 | describe('terminate', () => { 206 | it('应该终止所有 worker', () => { 207 | const terminateSpies = workerPool.pool.map((worker: BrowserWorkerWrapper) => 208 | jest.spyOn(worker, 'terminate'), 209 | ) 210 | 211 | workerPool.terminate() 212 | 213 | terminateSpies.forEach((spy: jest.SpyInstance) => { 214 | expect(spy).toHaveBeenCalledTimes(1) 215 | }) 216 | }) 217 | 218 | it('应该清空 worker 池', () => { 219 | workerPool.terminate() 220 | expect(workerPool.pool).toHaveLength(0) 221 | }) 222 | }) 223 | }) 224 | 225 | describe('集成测试', () => { 226 | it('应该能够创建、调整和终止 worker 池', () => { 227 | // 创建 228 | expect(workerPool.pool).toHaveLength(2) 229 | 230 | // 调整 - 增加 worker 数量 231 | workerPool.adjustPool(3) 232 | expect(workerPool.pool).toHaveLength(3) 233 | 234 | // 调整 - 减少 worker 数量 235 | workerPool.pool.forEach((worker: BrowserWorkerWrapper) => { 236 | worker.status = WorkerStatusEnum.WAITING 237 | }) 238 | workerPool.adjustPool(1) 239 | expect(workerPool.pool).toHaveLength(1) 240 | 241 | // 终止 - 应该清空数组 242 | workerPool.terminate() 243 | expect(workerPool.pool).toHaveLength(0) 244 | }) 245 | 246 | it('应该正确处理并发任务执行', async () => { 247 | const mockData = new ArrayBuffer(8) 248 | const params = [ 249 | { chunk: mockData, strategy: Strategy.md5 }, 250 | { chunk: mockData, strategy: Strategy.xxHash128 }, 251 | ] 252 | 253 | const executionOrder: number[] = [] 254 | workerPool.pool.forEach((worker: BrowserWorkerWrapper, index: number) => { 255 | jest.spyOn(worker, 'run').mockImplementation(async (param, taskIndex) => { 256 | executionOrder.push(index) 257 | // 模拟不同的执行时间 258 | await new Promise((resolve) => setTimeout(resolve, Math.random() * 100)) 259 | return `result-${taskIndex}` as any 260 | }) 261 | }) 262 | 263 | const results = await workerPool.exec(params) 264 | 265 | expect(results).toHaveLength(2) 266 | 267 | // 验证 worker 被使用 268 | expect(executionOrder.length).toBeGreaterThan(0) 269 | }) 270 | }) 271 | }) 272 | -------------------------------------------------------------------------------- /packages/core/__tests__/node/shared/baseHashWorker.spec.ts: -------------------------------------------------------------------------------- 1 | import { BaseHashWorker } from '../../../src/shared/baseHashWorker' 2 | import { WorkerService } from '../../../src/shared/workerService' 3 | import { Config, FileMetaInfo, HashWorkerOptions, Strategy } from '../../../src/types' 4 | 5 | // 创建一个具体的测试实现类 6 | class TestHashWorker extends BaseHashWorker { 7 | protected normalizeParams(param: HashWorkerOptions): Required { 8 | const baseConfig = { 9 | chunkSize: 1, 10 | workerCount: 2, 11 | strategy: Strategy.md5, 12 | borderCount: 10, 13 | isCloseWorkerImmediately: false, 14 | isShowLog: false, 15 | } 16 | 17 | if ('file' in param) { 18 | return { 19 | config: { 20 | ...baseConfig, 21 | ...param.config, 22 | }, 23 | file: param.file!, 24 | filePath: undefined as never, 25 | } 26 | } else { 27 | return { 28 | config: { 29 | ...baseConfig, 30 | ...param.config, 31 | }, 32 | file: undefined as never, 33 | filePath: param.filePath!, 34 | } 35 | } 36 | } 37 | 38 | protected async processFile({ 39 | file, 40 | filePath, 41 | config, 42 | }: { 43 | file?: File 44 | filePath?: string 45 | config: Required 46 | }): Promise<{ chunksBlob?: Blob[]; chunksHash: string[]; fileHash: string }> { 47 | // 模拟处理文件，使用参数避免 linter 警告 48 | void file 49 | void filePath 50 | void config 51 | // 增加1ms延迟，避免overTime为0 52 | await new Promise((resolve) => setTimeout(resolve, 1)) 53 | const chunksHash = ['hash1', 'hash2', 'hash3'] 54 | const fileHash = 'merkleHash123' 55 | return { chunksHash, fileHash } 56 | } 57 | 58 | protected async getFileMetadata({ 59 | file, 60 | filePath, 61 | }: { 62 | file?: File 63 | filePath?: string 64 | }): Promise { 65 | // 模拟获取文件元数据，使用参数避免 linter 警告 66 | void file 67 | void filePath 68 | return { 69 | name: 'test.txt', 70 | size: 1024, 71 | lastModified: Date.now(), 72 | type: 'txt', 73 | } 74 | } 75 | 76 | protected createWorkerService(workerCount: number): WorkerService { 77 | // 使用参数避免 linter 警告 78 | void workerCount 79 | // 创建一个模拟的 BaseWorkerPool 80 | const mockPool = { 81 | exec: jest.fn().mockResolvedValue([ 82 | { success: true, data: 'hash1', index: 0 }, 83 | { success: true, data: 'hash2', index: 1 }, 84 | ]), 85 | adjustPool: jest.fn(), 86 | terminate: jest.fn(), 87 | } 88 | return new WorkerService(mockPool as any) 89 | } 90 | } 91 | 92 | describe('BaseHashWorker', () => { 93 | let worker: TestHashWorker 94 | let mockConsoleLog: jest.SpyInstance 95 | 96 | beforeEach(() => { 97 | worker = new TestHashWorker() 98 | mockConsoleLog = jest.spyOn(console, 'log').mockImplementation() 99 | }) 100 | 101 | afterEach(() => { 102 | worker.destroyWorkerPool() 103 | mockConsoleLog.mockRestore() 104 | }) 105 | 106 | describe('getFileHashChunks', () => { 107 | it('应该正确处理文件参数并返回结果', async () => { 108 | const mockFile = new File(['test content'], 'test.txt', { type: 'text/plain' }) 109 | const param: HashWorkerOptions = { 110 | file: mockFile, 111 | config: { 112 | workerCount: 2, 113 | isShowLog: false, 114 | }, 115 | } 116 | 117 | const result = await worker.getFileHashChunks(param) 118 | 119 | expect(result).toEqual({ 120 | chunksBlob: undefined, 121 | chunksHash: ['hash1', 'hash2', 'hash3'], 122 | merkleHash: 'merkleHash123', 123 | metadata: { 124 | name: 'test.txt', 125 | size: 1024, 126 | lastModified: expect.any(Number), 127 | type: 'txt', 128 | }, 129 | }) 130 | }) 131 | 132 | it('应该正确处理文件路径参数', async () => { 133 | const param: HashWorkerOptions = { 134 | filePath: '/path/to/file.txt', 135 | config: { 136 | workerCount: 3, 137 | isShowLog: false, 138 | }, 139 | } 140 | 141 | const result = await worker.getFileHashChunks(param) 142 | 143 | expect(result).toEqual({ 144 | chunksBlob: undefined, 145 | chunksHash: ['hash1', 'hash2', 'hash3'], 146 | merkleHash: 'merkleHash123', 147 | metadata: { 148 | name: 'test.txt', 149 | size: 1024, 150 | lastModified: expect.any(Number), 151 | type: 'txt', 152 | }, 153 | }) 154 | }) 155 | 156 | it('应该显示日志当 isShowLog 为 true', async () => { 157 | const mockFile = new File(['test content'], 'test.txt', { type: 'text/plain' }) 158 | const param: HashWorkerOptions = { 159 | file: mockFile, 160 | config: { 161 | workerCount: 2, 162 | isShowLog: true, 163 | }, 164 | } 165 | 166 | await worker.getFileHashChunks(param) 167 | 168 | expect(mockConsoleLog).toHaveBeenCalledWith( 169 | expect.stringMatching( 170 | /Generated file Merkle hash in \d+ms using 2 worker\(s\) with md5 strategy, processing speed: [\d.]+ MB\/s/, 171 | ), 172 | ) 173 | }) 174 | 175 | it('应该立即关闭 worker 当 isCloseWorkerImmediately 为 true', async () => { 176 | const mockFile = new File(['test content'], 'test.txt', { type: 'text/plain' }) 177 | const param: HashWorkerOptions = { 178 | file: mockFile, 179 | config: { 180 | workerCount: 2, 181 | isCloseWorkerImmediately: true, 182 | isShowLog: false, 183 | }, 184 | } 185 | 186 | await worker.getFileHashChunks(param) 187 | 188 | // 验证 workerService 被设置为 null 189 | expect((worker as any).workerService).toBeNull() 190 | expect((worker as any).curWorkerCount).toBe(0) 191 | }) 192 | 193 | it('应该重用现有的 workerService 当 workerCount 相同', async () => { 194 | const mockFile = new File(['test content'], 'test.txt', { type: 'text/plain' }) 195 | const param: HashWorkerOptions = { 196 | file: mockFile, 197 | config: { 198 | workerCount: 2, 199 | isShowLog: false, 200 | }, 201 | } 202 | 203 | // 第一次调用 204 | await worker.getFileHashChunks(param) 205 | const firstWorkerService = (worker as any).workerService 206 | 207 | // 第二次调用，workerCount 相同 208 | await worker.getFileHashChunks(param) 209 | const secondWorkerService = (worker as any).workerService 210 | 211 | // 应该重用同一个 workerService 212 | expect(secondWorkerService).toBe(firstWorkerService) 213 | }) 214 | 215 | it('应该调整 worker 池当 workerCount 改变', async () => { 216 | const mockFile = new File(['test content'], 'test.txt', { type: 'text/plain' }) 217 | const param1: HashWorkerOptions = { 218 | file: mockFile, 219 | config: { 220 | workerCount: 2, 221 | isShowLog: false, 222 | }, 223 | } 224 | 225 | // 第一次调用 226 | await worker.getFileHashChunks(param1) 227 | 228 | const param2: HashWorkerOptions = { 229 | file: mockFile, 230 | config: { 231 | workerCount: 4, 232 | isShowLog: false, 233 | }, 234 | } 235 | 236 | // 第二次调用，workerCount 不同 237 | await worker.getFileHashChunks(param2) 238 | 239 | expect((worker as any).curWorkerCount).toBe(4) 240 | }) 241 | 242 | it('应该使用默认配置当没有提供配置', async () => { 243 | const mockFile = new File(['test content'], 'test.txt', { type: 'text/plain' }) 244 | const param: HashWorkerOptions = { 245 | file: mockFile, 246 | } 247 | 248 | const result = await worker.getFileHashChunks(param) 249 | 250 | expect(result).toBeDefined() 251 | expect(result.chunksHash).toEqual(['hash1', 'hash2', 'hash3']) 252 | expect(result.merkleHash).toBe('merkleHash123') 253 | }) 254 | }) 255 | 256 | describe('destroyWorkerPool', () => { 257 | it('应该正确销毁 worker 池', async () => { 258 | const mockFile = new File(['test content'], 'test.txt', { type: 'text/plain' }) 259 | const param: HashWorkerOptions = { 260 | file: mockFile, 261 | config: { 262 | workerCount: 2, 263 | isShowLog: false, 264 | }, 265 | } 266 | 267 | // 先创建 worker 268 | await worker.getFileHashChunks(param) 269 | expect((worker as any).workerService).not.toBeNull() 270 | 271 | // 销毁 worker 272 | worker.destroyWorkerPool() 273 | expect((worker as any).workerService).toBeNull() 274 | expect((worker as any).curWorkerCount).toBe(0) 275 | }) 276 | 277 | it('应该安全地处理重复销毁', () => { 278 | // 当没有 worker 时，销毁应该不会报错 279 | expect(() => worker.destroyWorkerPool()).not.toThrow() 280 | expect((worker as any).workerService).toBeNull() 281 | expect((worker as any).curWorkerCount).toBe(0) 282 | }) 283 | }) 284 | 285 | describe('参数验证', () => { 286 | it('应该正确处理空的配置对象', async () => { 287 | const mockFile = new File(['test content'], 'test.txt', { type: 'text/plain' }) 288 | const param: HashWorkerOptions = { 289 | file: mockFile, 290 | config: {}, 291 | } 292 | 293 | const result = await worker.getFileHashChunks(param) 294 | 295 | expect(result).toBeDefined() 296 | expect(result.chunksHash).toEqual(['hash1', 'hash2', 'hash3']) 297 | }) 298 | 299 | it('应该正确处理部分配置', async () => { 300 | const mockFile = new File(['test content'], 'test.txt', { type: 'text/plain' }) 301 | const param: HashWorkerOptions = { 302 | file: mockFile, 303 | config: { 304 | chunkSize: 5, 305 | strategy: Strategy.md5, 306 | }, 307 | } 308 | 309 | const result = await worker.getFileHashChunks(param) 310 | 311 | expect(result).toBeDefined() 312 | expect(result.chunksHash).toEqual(['hash1', 'hash2', 'hash3']) 313 | }) 314 | }) 315 | }) 316 | --------------------------------------------------------------------------------