├── .changeset
    ├── README.md
    └── config.json
├── .gitattributes
├── .github
    └── workflows
    │   └── ci.yml
├── .gitignore
├── .husky
    ├── commit-msg
    └── pre-commit
├── .lintstagedrc
├── .prettierignore
├── .prettierrc
├── LICENSE
├── README-zh.md
├── README.md
├── codecov.yml
├── commitlint.config.js
├── eslint.config.mjs
├── package.json
├── packages
    ├── benchmark
    │   ├── CHANGELOG.md
    │   ├── README-zh.md
    │   ├── README.md
    │   ├── dts-bundle.config.json
    │   ├── package.json
    │   ├── src
    │   │   ├── benchmark.ts
    │   │   ├── helper.ts
    │   │   ├── index.ts
    │   │   └── types.ts
    │   ├── tsconfig.json
    │   └── tsup.config.ts
    ├── core
    │   ├── CHANGELOG.md
    │   ├── LICENSE
    │   ├── README-zh.md
    │   ├── README.md
    │   ├── __tests__
    │   │   ├── browser
    │   │   │   ├── fileUtils.spec.ts
    │   │   │   ├── getFileHashChunks.spec.ts
    │   │   │   ├── helper.spec.ts
    │   │   │   ├── helper2.spec.ts
    │   │   │   ├── workerPoolForHash.spec.ts
    │   │   │   └── workerWrapper.spec.ts
    │   │   ├── fixture
    │   │   │   ├── mockBlob.ts
    │   │   │   ├── mockFile.txt
    │   │   │   ├── mockMiniSubject.ts
    │   │   │   ├── mockWebWorker.ts
    │   │   │   ├── mockWorkerPool.ts
    │   │   │   └── mockWorkerWrapper.ts
    │   │   └── node
    │   │   │   ├── arrayUtils.spec.ts
    │   │   │   ├── getRootHashByChunks.spec.ts
    │   │   │   ├── is.spec.ts
    │   │   │   ├── merkleTree.spec.ts
    │   │   │   ├── miniSubject.spec.ts
    │   │   │   ├── workerPool.spec.ts
    │   │   │   └── workerWrapper.spec.ts
    │   ├── jest.config.ts
    │   ├── package.json
    │   ├── rollup.config.ts
    │   ├── src
    │   │   ├── entity
    │   │   │   ├── index.ts
    │   │   │   ├── merkleTree.ts
    │   │   │   ├── workerPool.ts
    │   │   │   └── workerWrapper.ts
    │   │   ├── getFileHashChunks.ts
    │   │   ├── getRootHashByChunks.ts
    │   │   ├── helper.ts
    │   │   ├── iife.ts
    │   │   ├── interface
    │   │   │   ├── fileHashChunks.ts
    │   │   │   ├── fileMetaInfo.ts
    │   │   │   ├── fnTypes.ts
    │   │   │   ├── index.ts
    │   │   │   ├── strategy.ts
    │   │   │   ├── workerReq.ts
    │   │   │   └── workerRes.ts
    │   │   ├── main.ts
    │   │   ├── utils
    │   │   │   ├── arrayUtils.ts
    │   │   │   ├── fileUtils.ts
    │   │   │   ├── index.ts
    │   │   │   ├── is.ts
    │   │   │   ├── miniSubject.ts
    │   │   │   └── rand.ts
    │   │   └── worker
    │   │   │   ├── hash.worker.ts
    │   │   │   ├── workerPoolForHash.ts
    │   │   │   └── workerService.ts
    │   └── tsconfig.json
    └── playground
    │   ├── benchmark-demo
    │       ├── package.json
    │       ├── src
    │       │   └── index.ts
    │       ├── tsconfig.json
    │       └── tsup.config.ts
    │   ├── iife-demo
    │       ├── index.html
    │       ├── package.json
    │       ├── prepare.js
    │       └── server.js
    │   ├── node-demo
    │       ├── package.json
    │       ├── src
    │       │   └── index.ts
    │       ├── tsconfig.json
    │       └── tsup.config.ts
    │   ├── react-webpack-demo
    │       ├── .babelrc
    │       ├── package.json
    │       ├── public
    │       │   └── index.html
    │       ├── src
    │       │   └── index.tsx
    │       ├── tsconfig.json
    │       └── webpack.config.js
    │   └── vue-vite-demo
    │       ├── index.html
    │       ├── package.json
    │       ├── src
    │           ├── App.vue
    │           ├── hooks
    │           │   └── useFileHashInfo.ts
    │           └── main.ts
    │       ├── tsconfig.json
    │       └── vite.config.ts
├── pnpm-lock.yaml
├── pnpm-workspace.yaml
├── scripts
    ├── clear.js
    ├── fileCopier.js
    └── syncReadme.js
├── tsconfig.json
└── turbo.json


/.changeset/README.md:
--------------------------------------------------------------------------------
1 | # Changesets
2 | 
3 | Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works
4 | with multi-package repos, or single-package repos to help you version and publish your code. You can
5 | find the full documentation for it [in our repository](https://github.com/changesets/changesets)
6 | 
7 | We have a quick list of common questions to get you started engaging with this project in
8 | [our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md)
9 | 


--------------------------------------------------------------------------------
/.changeset/config.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "$schema": "https://unpkg.com/@changesets/config@3.0.5/schema.json",
 3 |   "changelog": "@changesets/cli/changelog",
 4 |   "commit": false,
 5 |   "fixed": [],
 6 |   "linked": [],
 7 |   "access": "restricted",
 8 |   "baseBranch": "main",
 9 |   "updateInternalDependencies": "patch",
10 |   "ignore": []
11 | }
12 | 


--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
1 | # Auto detect text files and perform LF normalization
2 | * text=auto
3 | 


--------------------------------------------------------------------------------
/.github/workflows/ci.yml:
--------------------------------------------------------------------------------
 1 | name: Workflow for Codecov
 2 | on: [push]
 3 | jobs:
 4 |   run:
 5 |     runs-on: ubuntu-latest
 6 |     steps:
 7 |       - uses: actions/checkout@v4
 8 |       - name: Install berry
 9 |         run: corepack enable
10 |       - uses: actions/setup-node@v3
11 |         with:
12 |           node-version: 22
13 |       - name: Install Dependencies
14 |         run: pnpm install --no-frozen-lockfile
15 |       - name: Run Test
16 |         run: pnpm test
17 |       - name: Upload coverage reports to Codecov
18 |         uses: codecov/codecov-action@v4.0.1
19 |         with:
20 |           token: ${{ secrets.CODECOV_TOKEN }}
21 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # See http://help.github.com/ignore-files/ for more about ignoring files.
 2 | 
 3 | # Compiled output, 排除打包产物
 4 | dist
 5 | /output
 6 | /tmp
 7 | /out-tsc
 8 | /bazel-out
 9 | 
10 | # 排除子项目的打包产物
11 | **/output
12 | **/.turbo
13 | 
14 | # Node, 忽略任意层级下的 node_modules 目录
15 | **/node_modules/
16 | npm-debug.log
17 | yarn-error.log
18 | 
19 | # 忽略覆盖率
20 | **/coverage/
21 | 
22 | # IDEs and editors
23 | .idea/
24 | .project
25 | .classpath
26 | .c9/
27 | *.launch
28 | .settings/
29 | *.sublime-workspace
30 | 
31 | # Visual Studio Code
32 | .vscode/*
33 | !.vscode/settings.json
34 | !.vscode/tasks.json
35 | !.vscode/launch.json
36 | !.vscode/extensions.json
37 | .history/*
38 | 
39 | # Miscellaneous
40 | /.angular/cache
41 | .sass-cache/
42 | /connect.lock
43 | /libpeerconnection.log
44 | testem.log
45 | /typings
46 | 
47 | # System files
48 | .DS_Store
49 | Thumbs.db
50 | 
51 | # browser-demo
52 | /packages/playground/iife-demo/global.js
53 | /packages/playground/iife-demo/worker/*
54 | 


--------------------------------------------------------------------------------
/.husky/commit-msg:
--------------------------------------------------------------------------------
1 | npx --no-install commitlint --edit "$1"


--------------------------------------------------------------------------------
/.husky/pre-commit:
--------------------------------------------------------------------------------
1 | npx lint-staged
2 | 


--------------------------------------------------------------------------------
/.lintstagedrc:
--------------------------------------------------------------------------------
1 | {
2 |   "*.{js,jsx,ts,tsx}": ["eslint --fix", "prettier --write"]
3 | }
4 | 


--------------------------------------------------------------------------------
/.prettierignore:
--------------------------------------------------------------------------------
1 | /dist/
2 | /node_modules/
3 | /coverage/
4 | /.idea/
5 | /.husky/
6 | /.github/
7 | /.changeset/
8 | 


--------------------------------------------------------------------------------
/.prettierrc:
--------------------------------------------------------------------------------
 1 | {
 2 |   "printWidth": 100,
 3 |   "semi": false,
 4 |   "singleQuote": true,
 5 |   "trailingComma": "all",
 6 |   "proseWrap": "never",
 7 |   "htmlWhitespaceSensitivity": "strict",
 8 |   "endOfLine": "auto"
 9 | }
10 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2024 Tkunl
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README-zh.md:
--------------------------------------------------------------------------------
  1 | # Hash Worker [![npm package](https://img.shields.io/npm/v/hash-worker.svg)](https://www.npmjs.com/package/hash-worker) [![Bundle size](https://badgen.net/bundlephobia/minzip/hash-worker)](https://bundlephobia.com/result?p=hash-worker)  [![codecov](https://codecov.io/gh/Tkunl/hash-worker/graph/badge.svg?token=G7GYAPEPYS)](https://codecov.io/gh/Tkunl/hash-worker) ![GitHub License](https://img.shields.io/github/license/Tkunl/hash-worker)
  2 | 
  3 | <p align="center">
  4 | <img src="https://socialify.git.ci/Tkunl/hash-worker/image?font=Inter&language=1&name=1&owner=1&pattern=Plus&theme=Auto" width="640" height="320" />
  5 | </p>
  6 | 
  7 | ## Introduce
  8 | 
  9 | [English Document](./README.md)
 10 | 
 11 | **Hash-worker** 是一个用于快速计算文件哈希值的库。
 12 | 
 13 | 它基于 hash-wasm 且利用了 WebWorker 进行并行计算，从而加快了计算文件分片的计算速度。
 14 | 
 15 | Hash-worker 支持三种哈希计算算法：`md5`, `crc32` 和 `xxHash64`。
 16 | 
 17 | 同时支持 `浏览器` 和 `Node.js` 环境。
 18 | 
 19 | > [!WARNING]
 20 | > Hash-worker 计算出的 MerkleHash 是基于文件块哈希值构建的 MerkleTree 的根哈希值。请注意，这并不直接等同于文件本身的哈希值。
 21 | 
 22 | ## Install
 23 | 
 24 | ```bash
 25 | $ pnpm install hash-worker
 26 | ```
 27 | 
 28 | ## Usage
 29 | 
 30 | ### Global
 31 | 
 32 | ```html
 33 | 
 34 | <script src="./global.js"></script>
 35 | <script src="./worker/hash.worker.mjs"></script>
 36 | <script>
 37 |   HashWorker.getFileHashChunks()
 38 | </script>
 39 | ```
 40 | 
 41 | 其中 `global.js` 和 `hash.worker.mjs` 是执行 `package.json` 中的 `build:core` 后的打包产物
 42 | 
 43 | 打包产物位于 `packages/core/dist` 目录
 44 | 
 45 | ### ESM
 46 | 
 47 | ``` ts
 48 | import { getFileHashChunks, destroyWorkerPool, HashChksRes, HashChksParam } from 'hash-worker'
 49 | 
 50 | function handleGetHash(file: File) {
 51 |   const param: HashChksParam = {
 52 |     file: file,
 53 |     config: {
 54 |       workerCount: 8,
 55 |       strategy: Strategy.md5
 56 |     }
 57 |   }
 58 | 
 59 |   getFileHashChunks(param).then((data: HashChksRes) => {
 60 |     console.log('chunksHash', data.chunksHash)
 61 |   })
 62 | }
 63 | 
 64 | /**
 65 |  * Destroy Worker Thread
 66 |  */
 67 | function handleDestroyWorkerPool() {
 68 |   destroyWorkerPool()
 69 | }
 70 | ```
 71 | 
 72 | > [!WARNING]
 73 | > 如果你在使用 `Vite` 作为构建工具, 需要在 `Vite` 的配置文件中, 添加如下配置, 用于排除 vite 的依赖优化
 74 | 
 75 |  ```js
 76 | // vite.config.js
 77 | import { defineConfig } from 'vite'
 78 | import vue from '@vitejs/plugin-vue'
 79 | 
 80 | export default defineConfig({
 81 |   plugins: [vue()],
 82 |   // other configurations ...
 83 |   optimizeDeps: {
 84 |     exclude: ['hash-worker'] // new added..
 85 |   }
 86 | })
 87 |  ```
 88 | 
 89 | > [!WARNING]
 90 | > 如果你在使用 `Webpack` 作为构建工具, 需要在 Webpack 的配置文件中, 添加如下配置, 用于排除 node 相关模块的解析
 91 | 
 92 | ```js
 93 | // webpack.config.js
 94 | module.exports = {
 95 |   // new added..
 96 |   resolve: { 
 97 |     fallback: {
 98 |       fs: false,
 99 |       path: false,
100 |       'fs/promises': false,
101 |       worker_threads: false,
102 |     },
103 |   },
104 |   // new added..
105 |   externals: {
106 |     fs: 'commonjs fs',
107 |     path: 'commonjs path',
108 |     'fs/promises': 'commonjs fs/promises',
109 |     worker_threads: 'commonjs worker_threads',
110 |   },
111 | }
112 | ```
113 | 
114 | ## Options
115 | 
116 | **HashChksParam**
117 | 
118 | HashChksParam 是用于配置计算哈希值所需的参数。
119 | 
120 | | filed    | type   | default | description                 |
121 | |----------|--------|---------|-----------------------------|
122 | | file     | File   | /       | 需要计算 Hash 的文件（浏览器环境下必填）     |
123 | | filePath | string | /       | 需要计算 Hash 的文件路径 （Node环境下必填） |
124 | | config   | Config | Config  | 计算 Hash 时的参数                |
125 | 
126 | **Config**
127 | 
128 | | filed                    | type     | default        | description               |
129 | |--------------------------|----------|----------------|---------------------------|
130 | | chunkSize                | number   | 10 (MB)        | 文件分片的大小                   |
131 | | workerCount              | number   | 8              | 计算 Hash 时同时开启的 worker 数量  |
132 | | strategy                 | Strategy | Strategy.mixed | hash 计算策略                 |
133 | | borderCount              | number   | 100            | 'mixed' 模式下 hash 计算规则的分界点 |
134 | | isCloseWorkerImmediately | boolean  | true           | 当计算完成时, 是否立即销毁 Worker 线程  |
135 | 
136 | ```ts
137 | // strategy.ts
138 | export enum Strategy {
139 |   md5 = 'md5',
140 |   crc32 = 'crc32',
141 |   xxHash64 = 'xxHash64',
142 |   mixed = 'mixed',
143 | }
144 | ```
145 | 
146 | 当采用 Strategy.mixed 策略时，若文件分片数量少于 borderCount，将采用 md5 算法计算哈希值来构建 MerkleTree。
147 | 否则，则切换至使用 crc32 算法进行 MerkleTree 的构建。
148 | 
149 | **HashChksRes**
150 | 
151 | HashChksRes 是计算哈希值之后的返回结果。
152 | 
153 | | filed      | type         | description              |
154 | |------------|--------------|--------------------------|
155 | | chunksBlob | Blob[]       | 仅在浏览器环境下，会返回文件分片的 Blob[] |
156 | | chunksHash | string[]     | 文件分片的 Hash[]             |
157 | | merkleHash | string       | 文件的 merkleHash           |
158 | | metadata   | FileMetaInfo | 文件的 metadata             |
159 | 
160 | **FileMetaInfo**
161 | 
162 | | filed        | type   | description    |
163 | |--------------|--------|----------------|
164 | | name         | string | 用于计算 hash 的文件名 |
165 | | size         | number | 文件大小，单位：KB     |
166 | | lastModified | number | 文件最后一次修改的时间戳   |
167 | | type         | string | 文件后缀名          |
168 | 
169 | ### [Benchmark (MD5)](./packages/benchmark/README-zh.md)
170 | 
171 | | Worker Count | Speed     |
172 | |--------------|-----------|
173 | | 1            | 229 MB/s  |
174 | | 4            | 632 MB/s  |
175 | | 8            | 886 MB/s  |
176 | | 12           | 1037 MB/s |
177 | 
178 | * 以上数据是运行在 `Chrome v131` 和 `AMD Ryzen9 5950X` CPU 下, 通过使用 md5 来计算 hash 得到的。
179 | 
180 | ## LICENSE
181 | 
182 | [MIT](./LICENSE)
183 | 
184 | ## Contributions
185 | 
186 | 欢迎贡献代码！如果你发现了一个 bug 或者想添加一个新功能，请提交一个 issue 或 pull request。
187 | 
188 | ## Author and contributors
189 | 
190 | <p align="center">
191 |   <a href="https://github.com/Tkunl">
192 |     <img src="https://avatars.githubusercontent.com/u/19854081?v=4" width="40" height="40" alt="Tkunl">
193 |   </a>
194 |   <a href="https://github.com/nonzzz">
195 |     <img src="https://avatars.githubusercontent.com/u/52351095?v=4&s=40" width="40" height="40" alt="Kanno">
196 |   </a>
197 |   <a href="https://github.com/Eternal-could">
198 |     <img src="https://avatars.githubusercontent.com/u/74654896?v=4" width="40" height="40" alt="Eternal-could">
199 |   </a>
200 | </p>
201 | 
202 | 
203 | 
204 | 
205 | 
206 | 
207 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Hash Worker [![npm package](https://img.shields.io/npm/v/hash-worker.svg)](https://www.npmjs.com/package/hash-worker) [![Bundle size](https://badgen.net/bundlephobia/minzip/hash-worker)](https://bundlephobia.com/result?p=hash-worker)  [![codecov](https://codecov.io/gh/Tkunl/hash-worker/graph/badge.svg?token=G7GYAPEPYS)](https://codecov.io/gh/Tkunl/hash-worker) ![GitHub License](https://img.shields.io/github/license/Tkunl/hash-worker)
  2 | 
  3 | <p align="center">
  4 | <img src="https://socialify.git.ci/Tkunl/hash-worker/image?font=Inter&language=1&name=1&owner=1&pattern=Plus&theme=Auto" width="640" height="320" />
  5 | </p>
  6 | 
  7 | ## Introduce
  8 | 
  9 | [中文文档](./README-zh.md)
 10 | 
 11 | **Hash-worker** is a library for fast calculation of file chunk hashes.
 12 | 
 13 | It is based on `hash-wasm` and utilizes `WebWorkers` for parallel computation, which speeds up computation when
 14 | processing file blocks.
 15 | 
 16 | Hash-worker supports three hash computation algorithms: `md5`, `crc32` and `xxHash64`.
 17 | 
 18 | Both `browser` and `Node.js` are supported.
 19 | 
 20 | > [!WARNING]
 21 | > The merkleHash computed by the Hash-worker is the root hash of a MerkleTree constructed based on file block hashes. Note that this is not directly equivalent to a hash of the file itself.
 22 | 
 23 | ## Install
 24 | 
 25 | ```bash
 26 | $ pnpm install hash-worker
 27 | ```
 28 | 
 29 | ## Usage
 30 | 
 31 | ### Global
 32 | 
 33 | ```html
 34 | 
 35 | <script src="./global.js"></script>
 36 | <script src="./worker/hash.worker.mjs"></script>
 37 | <script>
 38 |   HashWorker.getFileHashChunks()
 39 | </script>
 40 | ```
 41 | 
 42 | The `global.js` and `hash.worker.mjs` are the build artifacts resulting from executing `build:core` in `package.json`.
 43 | 
 44 | The build artifacts are located in the `packages/core/dist` directory.
 45 | 
 46 | ### ESM
 47 | 
 48 | ``` ts
 49 | import { getFileHashChunks, destroyWorkerPool, HashChksRes, HashChksParam } from 'hash-worker'
 50 | 
 51 | function handleGetHash(file: File) {
 52 |   const param: HashChksParam = {
 53 |     file: file,
 54 |     config: {
 55 |       workerCount: 8,
 56 |       strategy: Strategy.md5
 57 |     }
 58 |   }
 59 | 
 60 |   getFileHashChunks(param).then((data: HashChksRes) => {
 61 |     console.log('chunksHash', data.chunksHash)
 62 |   })
 63 | }
 64 | 
 65 | /**
 66 |  * Destroy Worker Thread
 67 |  */
 68 | function handleDestroyWorkerPool() {
 69 |   destroyWorkerPool()
 70 | }
 71 | ```
 72 | 
 73 | > [!WARNING]
 74 | If you are using `Vite` as your build tool, you need to add some configurations in your `vite.config.js` to exclude hash-worker from optimizeDeps.
 75 | 
 76 | ```js
 77 | // vite.config.js
 78 | import { defineConfig } from 'vite'
 79 | import vue from '@vitejs/plugin-vue'
 80 | 
 81 | export default defineConfig({
 82 |   plugins: [vue()],
 83 |   // other configurations ...
 84 |   optimizeDeps: {
 85 |     exclude: ['hash-worker'] // new added..
 86 |   }
 87 | })
 88 | ```
 89 | 
 90 | > [!WARNING]
 91 | >
 92 | > If you are using `Webpack` as your build tool, you need add some configs in your `webpack.config.js` for exclude the parsing of node related modules.
 93 | 
 94 | ```js
 95 | // webpack.config.js
 96 | module.exports = {
 97 |   // new added..
 98 |   resolve: { 
 99 |     fallback: {
100 |       fs: false,
101 |       path: false,
102 |       'fs/promises': false,
103 |       worker_threads: false,
104 |     },
105 |   },
106 |   // new added..
107 |   externals: {
108 |     fs: 'commonjs fs',
109 |     path: 'commonjs path',
110 |     'fs/promises': 'commonjs fs/promises',
111 |     worker_threads: 'commonjs worker_threads',
112 |   },
113 | }
114 | ```
115 | 
116 | ## Options
117 | 
118 | **HashChksParam**
119 | 
120 | HashChksParam is used to configure the parameters needed to calculate the hash.
121 | 
122 | | filed    | type   | default | description                                                                          |
123 | |----------|--------|---------|--------------------------------------------------------------------------------------|
124 | | file     | File   | /       | Files that need to calculate the hash (required for browser environments)            |
125 | | filePath | string | /       | Path to the file where the hash is to be calculated (required for Node environments) |
126 | | config   | Config | Config  | Parameters for calculating the Hash                                                  |
127 | 
128 | **Config**
129 | 
130 | | filed                    | type     | default        | description                                                                       |
131 | |--------------------------|----------|----------------|-----------------------------------------------------------------------------------|
132 | | chunkSize                | number   | 10 (MB)        | Size of the file slice                                                            |
133 | | workerCount              | number   | 8              | Number of workers turned on at the same time as the hash is calculated            |
134 | | strategy                 | Strategy | Strategy.mixed | Hash computation strategy                                                         |
135 | | borderCount              | number   | 100            | The cutoff for the hash calculation rule in 'mixed' mode                          |
136 | | isCloseWorkerImmediately | boolean  | true           | Whether to destroy the worker thread immediately when the calculation is complete |
137 | 
138 | ```ts
139 | // strategy.ts
140 | export enum Strategy {
141 |   md5 = 'md5',
142 |   crc32 = 'crc32',
143 |   xxHash64 = 'xxHash64',
144 |   mixed = 'mixed',
145 | }
146 | ```
147 | 
148 | When Strategy.mixed strategy is used, if the number of file fragments is less than borderCount, the md5 algorithm will
149 | be used to calculate the hash value to build the MerkleTree.
150 | Otherwise, it switches to using the crc32 algorithm for MerkleTree construction.
151 | 
152 | **HashChksRes**
153 | 
154 | HashChksRes is the returned result after calculating the hash value.
155 | 
156 | | filed      | type         | description                                                             |
157 | |------------|--------------|-------------------------------------------------------------------------|
158 | | chunksBlob | Blob[]       | In a browser environment only, the Blob[] of the file slice is returned |
159 | | chunksHash | string[]     | Hash[] for file slicing                                                 |
160 | | merkleHash | string       | The merkleHash of the file                                              |
161 | | metadata   | FileMetaInfo | The metadata of the file                                                |
162 | 
163 | **FileMetaInfo**
164 | 
165 | | filed        | type   | description                                     |
166 | |--------------|--------|-------------------------------------------------|
167 | | name         | string | The name of the file used to calculate the hash |
168 | | size         | number | File size in KB                                 |
169 | | lastModified | number | Timestamp of the last modification of the file  |
170 | | type         | string | file extension                                  |
171 | 
172 | ### [Benchmark (MD5)](./packages/benchmark/README.md)
173 | 
174 | | Worker Count | Speed     |
175 | |--------------|-----------|
176 | | 1            | 229 MB/s  |
177 | | 4            | 632 MB/s  |
178 | | 8            | 886 MB/s  |
179 | | 12           | 1037 MB/s |
180 | 
181 | The above data is run on the `Chrome v131` and `AMD Ryzen9 5950X` CPU, by using md5 to calculate hash.
182 | 
183 | ## LICENSE
184 | 
185 | [MIT](./LICENSE)
186 | 
187 | ## Contributions
188 | 
189 | Contributions are welcome! If you find a bug or want to add a new feature, please open an issue or submit a pull
190 | request.
191 | 
192 | ## Author and contributors
193 | 
194 | <p align="center">
195 |   <a href="https://github.com/Tkunl">
196 |     <img src="https://avatars.githubusercontent.com/u/19854081?v=4" width="40" height="40" alt="Tkunl">
197 |   </a>
198 |   <a href="https://github.com/Eternal-could">
199 |     <img src="https://avatars.githubusercontent.com/u/74654896?v=4" width="40" height="40" alt="Eternal-could">
200 |   </a>
201 |   <a href="https://github.com/nonzzz">
202 |     <img src="https://avatars.githubusercontent.com/u/52351095?v=4&s=40" width="40" height="40" alt="Kanno">
203 |   </a>
204 | </p>
205 | 


--------------------------------------------------------------------------------
/codecov.yml:
--------------------------------------------------------------------------------
 1 | codecov:
 2 |   require_ci_to_pass: true
 3 | 
 4 | coverage:
 5 |   status:
 6 |     project:
 7 |       default:
 8 |         target: 90%
 9 |         threshold: 2%
10 |         base: auto
11 |         if_ci_failed: error
12 |         branches:
13 |           - 'master'
14 | 
15 | comment:
16 |   layout: 'reach, diff, flags, files'
17 |   behavior: default
18 |   require_changes: false
19 |   branches:
20 |     - 'master'
21 | 


--------------------------------------------------------------------------------
/commitlint.config.js:
--------------------------------------------------------------------------------
 1 | module.exports = {
 2 |   extends: ['@commitlint/config-conventional'],
 3 |   rules: {
 4 |     'type-enum': [
 5 |       // type枚举
 6 |       2,
 7 |       'always',
 8 |       [
 9 |         'build', // 编译相关的修改，例如发布版本、对项目构建或者依赖的改动
10 |         'feat', // 新功能
11 |         'fix', // 修补bug
12 |         'docs', // 文档修改
13 |         'style', // 代码格式修改, 注意不是 css 修改
14 |         'refactor', // 重构
15 |         'perf', // 优化相关，比如提升性能、体验
16 |         'test', // 测试用例修改
17 |         'revert', // 代码回滚
18 |         'ci', // 持续集成修改
19 |         'config', // 配置修改
20 |         'chore', // 其他改动
21 |       ],
22 |     ],
23 |     'type-empty': [2, 'never'], // never: type 不能为空; always: type 必须为空
24 |     'type-case': [0, 'always', 'lower-case'], // type 必须小写，upper-case大写，camel-case小驼峰，kebab-case短横线，pascal-case大驼峰，等等
25 |     'scope-empty': [0],
26 |     'scope-case': [0],
27 |     'subject-empty': [2, 'never'], // subject不能为空
28 |     'subject-case': [0],
29 |     'subject-full-stop': [0, 'never', '.'], // subject 以.为结束标记
30 |     'header-max-length': [2, 'always', 72], // header 最长72
31 |     'body-leading-blank': [0], // body 换行
32 |     'footer-leading-blank': [0, 'always'], // footer 以空行开头
33 |   },
34 | }
35 | 


--------------------------------------------------------------------------------
/eslint.config.mjs:
--------------------------------------------------------------------------------
 1 | import typescriptEslint from '@typescript-eslint/eslint-plugin'
 2 | import prettier from 'eslint-plugin-prettier'
 3 | import globals from 'globals'
 4 | import tsParser from '@typescript-eslint/parser'
 5 | import path from 'node:path'
 6 | import { fileURLToPath } from 'node:url'
 7 | import js from '@eslint/js'
 8 | import { FlatCompat } from '@eslint/eslintrc'
 9 | 
10 | const __filename = fileURLToPath(import.meta.url)
11 | const __dirname = path.dirname(__filename)
12 | const compat = new FlatCompat({
13 |   baseDirectory: __dirname,
14 |   recommendedConfig: js.configs.recommended,
15 |   allConfig: js.configs.all,
16 | })
17 | 
18 | export default [
19 |   {
20 |     ignores: ['**/node_modules/', '**/output/', '**/dist/', '**/playground/'],
21 |   },
22 |   ...compat.extends(
23 |     'eslint:recommended',
24 |     'plugin:@typescript-eslint/recommended',
25 |     'plugin:prettier/recommended',
26 |   ),
27 |   {
28 |     plugins: {
29 |       '@typescript-eslint': typescriptEslint,
30 |       prettier,
31 |     },
32 | 
33 |     languageOptions: {
34 |       globals: {
35 |         ...globals.browser,
36 |         ...globals.node,
37 |       },
38 | 
39 |       parser: tsParser,
40 |       ecmaVersion: 2022,
41 |       sourceType: 'module',
42 |     },
43 | 
44 |     rules: {
45 |       'prettier/prettier': 'error',
46 |       '@typescript-eslint/no-unused-vars': 'error',
47 |       '@typescript-eslint/no-explicit-any': 'off',
48 |       '@typescript-eslint/ban-ts-comment': 'off',
49 |       '@typescript-eslint/no-require-imports': 'off',
50 | 
51 |       '@typescript-eslint/no-unused-expressions': [
52 |         'error',
53 |         {
54 |           allowShortCircuit: true,
55 |           allowTernary: true,
56 |           allowTaggedTemplates: true,
57 |         },
58 |       ],
59 |     },
60 |   },
61 | ]
62 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "private": true,
 3 |   "packageManager": "pnpm@9.15.2",
 4 |   "scripts": {
 5 |     "dev:core": "pnpm --filter=./packages/core run dev",
 6 |     "dev:benchmark": "pnpm --filter=./packages/benchmark run dev",
 7 |     "build:core": "turbo run build",
 8 |     "build:benchmark": "turbo run build:benchmark",
 9 |     "build:node-demo": "turbo run build:node-demo",
10 |     "build:benchmark-demo": "turbo run build:benchmark-demo",
11 |     "build:all": "turbo run build:all",
12 |     "play-benchmark": "pnpm run build:benchmark-demo && pnpm --filter=./packages/playground/benchmark-demo run play",
13 |     "play-node": "pnpm run build:node-demo && pnpm --filter=./packages/playground/node-demo run play",
14 |     "play-iife": "pnpm run build:core && pnpm --filter=./packages/playground/iife-demo run play",
15 |     "play-vue-vite": "pnpm run build:core && pnpm --filter=./packages/playground/vue-vite-demo run play",
16 |     "play-react-webpack": "pnpm run build:core && pnpm --filter=./packages/playground/react-webpack-demo run play",
17 |     "test": "pnpm --filter=./packages/core run test",
18 |     "lint": "eslint --fix",
19 |     "format": "prettier --write '**/*.{js,jsx,ts,tsx,json}'",
20 |     "check-format": "prettier --check '**/*.{js,jsx,ts,tsx,json}'",
21 |     "prepare": "husky",
22 |     "pre-commit": "lint-staged",
23 |     "commitlint": "commitlint --config commitlint.config.js -e -V",
24 |     "sync-readme": "node scripts/syncReadme.js",
25 |     "clear:node_modules": "node scripts/clear.js --pattern=node_modules",
26 |     "clear:dist": "node scripts/clear.js --pattern=dist",
27 |     "clear:cache": "node scripts/clear.js --pattern=cache",
28 |     "clear:coverage": "node scripts/clear.js --pattern=coverage",
29 |     "clear:all": "node scripts/clear.js --pattern=all"
30 |   },
31 |   "keywords": [
32 |     "hash-worker",
33 |     "hash"
34 |   ],
35 |   "author": "Tkunl",
36 |   "license": "MIT",
37 |   "devDependencies": {
38 |     "@changesets/cli": "^2.27.11",
39 |     "@commitlint/cli": "^19.6.1",
40 |     "@commitlint/config-conventional": "^19.6.0",
41 |     "@eslint/eslintrc": "^3.2.0",
42 |     "@eslint/js": "^9.18.0",
43 |     "@jest/types": "^29.6.3",
44 |     "@rollup/plugin-node-resolve": "^15.3.1",
45 |     "@swc/core": "^1.10.7",
46 |     "@types/jest": "^29.5.12",
47 |     "@types/node": "^20.14.2",
48 |     "@typescript-eslint/eslint-plugin": "^8.19.1",
49 |     "@typescript-eslint/parser": "^8.19.1",
50 |     "browserslist": "^4.23.0",
51 |     "chalk": "^5.4.1",
52 |     "dts-bundle-generator": "^9.5.1",
53 |     "eslint": "^9.18.0",
54 |     "eslint-config-prettier": "^9.1.0",
55 |     "eslint-plugin-prettier": "^5.2.1",
56 |     "globals": "^15.14.0",
57 |     "husky": "^9.1.7",
58 |     "jest": "^29.7.0",
59 |     "jest-environment-jsdom": "^29.7.0",
60 |     "lint-staged": "^15.3.0",
61 |     "prettier": "^3.4.2",
62 |     "rimraf": "^6.0.1",
63 |     "rollup": "^4.30.1",
64 |     "rollup-plugin-dts": "^6.1.0",
65 |     "rollup-plugin-swc3": "^0.12.1",
66 |     "ts-jest": "^29.1.5",
67 |     "ts-jest-mock-import-meta": "^1.2.0",
68 |     "tsup": "^8.3.5",
69 |     "tsx": "^4.11.0",
70 |     "turbo": "^2.3.3",
71 |     "typescript": "^5.4.5"
72 |   }
73 | }
74 | 


--------------------------------------------------------------------------------
/packages/benchmark/CHANGELOG.md:
--------------------------------------------------------------------------------
 1 | # hash-worker-benchmark
 2 | 
 3 | ## 1.0.0
 4 | 
 5 | ### Major Changes
 6 | 
 7 | - release version 1.0.0
 8 | 
 9 | ### Patch Changes
10 | 
11 | - Updated dependencies
12 |   - hash-worker@1.0.0
13 | 
14 | ## 0.0.1
15 | 
16 | ### Patch Changes
17 | 
18 | - Updated dependencies
19 |   - hash-worker@0.1.3
20 | 


--------------------------------------------------------------------------------
/packages/benchmark/README-zh.md:
--------------------------------------------------------------------------------
 1 | ## Introduce for benchmark
 2 | 
 3 | 该项目用于测试 Hash worker 在不同线程下的哈希计算速度
 4 | 
 5 | 它同时支持 `浏览器` 环境和 `Node.js` 环境
 6 | 
 7 | ### Usage
 8 | 
 9 | ```ts
10 | import { benchmark, BenchmarkOptions } from 'hash-worker-benchmark'
11 | 
12 | // options 是可选的
13 | const options: BenchmarkOptions = {}
14 | benchmark(options)
15 | ```
16 | 
17 | ### Options
18 | 
19 | **BenchmarkOptions**
20 | 
21 | | filed               | type     | default                                 | description         |
22 | | ------------------- | -------- | --------------------------------------- |---------------------|
23 | | sizeInMB            | number   | 500                                     | 用于测试的文件大小 (MB)      |
24 | | strategy            | Strategy | Strategy.md5                            | hash 计算策略           |
25 | | workerCountTobeTest | number[] | [1, 1, 1, 4, 4, 4, 8, 8, 8, 12, 12, 12] | 1/4/8/12 线程下各测试 3 次 |
26 | 
27 | ```ts
28 | // strategy.ts
29 | export enum Strategy {
30 |   md5 = 'md5',
31 |   crc32 = 'crc32',
32 |   xxHash64 = 'xxHash64',
33 |   mixed = 'mixed',
34 | }
35 | ```
36 | ### LICENSE
37 | 
38 | [MIT](./../../LICENSE)
39 | 


--------------------------------------------------------------------------------
/packages/benchmark/README.md:
--------------------------------------------------------------------------------
 1 | ## Introduce for benchmark
 2 | 
 3 | This project is used to test the hash calculation speed of Hash worker in different threads.
 4 | 
 5 | It supports both `Browser` and `Node.js` environments.
 6 | 
 7 | ### Usage
 8 | 
 9 | ```ts
10 | import { benchmark, BenchmarkOptions } from 'hash-worker-benchmark'
11 | 
12 | // options is optional.
13 | const options: BenchmarkOptions = {}
14 | benchmark(options)
15 | ```
16 | 
17 | ### Options
18 | 
19 | **BenchmarkOptions**
20 | 
21 | | filed               | type     | default                                 | description                |
22 | | ------------------- | -------- | --------------------------------------- | -------------------------- |
23 | | sizeInMB            | number   | 500                                     | File size for testing (MB)    |
24 | | strategy            | Strategy | Strategy.md5                            | Hash computation strategy     |
25 | | workerCountTobeTest | number[] | [1, 1, 1, 4, 4, 4, 8, 8, 8, 12, 12, 12] | Hashing performance was measured 3 times in each of the 1/4/8/12 threads |
26 | 
27 | ```ts
28 | // strategy.ts
29 | export enum Strategy {
30 |   md5 = 'md5',
31 |   crc32 = 'crc32',
32 |   xxHash64 = 'xxHash64',
33 |   mixed = 'mixed',
34 | }
35 | ```
36 | ### LICENSE
37 | 
38 | [MIT](./../../LICENSE)
39 | 


--------------------------------------------------------------------------------
/packages/benchmark/dts-bundle.config.json:
--------------------------------------------------------------------------------
1 | {
2 |   "entries": [
3 |     {
4 |       "filePath": "./src/index.ts",
5 |       "outFile": "./dist/index.d.ts"
6 |     }
7 |   ]
8 | }
9 | 


--------------------------------------------------------------------------------
/packages/benchmark/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "hash-worker-benchmark",
 3 |   "version": "1.0.0",
 4 |   "private": true,
 5 |   "type": "module",
 6 |   "main": "./dist/index.cjs",
 7 |   "module": "./dist/index.js",
 8 |   "types": "./dist/index.d.ts",
 9 |   "exports": {
10 |     ".": {
11 |       "types": "./dist/index.d.ts",
12 |       "require": "./dist/index.cjs",
13 |       "import": "./dist/index.js"
14 |     }
15 |   },
16 |   "scripts": {
17 |     "dev": "tsup --config tsup.config.ts --watch",
18 |     "build:benchmark": "pnpm rm:dist && tsup --config tsup.config.ts && pnpm build:dts",
19 |     "build:dts": "dts-bundle-generator --config dts-bundle.config.json",
20 |     "rm:dist": "rimraf ./dist"
21 |   },
22 |   "license": "MIT",
23 |   "dependencies": {
24 |     "hash-worker": "workspace:*"
25 |   },
26 |   "peerDependencies": {
27 |     "hash-worker": "workspace:*"
28 |   }
29 | }
30 | 


--------------------------------------------------------------------------------
/packages/benchmark/src/benchmark.ts:
--------------------------------------------------------------------------------
  1 | import { getFileHashChunks, isBrowser, isNode, Strategy } from 'hash-worker'
  2 | import { BenchmarkOptions, NormalizeOptions } from './types'
  3 | import {
  4 |   createMockFile,
  5 |   createMockFileInLocal,
  6 |   deleteLocalFile,
  7 |   normalizeBenchmarkOptions,
  8 |   sleep,
  9 | } from './helper'
 10 | import { ChalkInstance } from 'chalk'
 11 | 
 12 | const filePath = './data.txt'
 13 | const fileName = 'data.txt'
 14 | 
 15 | function buildParamsForBrowser(options: BenchmarkOptions): NormalizeOptions {
 16 |   const { sizeInMB, strategy, workerCountTobeTest } = normalizeBenchmarkOptions(options)
 17 |   const mockFile = createMockFile(fileName, sizeInMB)
 18 | 
 19 |   return {
 20 |     sizeInMB,
 21 |     params: workerCountTobeTest.map((workerCount) => ({
 22 |       file: mockFile,
 23 |       config: {
 24 |         workerCount,
 25 |         strategy,
 26 |       },
 27 |     })),
 28 |   }
 29 | }
 30 | 
 31 | function buildParamsForNode(options: BenchmarkOptions): NormalizeOptions {
 32 |   const { sizeInMB, strategy, workerCountTobeTest } = normalizeBenchmarkOptions(options)
 33 | 
 34 |   return {
 35 |     sizeInMB,
 36 |     params: workerCountTobeTest.map((workerCount) => ({
 37 |       filePath,
 38 |       config: {
 39 |         workerCount,
 40 |         strategy,
 41 |       },
 42 |     })),
 43 |   }
 44 | }
 45 | 
 46 | export async function benchmark(options: BenchmarkOptions = {}) {
 47 |   const isNodeEnv = isNode()
 48 |   const isBrowserEnv = isBrowser()
 49 |   const yellow = 'color: #FFB049;'
 50 |   let chalkYellow: ChalkInstance
 51 | 
 52 |   await initChalk()
 53 | 
 54 |   logInitialMsg()
 55 |   let normalizeOptions: NormalizeOptions
 56 |   if (isBrowserEnv) {
 57 |     console.log('Creating mock file ⏳')
 58 |     normalizeOptions = buildParamsForBrowser(options)
 59 |   } else if (isNodeEnv) {
 60 |     normalizeOptions = buildParamsForNode(options)
 61 |   } else {
 62 |     throw new Error('Unsupported environment')
 63 |   }
 64 | 
 65 |   const { sizeInMB, params } = normalizeOptions
 66 | 
 67 |   if (isNodeEnv) {
 68 |     console.log('Creating mock file ⏳')
 69 |     await createMockFileInLocal(filePath, sizeInMB)
 70 |   }
 71 | 
 72 |   let preWorkerCount = 1
 73 |   const preSpeed: number[] = []
 74 | 
 75 |   const getAverageSpeed = (workerCount = 0) => {
 76 |     if (preSpeed.length === 0) return
 77 |     const averageSpeed = preSpeed.reduce((acc, cur) => acc + cur, 0) / preSpeed.length
 78 |     logAvgSpeed(averageSpeed)
 79 |     preWorkerCount = workerCount
 80 |     preSpeed.length = 0
 81 |   }
 82 | 
 83 |   logStrategy(normalizeOptions.params[0].config?.strategy)
 84 | 
 85 |   for (const param of params) {
 86 |     const workerCount = param.config!.workerCount!
 87 |     if (workerCount !== preWorkerCount) getAverageSpeed(workerCount)
 88 |     const beforeDate = Date.now()
 89 |     await getFileHashChunks(param)
 90 |     const overTime = Date.now() - beforeDate
 91 |     const speed = sizeInMB / (overTime / 1000)
 92 |     if (workerCount === preWorkerCount) preSpeed.push(speed)
 93 |     logCurSpeed(overTime, workerCount, speed)
 94 |     await sleep(1000)
 95 |   }
 96 |   getAverageSpeed(preWorkerCount)
 97 | 
 98 |   if (isNodeEnv) {
 99 |     console.log('Clearing temp file ⏳')
100 |     await deleteLocalFile(filePath)
101 |   }
102 | 
103 |   logCompletion()
104 | 
105 |   async function initChalk() {
106 |     if (isNodeEnv) {
107 |       const chalk = (await import('chalk')).default
108 |       chalkYellow = chalk.hex('#FFB049')
109 |     }
110 |   }
111 | 
112 |   function logInitialMsg() {
113 |     isBrowserEnv && console.log('%cHash Worker Benchmark 🎯', yellow)
114 |     isNodeEnv && console.log(`${chalkYellow!('Hash Worker Benchmark')} 🎯`)
115 |   }
116 | 
117 |   function logStrategy(strategy?: Strategy) {
118 |     isBrowserEnv && console.log(`Running benchmark for %c${strategy} %cstrategy 🚀`, yellow, '')
119 |     isNodeEnv && console.log(`Running benchmark for ${chalkYellow!(strategy + ' strategy')} 🚀`)
120 |   }
121 | 
122 |   function logAvgSpeed(averageSpeed: number) {
123 |     isBrowserEnv && console.log(`Average speed: %c${averageSpeed.toFixed(2)} Mb/s`, yellow)
124 |     isNodeEnv && console.log(`Average speed: ${chalkYellow!(averageSpeed.toFixed(2) + 'Mb/s')}`)
125 |   }
126 | 
127 |   function logCurSpeed(overTime: number, workerCount: number, speed: number) {
128 |     isBrowserEnv &&
129 |       console.log(
130 |         `Get file hash in: %c${overTime} ms%c by using %c${workerCount} worker%c, speed: %c${speed.toFixed(2)} Mb/s`,
131 |         yellow, // 为 overTime 设置黄色
132 |         '', // 重置为默认颜色
133 |         yellow, // 为 workerCount 设置黄色
134 |         '', // 重置为默认颜色
135 |         yellow, // 为 speed 设置黄色
136 |       )
137 |     isNodeEnv &&
138 |       console.log(
139 |         `Get file hash in: ${chalkYellow!(overTime + ' ms')} by using ${chalkYellow!(workerCount) + ' worker'}, ` +
140 |           `speed: ${chalkYellow!(speed.toFixed(2) + ' Mb/s')}`,
141 |       )
142 |   }
143 | 
144 |   function logCompletion() {
145 |     isBrowserEnv && console.log('%cDone 🎈', yellow)
146 |     isNodeEnv && console.log(chalkYellow!('Done ') + '🎈')
147 |     if (isBrowserEnv) {
148 |       alert('Please check the console for benchmark information ~')
149 |     }
150 |   }
151 | }
152 | 


--------------------------------------------------------------------------------
/packages/benchmark/src/helper.ts:
--------------------------------------------------------------------------------
 1 | import { BenchmarkOptions } from './types'
 2 | import { Strategy } from 'hash-worker'
 3 | 
 4 | export function normalizeBenchmarkOptions(options: BenchmarkOptions): Required<BenchmarkOptions> {
 5 |   const defaultWorkerCountTobeTest = [1, 1, 1, 4, 4, 4, 8, 8, 8, 12, 12, 12]
 6 |   const { sizeInMB, strategy, workerCountTobeTest } = options
 7 | 
 8 |   const normalizeOptions = {
 9 |     sizeInMB: sizeInMB ?? 500,
10 |     strategy: strategy ?? Strategy.md5,
11 |     workerCountTobeTest: workerCountTobeTest ?? defaultWorkerCountTobeTest,
12 |   }
13 | 
14 |   const { workerCountTobeTest: _workerCountTobeTest } = normalizeOptions
15 | 
16 |   if (
17 |     _workerCountTobeTest.length === 0 ||
18 |     _workerCountTobeTest.find((num: number) => num <= 0 || num > 32 || !Number.isInteger(num))
19 |   ) {
20 |     throw new Error('Illegal workerCount')
21 |   }
22 | 
23 |   return normalizeOptions
24 | }
25 | 
26 | export async function sleep(ms: number) {
27 |   await new Promise<void>((rs) => setTimeout(() => rs(), ms))
28 | }
29 | 
30 | export async function createMockFileInLocal(filePath: string, sizeInMB: number): Promise<void> {
31 |   const { createWriteStream } = await import('fs')
32 |   const { randomBytes } = await import('crypto')
33 |   const generateRandomData = (size: number) => randomBytes(size).toString('hex')
34 | 
35 |   const stream = createWriteStream(filePath)
36 |   const size = 1024 * 1024 * sizeInMB // 总大小转换为字节
37 |   const chunkSize = 1024 * 512 // 每次写入512KB
38 | 
39 |   let written = 0
40 | 
41 |   return new Promise<void>((resolve, reject) => {
42 |     stream.on('error', reject)
43 |     const write = (): void => {
44 |       let ok = true
45 |       do {
46 |         const chunk = generateRandomData(chunkSize > size - written ? size - written : chunkSize)
47 |         written += chunk.length / 2 // 更新已写入的长度，除以2因为hex字符串表示的字节长度是实际长度的一半
48 | 
49 |         if (written >= size) {
50 |           // 如果达到或超过预定大小，则写入最后一个块并结束
51 |           stream.write(chunk, () => stream.end())
52 |           resolve()
53 |         } else {
54 |           // 否则，继续写入
55 |           ok = stream.write(chunk)
56 |         }
57 |       } while (written < size && ok)
58 |       if (written < size) {
59 |         // 'drain' 事件会在可以安全地继续写入数据到流中时触发
60 |         stream.once('drain', write)
61 |       }
62 |     }
63 |     write()
64 |   })
65 | }
66 | 
67 | export async function deleteLocalFile(path: string) {
68 |   const { unlinkSync } = await import('fs')
69 |   unlinkSync(path)
70 | }
71 | 
72 | export function createMockFile(fileName: string, sizeInMB: number): File {
73 |   // 每 MB 大约为 1048576 字节
74 |   const size = sizeInMB * 1048576
75 |   const buffer = new ArrayBuffer(size)
76 |   const view = new Uint8Array(buffer)
77 | 
78 |   // 填充随机内容
79 |   for (let i = 0; i < size; i++) {
80 |     // 随机填充每个字节，这里是填充 0-255 的随机数
81 |     // 实际应用中，你可能需要调整生成随机数的方式以达到所需的随机性
82 |     view[i] = Math.floor(Math.random() * 256)
83 |   }
84 | 
85 |   // 将 ArrayBuffer 转换为Blob
86 |   const blob = new Blob([view], { type: 'application/octet-stream' })
87 | 
88 |   // 将 Blob 转换为File
89 |   return new File([blob], fileName, { type: 'application/octet-stream' })
90 | }
91 | 


--------------------------------------------------------------------------------
/packages/benchmark/src/index.ts:
--------------------------------------------------------------------------------
1 | export type * from './types'
2 | export * from './benchmark'
3 | 


--------------------------------------------------------------------------------
/packages/benchmark/src/types.ts:
--------------------------------------------------------------------------------
 1 | import { HashChksParam, Strategy } from 'hash-worker'
 2 | 
 3 | export interface BenchmarkOptions {
 4 |   sizeInMB?: number // 默认: 测试文件 500MB
 5 |   strategy?: Strategy // 默认: 使用 MD5 作为 hash 策略
 6 |   workerCountTobeTest?: number[] // 默认: 1, 4, 8, 12 线程各测三次
 7 | }
 8 | 
 9 | export interface NormalizeOptions {
10 |   sizeInMB: number
11 |   params: HashChksParam[]
12 | }
13 | 


--------------------------------------------------------------------------------
/packages/benchmark/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 |   "extends": "../../tsconfig.json",
3 |   "include": ["src/**/*.ts"]
4 | }
5 | 


--------------------------------------------------------------------------------
/packages/benchmark/tsup.config.ts:
--------------------------------------------------------------------------------
1 | import { defineConfig } from 'tsup'
2 | 
3 | export default defineConfig({
4 |   entry: ['src/index.ts'],
5 |   format: ['esm', 'cjs'],
6 |   external: ['hash-worker'],
7 | })
8 | 


--------------------------------------------------------------------------------
/packages/core/CHANGELOG.md:
--------------------------------------------------------------------------------
 1 | # hash-worker
 2 | 
 3 | ## 1.0.1
 4 | 
 5 | ### Minor Changes
 6 | 
 7 | - fix: 当 0 <= chunkSize < 1 时, 导致分片函数死循环的问题
 8 | 
 9 | ## 1.0.0
10 | 
11 | ### Major Changes
12 | 
13 | - release version 1.0.0
14 | 
15 | ## 0.1.3
16 | 
17 | ### Minor Changes
18 | 
19 | - feat: 添加了 Webpack 下报错的解决方案, 升级了项目到最新的依赖
20 | 


--------------------------------------------------------------------------------
/packages/core/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2024 Tkunl
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/packages/core/README-zh.md:
--------------------------------------------------------------------------------
  1 | # Hash Worker [![npm package](https://img.shields.io/npm/v/hash-worker.svg)](https://www.npmjs.com/package/hash-worker) [![Bundle size](https://badgen.net/bundlephobia/minzip/hash-worker)](https://bundlephobia.com/result?p=hash-worker)  [![codecov](https://codecov.io/gh/Tkunl/hash-worker/graph/badge.svg?token=G7GYAPEPYS)](https://codecov.io/gh/Tkunl/hash-worker) ![GitHub License](https://img.shields.io/github/license/Tkunl/hash-worker)
  2 | 
  3 | <p align="center">
  4 | <img src="https://socialify.git.ci/Tkunl/hash-worker/image?font=Inter&language=1&name=1&owner=1&pattern=Plus&theme=Auto" width="640" height="320" />
  5 | </p>
  6 | 
  7 | ## Introduce
  8 | 
  9 | [English Document](./README.md)
 10 | 
 11 | **Hash-worker** 是一个用于快速计算文件哈希值的库。
 12 | 
 13 | 它基于 hash-wasm 且利用了 WebWorker 进行并行计算，从而加快了计算文件分片的计算速度。
 14 | 
 15 | Hash-worker 支持三种哈希计算算法：`md5`, `crc32` 和 `xxHash64`。
 16 | 
 17 | 同时支持 `浏览器` 和 `Node.js` 环境。
 18 | 
 19 | > [!WARNING]
 20 | > Hash-worker 计算出的 MerkleHash 是基于文件块哈希值构建的 MerkleTree 的根哈希值。请注意，这并不直接等同于文件本身的哈希值。
 21 | 
 22 | ## Install
 23 | 
 24 | ```bash
 25 | $ pnpm install hash-worker
 26 | ```
 27 | 
 28 | ## Usage
 29 | 
 30 | ### Global
 31 | 
 32 | ```html
 33 | 
 34 | <script src="./global.js"></script>
 35 | <script src="./worker/hash.worker.mjs"></script>
 36 | <script>
 37 |   HashWorker.getFileHashChunks()
 38 | </script>
 39 | ```
 40 | 
 41 | 其中 `global.js` 和 `hash.worker.mjs` 是执行 `package.json` 中的 `build:core` 后的打包产物
 42 | 
 43 | 打包产物位于 `packages/core/dist` 目录
 44 | 
 45 | ### ESM
 46 | 
 47 | ``` ts
 48 | import { getFileHashChunks, destroyWorkerPool, HashChksRes, HashChksParam } from 'hash-worker'
 49 | 
 50 | function handleGetHash(file: File) {
 51 |   const param: HashChksParam = {
 52 |     file: file,
 53 |     config: {
 54 |       workerCount: 8,
 55 |       strategy: Strategy.md5
 56 |     }
 57 |   }
 58 | 
 59 |   getFileHashChunks(param).then((data: HashChksRes) => {
 60 |     console.log('chunksHash', data.chunksHash)
 61 |   })
 62 | }
 63 | 
 64 | /**
 65 |  * Destroy Worker Thread
 66 |  */
 67 | function handleDestroyWorkerPool() {
 68 |   destroyWorkerPool()
 69 | }
 70 | ```
 71 | 
 72 | > [!WARNING]
 73 | > 如果你在使用 `Vite` 作为构建工具, 需要在 `Vite` 的配置文件中, 添加如下配置, 用于排除 vite 的依赖优化
 74 | 
 75 |  ```js
 76 | // vite.config.js
 77 | import { defineConfig } from 'vite'
 78 | import vue from '@vitejs/plugin-vue'
 79 | 
 80 | export default defineConfig({
 81 |   plugins: [vue()],
 82 |   // other configurations ...
 83 |   optimizeDeps: {
 84 |     exclude: ['hash-worker'] // new added..
 85 |   }
 86 | })
 87 |  ```
 88 | 
 89 | > [!WARNING]
 90 | > 如果你在使用 `Webpack` 作为构建工具, 需要在 Webpack 的配置文件中, 添加如下配置, 用于排除 node 相关模块的解析
 91 | 
 92 | ```js
 93 | // webpack.config.js
 94 | module.exports = {
 95 |   // new added..
 96 |   resolve: { 
 97 |     fallback: {
 98 |       fs: false,
 99 |       path: false,
100 |       'fs/promises': false,
101 |       worker_threads: false,
102 |     },
103 |   },
104 |   // new added..
105 |   externals: {
106 |     fs: 'commonjs fs',
107 |     path: 'commonjs path',
108 |     'fs/promises': 'commonjs fs/promises',
109 |     worker_threads: 'commonjs worker_threads',
110 |   },
111 | }
112 | ```
113 | 
114 | ## Options
115 | 
116 | **HashChksParam**
117 | 
118 | HashChksParam 是用于配置计算哈希值所需的参数。
119 | 
120 | | filed    | type   | default | description                 |
121 | |----------|--------|---------|-----------------------------|
122 | | file     | File   | /       | 需要计算 Hash 的文件（浏览器环境下必填）     |
123 | | filePath | string | /       | 需要计算 Hash 的文件路径 （Node环境下必填） |
124 | | config   | Config | Config  | 计算 Hash 时的参数                |
125 | 
126 | **Config**
127 | 
128 | | filed                    | type     | default        | description               |
129 | |--------------------------|----------|----------------|---------------------------|
130 | | chunkSize                | number   | 10 (MB)        | 文件分片的大小                   |
131 | | workerCount              | number   | 8              | 计算 Hash 时同时开启的 worker 数量  |
132 | | strategy                 | Strategy | Strategy.mixed | hash 计算策略                 |
133 | | borderCount              | number   | 100            | 'mixed' 模式下 hash 计算规则的分界点 |
134 | | isCloseWorkerImmediately | boolean  | true           | 当计算完成时, 是否立即销毁 Worker 线程  |
135 | 
136 | ```ts
137 | // strategy.ts
138 | export enum Strategy {
139 |   md5 = 'md5',
140 |   crc32 = 'crc32',
141 |   xxHash64 = 'xxHash64',
142 |   mixed = 'mixed',
143 | }
144 | ```
145 | 
146 | 当采用 Strategy.mixed 策略时，若文件分片数量少于 borderCount，将采用 md5 算法计算哈希值来构建 MerkleTree。
147 | 否则，则切换至使用 crc32 算法进行 MerkleTree 的构建。
148 | 
149 | **HashChksRes**
150 | 
151 | HashChksRes 是计算哈希值之后的返回结果。
152 | 
153 | | filed      | type         | description              |
154 | |------------|--------------|--------------------------|
155 | | chunksBlob | Blob[]       | 仅在浏览器环境下，会返回文件分片的 Blob[] |
156 | | chunksHash | string[]     | 文件分片的 Hash[]             |
157 | | merkleHash | string       | 文件的 merkleHash           |
158 | | metadata   | FileMetaInfo | 文件的 metadata             |
159 | 
160 | **FileMetaInfo**
161 | 
162 | | filed        | type   | description    |
163 | |--------------|--------|----------------|
164 | | name         | string | 用于计算 hash 的文件名 |
165 | | size         | number | 文件大小，单位：KB     |
166 | | lastModified | number | 文件最后一次修改的时间戳   |
167 | | type         | string | 文件后缀名          |
168 | 
169 | ### [Benchmark (MD5)](./packages/benchmark/README-zh.md)
170 | 
171 | | Worker Count | Speed     |
172 | |--------------|-----------|
173 | | 1            | 229 MB/s  |
174 | | 4            | 632 MB/s  |
175 | | 8            | 886 MB/s  |
176 | | 12           | 1037 MB/s |
177 | 
178 | * 以上数据是运行在 `Chrome v131` 和 `AMD Ryzen9 5950X` CPU 下, 通过使用 md5 来计算 hash 得到的。
179 | 
180 | ## LICENSE
181 | 
182 | [MIT](./LICENSE)
183 | 
184 | ## Contributions
185 | 
186 | 欢迎贡献代码！如果你发现了一个 bug 或者想添加一个新功能，请提交一个 issue 或 pull request。
187 | 
188 | ## Author and contributors
189 | 
190 | <p align="center">
191 |   <a href="https://github.com/Tkunl">
192 |     <img src="https://avatars.githubusercontent.com/u/19854081?v=4" width="40" height="40" alt="Tkunl">
193 |   </a>
194 |   <a href="https://github.com/nonzzz">
195 |     <img src="https://avatars.githubusercontent.com/u/52351095?v=4&s=40" width="40" height="40" alt="Kanno">
196 |   </a>
197 |   <a href="https://github.com/Eternal-could">
198 |     <img src="https://avatars.githubusercontent.com/u/74654896?v=4" width="40" height="40" alt="Eternal-could">
199 |   </a>
200 | </p>
201 | 
202 | 
203 | 
204 | 
205 | 
206 | 
207 | 


--------------------------------------------------------------------------------
/packages/core/README.md:
--------------------------------------------------------------------------------
  1 | # Hash Worker [![npm package](https://img.shields.io/npm/v/hash-worker.svg)](https://www.npmjs.com/package/hash-worker) [![Bundle size](https://badgen.net/bundlephobia/minzip/hash-worker)](https://bundlephobia.com/result?p=hash-worker)  [![codecov](https://codecov.io/gh/Tkunl/hash-worker/graph/badge.svg?token=G7GYAPEPYS)](https://codecov.io/gh/Tkunl/hash-worker) ![GitHub License](https://img.shields.io/github/license/Tkunl/hash-worker)
  2 | 
  3 | <p align="center">
  4 | <img src="https://socialify.git.ci/Tkunl/hash-worker/image?font=Inter&language=1&name=1&owner=1&pattern=Plus&theme=Auto" width="640" height="320" />
  5 | </p>
  6 | 
  7 | ## Introduce
  8 | 
  9 | [中文文档](./README-zh.md)
 10 | 
 11 | **Hash-worker** is a library for fast calculation of file chunk hashes.
 12 | 
 13 | It is based on `hash-wasm` and utilizes `WebWorkers` for parallel computation, which speeds up computation when
 14 | processing file blocks.
 15 | 
 16 | Hash-worker supports three hash computation algorithms: `md5`, `crc32` and `xxHash64`.
 17 | 
 18 | Both `browser` and `Node.js` are supported.
 19 | 
 20 | > [!WARNING]
 21 | > The merkleHash computed by the Hash-worker is the root hash of a MerkleTree constructed based on file block hashes. Note that this is not directly equivalent to a hash of the file itself.
 22 | 
 23 | ## Install
 24 | 
 25 | ```bash
 26 | $ pnpm install hash-worker
 27 | ```
 28 | 
 29 | ## Usage
 30 | 
 31 | ### Global
 32 | 
 33 | ```html
 34 | 
 35 | <script src="./global.js"></script>
 36 | <script src="./worker/hash.worker.mjs"></script>
 37 | <script>
 38 |   HashWorker.getFileHashChunks()
 39 | </script>
 40 | ```
 41 | 
 42 | The `global.js` and `hash.worker.mjs` are the build artifacts resulting from executing `build:core` in `package.json`.
 43 | 
 44 | The build artifacts are located in the `packages/core/dist` directory.
 45 | 
 46 | ### ESM
 47 | 
 48 | ``` ts
 49 | import { getFileHashChunks, destroyWorkerPool, HashChksRes, HashChksParam } from 'hash-worker'
 50 | 
 51 | function handleGetHash(file: File) {
 52 |   const param: HashChksParam = {
 53 |     file: file,
 54 |     config: {
 55 |       workerCount: 8,
 56 |       strategy: Strategy.md5
 57 |     }
 58 |   }
 59 | 
 60 |   getFileHashChunks(param).then((data: HashChksRes) => {
 61 |     console.log('chunksHash', data.chunksHash)
 62 |   })
 63 | }
 64 | 
 65 | /**
 66 |  * Destroy Worker Thread
 67 |  */
 68 | function handleDestroyWorkerPool() {
 69 |   destroyWorkerPool()
 70 | }
 71 | ```
 72 | 
 73 | > [!WARNING]
 74 | If you are using `Vite` as your build tool, you need to add some configurations in your `vite.config.js` to exclude hash-worker from optimizeDeps.
 75 | 
 76 | ```js
 77 | // vite.config.js
 78 | import { defineConfig } from 'vite'
 79 | import vue from '@vitejs/plugin-vue'
 80 | 
 81 | export default defineConfig({
 82 |   plugins: [vue()],
 83 |   // other configurations ...
 84 |   optimizeDeps: {
 85 |     exclude: ['hash-worker'] // new added..
 86 |   }
 87 | })
 88 | ```
 89 | 
 90 | > [!WARNING]
 91 | >
 92 | > If you are using `Webpack` as your build tool, you need add some configs in your `webpack.config.js` for exclude the parsing of node related modules.
 93 | 
 94 | ```js
 95 | // webpack.config.js
 96 | module.exports = {
 97 |   // new added..
 98 |   resolve: { 
 99 |     fallback: {
100 |       fs: false,
101 |       path: false,
102 |       'fs/promises': false,
103 |       worker_threads: false,
104 |     },
105 |   },
106 |   // new added..
107 |   externals: {
108 |     fs: 'commonjs fs',
109 |     path: 'commonjs path',
110 |     'fs/promises': 'commonjs fs/promises',
111 |     worker_threads: 'commonjs worker_threads',
112 |   },
113 | }
114 | ```
115 | 
116 | ## Options
117 | 
118 | **HashChksParam**
119 | 
120 | HashChksParam is used to configure the parameters needed to calculate the hash.
121 | 
122 | | filed    | type   | default | description                                                                          |
123 | |----------|--------|---------|--------------------------------------------------------------------------------------|
124 | | file     | File   | /       | Files that need to calculate the hash (required for browser environments)            |
125 | | filePath | string | /       | Path to the file where the hash is to be calculated (required for Node environments) |
126 | | config   | Config | Config  | Parameters for calculating the Hash                                                  |
127 | 
128 | **Config**
129 | 
130 | | filed                    | type     | default        | description                                                                       |
131 | |--------------------------|----------|----------------|-----------------------------------------------------------------------------------|
132 | | chunkSize                | number   | 10 (MB)        | Size of the file slice                                                            |
133 | | workerCount              | number   | 8              | Number of workers turned on at the same time as the hash is calculated            |
134 | | strategy                 | Strategy | Strategy.mixed | Hash computation strategy                                                         |
135 | | borderCount              | number   | 100            | The cutoff for the hash calculation rule in 'mixed' mode                          |
136 | | isCloseWorkerImmediately | boolean  | true           | Whether to destroy the worker thread immediately when the calculation is complete |
137 | 
138 | ```ts
139 | // strategy.ts
140 | export enum Strategy {
141 |   md5 = 'md5',
142 |   crc32 = 'crc32',
143 |   xxHash64 = 'xxHash64',
144 |   mixed = 'mixed',
145 | }
146 | ```
147 | 
148 | When Strategy.mixed strategy is used, if the number of file fragments is less than borderCount, the md5 algorithm will
149 | be used to calculate the hash value to build the MerkleTree.
150 | Otherwise, it switches to using the crc32 algorithm for MerkleTree construction.
151 | 
152 | **HashChksRes**
153 | 
154 | HashChksRes is the returned result after calculating the hash value.
155 | 
156 | | filed      | type         | description                                                             |
157 | |------------|--------------|-------------------------------------------------------------------------|
158 | | chunksBlob | Blob[]       | In a browser environment only, the Blob[] of the file slice is returned |
159 | | chunksHash | string[]     | Hash[] for file slicing                                                 |
160 | | merkleHash | string       | The merkleHash of the file                                              |
161 | | metadata   | FileMetaInfo | The metadata of the file                                                |
162 | 
163 | **FileMetaInfo**
164 | 
165 | | filed        | type   | description                                     |
166 | |--------------|--------|-------------------------------------------------|
167 | | name         | string | The name of the file used to calculate the hash |
168 | | size         | number | File size in KB                                 |
169 | | lastModified | number | Timestamp of the last modification of the file  |
170 | | type         | string | file extension                                  |
171 | 
172 | ### [Benchmark (MD5)](./packages/benchmark/README.md)
173 | 
174 | | Worker Count | Speed     |
175 | |--------------|-----------|
176 | | 1            | 229 MB/s  |
177 | | 4            | 632 MB/s  |
178 | | 8            | 886 MB/s  |
179 | | 12           | 1037 MB/s |
180 | 
181 | The above data is run on the `Chrome v131` and `AMD Ryzen9 5950X` CPU, by using md5 to calculate hash.
182 | 
183 | ## LICENSE
184 | 
185 | [MIT](./LICENSE)
186 | 
187 | ## Contributions
188 | 
189 | Contributions are welcome! If you find a bug or want to add a new feature, please open an issue or submit a pull
190 | request.
191 | 
192 | ## Author and contributors
193 | 
194 | <p align="center">
195 |   <a href="https://github.com/Tkunl">
196 |     <img src="https://avatars.githubusercontent.com/u/19854081?v=4" width="40" height="40" alt="Tkunl">
197 |   </a>
198 |   <a href="https://github.com/Eternal-could">
199 |     <img src="https://avatars.githubusercontent.com/u/74654896?v=4" width="40" height="40" alt="Eternal-could">
200 |   </a>
201 |   <a href="https://github.com/nonzzz">
202 |     <img src="https://avatars.githubusercontent.com/u/52351095?v=4&s=40" width="40" height="40" alt="Kanno">
203 |   </a>
204 | </p>
205 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/browser/fileUtils.spec.ts:
--------------------------------------------------------------------------------
 1 | import { getArrayBufFromBlobs, getFileMetadata, sliceFile } from '../../src/utils'
 2 | import path from 'path'
 3 | import fs from 'fs/promises'
 4 | import { MockBlob } from '../fixture/mockBlob'
 5 | 
 6 | // 在测试文件的顶部，模拟 Blob.prototype.arrayBuffer
 7 | function mockArrayBuffer(): void {
 8 |   // 将全局 Blob 替换为 MockBlob
 9 |   global.Blob = MockBlob
10 | }
11 | 
12 | describe('sliceFile', () => {
13 |   it('should slice a file into chunks of specified size', () => {
14 |     // 创建一个模拟的 File 对象
15 |     const file = new File([new ArrayBuffer(5 * 1024 * 1024)], 'test.pdf', {
16 |       type: 'application/pdf',
17 |     }) // 创建一个5MB的文件
18 | 
19 |     const chunks = sliceFile(file, 1) // 指定将文件分割成1MB大小的块
20 | 
21 |     // 断言：检查chunks的长度是否为5
22 |     expect(chunks.length).toBe(5)
23 | 
24 |     // 遍历chunks，验证每个块的大小是否符合预期（除了最后一个块可能小于1MB）
25 |     chunks.forEach((chunk, index) => {
26 |       if (index === chunks.length - 1) {
27 |         // 若是最后一个块，则其大小应小于等于1MB
28 |         expect(chunk.size).toBeLessThanOrEqual(1 * 1024 * 1024)
29 |       } else {
30 |         // 否则，每个块的大小应该严格等于1MB
31 |         expect(chunk.size).toBe(1 * 1024 * 1024)
32 |       }
33 |     })
34 |   })
35 | })
36 | 
37 | describe('getArrayBufFromBlobs', () => {
38 |   beforeAll(() => {
39 |     // 在所有测试运行之前模拟 arrayBuffer
40 |     mockArrayBuffer()
41 |   })
42 | 
43 |   it('should correctly return array buffers from an array of blobs', async () => {
44 |     // 创建测试用的 ArrayBuffer 和 Blob
45 |     const buffer1 = new ArrayBuffer(10)
46 |     const buffer2 = new ArrayBuffer(20)
47 |     const blob1 = new Blob([buffer1])
48 |     const blob2 = new Blob([buffer2])
49 | 
50 |     // 调用你的函数
51 |     const result = await getArrayBufFromBlobs([blob1, blob2])
52 | 
53 |     // 进行断言
54 |     expect(result.length).toBe(2)
55 |     expect(result[0]).toBeInstanceOf(ArrayBuffer)
56 |     expect(result[1]).toBeInstanceOf(ArrayBuffer)
57 |     expect(result[0].byteLength).toEqual(10)
58 |     expect(result[1].byteLength).toEqual(20)
59 |   })
60 | })
61 | 
62 | describe('getFileMetadata', () => {
63 |   it('should correctly return file metadata in browser env', async () => {
64 |     const fileName = 'test.pdf'
65 |     // 创建一个模拟的 File 对象
66 |     const file = new File([new ArrayBuffer(5 * 1024 * 1024)], fileName, {
67 |       type: 'application/pdf',
68 |     }) // 创建一个 5MB 的文件
69 | 
70 |     const fileInfo = await getFileMetadata(file)
71 |     expect(fileInfo.name).toBe(fileName)
72 |     expect(fileInfo.type).toBe('.pdf')
73 |   })
74 | 
75 |   it('should correctly return file metadata in node env', async () => {
76 |     const filePath = path.join(__dirname, './../fixture/mockFile.txt')
77 | 
78 |     const fileInfo = await getFileMetadata(undefined, filePath)
79 |     const stats = await fs.stat(filePath)
80 | 
81 |     expect(fileInfo.name).toBe('mockFile.txt')
82 |     expect(fileInfo.size).toBe(stats.size / 1024)
83 |     expect(fileInfo.type).toBe('.txt')
84 |   })
85 | })
86 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/browser/getFileHashChunks.spec.ts:
--------------------------------------------------------------------------------
 1 | import { getFileHashChunks } from '../../src/getFileHashChunks'
 2 | 
 3 | jest.mock('../../src/utils/is', () => ({
 4 |   isNode: jest.fn(() => false),
 5 |   isBrowser: jest.fn(() => false),
 6 | }))
 7 | 
 8 | jest.mock('../../src/worker/workerService', () => ({
 9 |   WorkerService: jest.fn(() => ({
10 |     terminate: jest.fn(),
11 |   })),
12 | }))
13 | 
14 | jest.mock('../../src/utils/fileUtils', () => ({
15 |   getFileMetadata: jest.fn(() => ({
16 |     name: 'fakeFileName.txt',
17 |   })),
18 | }))
19 | 
20 | jest.mock('../../src/helper', () => ({
21 |   ...jest.requireActual('../../src/helper'), // 从中导入所有原始实现
22 |   processFileInBrowser: jest.fn(() => ({
23 |     chunksBlob: [],
24 |     chunksHash: ['hash in browser'],
25 |     fileHash: 'hash in browser',
26 |   })),
27 |   processFileInNode: jest.fn(() => ({
28 |     chunksHash: ['hash in node'],
29 |     fileHash: 'hash in node',
30 |   })),
31 | }))
32 | 
33 | import * as is from '../../src/utils/is'
34 | import { HashChksParam } from '../../src/interface'
35 | 
36 | function setBrowserEnv() {
37 |   ;(is.isNode as jest.Mock).mockImplementation(() => false)
38 |   ;(is.isBrowser as jest.Mock).mockImplementation(() => true)
39 | }
40 | 
41 | function setNodeEnv() {
42 |   ;(is.isNode as jest.Mock).mockImplementation(() => true)
43 |   ;(is.isBrowser as jest.Mock).mockImplementation(() => false)
44 | }
45 | 
46 | describe('getFileHashChunks', () => {
47 |   it('should return getFileHashChunks result correctly in browser env', async () => {
48 |     setBrowserEnv()
49 |     const param = <HashChksParam>{
50 |       file: new File([], 'test.pdf', {
51 |         type: 'application/pdf',
52 |       }),
53 |     }
54 |     const result = await getFileHashChunks(param)
55 | 
56 |     expect(result.merkleHash).toEqual('hash in browser')
57 |     expect(result.metadata.name).toEqual('fakeFileName.txt')
58 |   })
59 | 
60 |   it('should return getFileHashChunks result correctly in node env', async () => {
61 |     setNodeEnv()
62 |     const param = <HashChksParam>{
63 |       filePath: 'dummyPath',
64 |     }
65 |     const result = await getFileHashChunks(param)
66 | 
67 |     expect(result.merkleHash).toEqual('hash in node')
68 |     expect(result.metadata.name).toEqual('fakeFileName.txt')
69 |   })
70 | })
71 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/browser/helper.spec.ts:
--------------------------------------------------------------------------------
  1 | import { Strategy } from '../../src/interface'
  2 | 
  3 | jest.mock('hash-wasm', () => ({
  4 |   crc32: jest.fn(() => Promise.resolve('crc32hash')),
  5 |   md5: jest.fn(() => Promise.resolve('md5hash')),
  6 |   xxhash64: jest.fn(() => Promise.resolve('xxhash64')),
  7 | }))
  8 | 
  9 | jest.mock('../../src/utils/is', () => ({
 10 |   isNode: jest.fn(() => false),
 11 |   isBrowser: jest.fn(() => false),
 12 | }))
 13 | 
 14 | jest.mock('../../src/worker/workerService', () => {
 15 |   return {
 16 |     WorkerService: jest.fn().mockImplementation(() => ({
 17 |       getMD5ForFiles: jest.fn(),
 18 |       getCRC32ForFiles: jest.fn(),
 19 |       getXxHash64ForFiles: jest.fn(),
 20 |       terminate: jest.fn(),
 21 |     })),
 22 |   }
 23 | })
 24 | 
 25 | import { getChunksHashMultiple, getChunksHashSingle, normalizeParam } from '../../src/helper'
 26 | import * as is from '../../src/utils/is'
 27 | import { WorkerService } from '../../src/worker/workerService'
 28 | 
 29 | function setNodeEnv() {
 30 |   ;(is.isNode as jest.Mock).mockImplementation(() => true)
 31 |   ;(is.isBrowser as jest.Mock).mockImplementation(() => false)
 32 | }
 33 | 
 34 | function setBrowserEnv() {
 35 |   ;(is.isNode as jest.Mock).mockImplementation(() => false)
 36 |   ;(is.isBrowser as jest.Mock).mockImplementation(() => true)
 37 | }
 38 | 
 39 | describe('normalizeParam', () => {
 40 |   let mockFile: File
 41 | 
 42 |   beforeAll(async () => {
 43 |     mockFile = new File([new ArrayBuffer(5)], 'test.pdf', {
 44 |       type: 'application/pdf',
 45 |     })
 46 |   })
 47 | 
 48 |   it('throws an error for unsupported environment', () => {
 49 |     expect(() => {
 50 |       normalizeParam({ filePath: '' })
 51 |     }).toThrow('Unsupported environment')
 52 |   })
 53 | 
 54 |   it('requires filePath attribute in node environment', () => {
 55 |     setNodeEnv()
 56 |     expect(() => {
 57 |       normalizeParam({ file: mockFile })
 58 |     }).toThrow('The filePath attribute is required in node environment')
 59 |   })
 60 | 
 61 |   it('requires filePath attribute in browser environment', () => {
 62 |     setBrowserEnv()
 63 |     expect(() => {
 64 |       normalizeParam({ filePath: 'mockPath' })
 65 |     }).toThrow('The file attribute is required in browser environment')
 66 |   })
 67 | 
 68 |   it('get filePath in param correctly in node environment', () => {
 69 |     setNodeEnv()
 70 |     const param = normalizeParam({ filePath: 'mockPath' })
 71 |     expect(param.filePath).toBe('mockPath')
 72 |   })
 73 | 
 74 |   it('get file in param correctly in browser environment', () => {
 75 |     setBrowserEnv()
 76 |     const param = normalizeParam({ file: mockFile })
 77 |     expect(param.file).toBeTruthy()
 78 |   })
 79 | })
 80 | 
 81 | describe('getChunksHashSingle', () => {
 82 |   const testArrayBuffer = new Uint8Array([1, 2, 3]).buffer
 83 | 
 84 |   it('should use md5 hashing strategy for md5 strategy option', async () => {
 85 |     const result = await getChunksHashSingle(Strategy.md5, testArrayBuffer)
 86 |     expect(result).toEqual(['md5hash'])
 87 |   })
 88 | 
 89 |   it('should use mixed hashing strategy for mixed strategy option', async () => {
 90 |     const result = await getChunksHashSingle(Strategy.mixed, testArrayBuffer)
 91 |     expect(result).toEqual(['md5hash'])
 92 |   })
 93 | 
 94 |   it('should use crc32 hashing strategy for crc32 strategy option', async () => {
 95 |     const result = await getChunksHashSingle(Strategy.crc32, testArrayBuffer)
 96 |     expect(result).toEqual(['crc32hash'])
 97 |   })
 98 | 
 99 |   it('should use xxHash64 hashing strategy for xxHash64 strategy option', async () => {
100 |     const result = await getChunksHashSingle(Strategy.xxHash64, testArrayBuffer)
101 |     expect(result).toEqual(['xxhash64'])
102 |   })
103 | })
104 | 
105 | describe('getChunksHashMultiple', () => {
106 |   const arrayBuffers: ArrayBuffer[] = [new ArrayBuffer(10), new ArrayBuffer(20)]
107 |   let workerSvc: WorkerService
108 | 
109 |   beforeEach(() => {
110 |     workerSvc = new WorkerService(6)
111 |   })
112 | 
113 |   it('should use MD5 hashing for MD5 strategy', async () => {
114 |     await getChunksHashMultiple(Strategy.md5, arrayBuffers, 5, 3, workerSvc)
115 |     expect(workerSvc.getMD5ForFiles).toHaveBeenCalledWith(arrayBuffers)
116 |   })
117 | 
118 |   it('should use CRC32 hashing for CRC32 strategy', async () => {
119 |     await getChunksHashMultiple(Strategy.crc32, arrayBuffers, 5, 3, workerSvc)
120 |     expect(workerSvc.getCRC32ForFiles).toHaveBeenCalledWith(arrayBuffers)
121 |   })
122 | 
123 |   it('should use xxHash64 hashing for xxHash64 strategy', async () => {
124 |     await getChunksHashMultiple(Strategy.xxHash64, arrayBuffers, 5, 3, workerSvc)
125 |     expect(workerSvc.getXxHash64ForFiles).toHaveBeenCalledWith(arrayBuffers)
126 |   })
127 | 
128 |   it('should use MD5 hashing for mixed strategy when chunksCount <= borderCount', async () => {
129 |     await getChunksHashMultiple(Strategy.mixed, arrayBuffers, 2, 3, workerSvc)
130 |     expect(workerSvc.getMD5ForFiles).toHaveBeenCalledWith(arrayBuffers)
131 |   })
132 | 
133 |   it('should use CRC32 hashing for mixed strategy when chunksCount > borderCount', async () => {
134 |     await getChunksHashMultiple(Strategy.mixed, arrayBuffers, 4, 3, workerSvc)
135 |     expect(workerSvc.getCRC32ForFiles).toHaveBeenCalledWith(arrayBuffers)
136 |   })
137 | })
138 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/browser/helper2.spec.ts:
--------------------------------------------------------------------------------
  1 | import * as helper from '../../src/helper'
  2 | import { Config, Strategy } from '../../src/interface'
  3 | import { getFileSliceLocations, readFileAsArrayBuffer, sliceFile } from '../../src/utils'
  4 | import { WorkerService } from '../../src/worker/workerService'
  5 | import { MockBlob } from '../fixture/mockBlob'
  6 | import { getRootHashByChunks } from '../../src/getRootHashByChunks'
  7 | import { processFileInBrowser, processFileInNode } from '../../src/helper'
  8 | 
  9 | global.Blob = MockBlob
 10 | 
 11 | jest.mock('../../src/worker/workerService', () => ({
 12 |   WorkerService: jest.fn().mockImplementation(() => ({
 13 |     terminate: jest.fn(),
 14 |   })),
 15 | }))
 16 | 
 17 | jest.mock('../../src/utils/fileUtils', () => ({
 18 |   sliceFile: jest.fn(),
 19 |   getArrayBufFromBlobs: jest.fn(),
 20 |   getFileSliceLocations: jest.fn(),
 21 |   readFileAsArrayBuffer: jest.fn(),
 22 | }))
 23 | 
 24 | jest.mock('../../src/helper', () => ({
 25 |   ...jest.requireActual('../../src/helper'), // 从中导入所有原始实现
 26 |   getChunksHashSingle: jest.fn(),
 27 |   getChunksHashMultiple: jest.fn(),
 28 | }))
 29 | 
 30 | jest.mock('../../src/getRootHashByChunks', () => ({
 31 |   getRootHashByChunks: jest.fn(),
 32 | }))
 33 | 
 34 | describe('processFileInBrowser', () => {
 35 |   const fakeFile = new File([], 'test.pdf', {
 36 |     type: 'application/pdf',
 37 |   })
 38 |   const config: Required<Config> = {
 39 |     chunkSize: 10,
 40 |     strategy: Strategy.md5,
 41 |     workerCount: 6,
 42 |     isCloseWorkerImmediately: true,
 43 |     borderCount: 2,
 44 |     isShowLog: false,
 45 |   }
 46 |   const workerSvc = new WorkerService(1)
 47 | 
 48 |   beforeEach(() => {
 49 |     // 用于清除所有已模拟函数（mock functions）的调用记录和实例数据
 50 |     jest.clearAllMocks()
 51 |   })
 52 | 
 53 |   it('should process single chunk file by using processFileInBrowser function', async () => {
 54 |     ;(sliceFile as jest.Mock).mockReturnValue([new Blob(['chunk'])])
 55 |     ;(helper.getChunksHashSingle as jest.Mock).mockResolvedValue(['hash1'])
 56 |     ;(getRootHashByChunks as jest.Mock).mockResolvedValue('rootHash')
 57 | 
 58 |     const result = await helper.processFileInBrowser(
 59 |       fakeFile,
 60 |       config,
 61 |       workerSvc,
 62 |       helper.getChunksHashSingle,
 63 |       helper.getChunksHashMultiple,
 64 |     )
 65 | 
 66 |     expect(sliceFile).toHaveBeenCalledWith(fakeFile, config.chunkSize)
 67 |     expect(helper.getChunksHashSingle).toHaveBeenCalled()
 68 |     expect(getRootHashByChunks).toHaveBeenCalledWith(['hash1'])
 69 |     expect(result.fileHash).toEqual('rootHash')
 70 |   })
 71 | 
 72 |   it('should process multiple chunk file by using processFileInBrowser function', async () => {
 73 |     ;(sliceFile as jest.Mock).mockReturnValue([new Blob(['chunk']), new Blob(['chunk2'])])
 74 |     ;(helper.getChunksHashMultiple as jest.Mock).mockResolvedValue(['hash1', 'hash2'])
 75 |     ;(getRootHashByChunks as jest.Mock).mockResolvedValue('rootHash')
 76 | 
 77 |     const result = await processFileInBrowser(
 78 |       fakeFile,
 79 |       config,
 80 |       workerSvc,
 81 |       helper.getChunksHashSingle,
 82 |       helper.getChunksHashMultiple,
 83 |     )
 84 | 
 85 |     expect(sliceFile).toHaveBeenCalledWith(fakeFile, config.chunkSize)
 86 |     expect(helper.getChunksHashMultiple as jest.Mock).toHaveBeenCalled()
 87 |     expect(getRootHashByChunks).toHaveBeenCalledWith(['hash1', 'hash2'])
 88 |     expect(result.fileHash).toEqual('rootHash')
 89 |   })
 90 | })
 91 | 
 92 | describe('processFileInNode function', () => {
 93 |   const config: Required<Config> = {
 94 |     chunkSize: 10,
 95 |     strategy: Strategy.md5,
 96 |     workerCount: 6,
 97 |     isCloseWorkerImmediately: true,
 98 |     borderCount: 2,
 99 |     isShowLog: false,
100 |   }
101 | 
102 |   const workerSvc = new WorkerService(1)
103 | 
104 |   beforeEach(() => {
105 |     // 用于清除所有已模拟函数（mock functions）的调用记录和实例数据
106 |     jest.clearAllMocks()
107 |   })
108 | 
109 |   it('should process single chunk file by using processFileInNode function', async () => {
110 |     ;(helper.getChunksHashSingle as jest.Mock).mockResolvedValue(['hash1'])
111 |     ;(getFileSliceLocations as jest.Mock).mockResolvedValue({ sliceLocation: [0], endLocation: 10 })
112 |     ;(readFileAsArrayBuffer as jest.Mock).mockResolvedValue(new ArrayBuffer(10))
113 |     ;(getRootHashByChunks as jest.Mock).mockResolvedValue('rootHash')
114 | 
115 |     const result = await processFileInNode(
116 |       'dummyPath',
117 |       config,
118 |       workerSvc,
119 |       helper.getChunksHashSingle,
120 |       helper.getChunksHashMultiple,
121 |     )
122 | 
123 |     expect(result.chunksHash).toEqual(['hash1'])
124 |     expect(helper.getChunksHashSingle).toHaveBeenCalled()
125 |     expect(result.fileHash).toEqual('rootHash')
126 |   })
127 | 
128 |   it('should process multiple chunk file by using processFileInNode function', async () => {
129 |     ;(helper.getChunksHashMultiple as jest.Mock).mockResolvedValue(['hash1', 'hash2'])
130 |     ;(getFileSliceLocations as jest.Mock).mockResolvedValue({
131 |       sliceLocation: [0, 10],
132 |       endLocation: 20,
133 |     })
134 |     ;(readFileAsArrayBuffer as jest.Mock).mockResolvedValue(new ArrayBuffer(10))
135 |     ;(getRootHashByChunks as jest.Mock).mockResolvedValue('rootHash')
136 | 
137 |     const result = await processFileInNode(
138 |       'dummyPath',
139 |       config,
140 |       workerSvc,
141 |       helper.getChunksHashSingle,
142 |       helper.getChunksHashMultiple,
143 |     )
144 | 
145 |     expect(result.chunksHash).toEqual(['hash1', 'hash2'])
146 |     expect(helper.getChunksHashMultiple).toHaveBeenCalled()
147 |     expect(result.fileHash).toEqual('rootHash')
148 |   })
149 | })
150 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/browser/workerPoolForHash.spec.ts:
--------------------------------------------------------------------------------
 1 | import { MockWebWorker } from '../fixture/mockWebWorker'
 2 | import { WorkerPoolForHash } from '../../src/worker/workerPoolForHash'
 3 | 
 4 | // 模拟浏览器下的 Web Worker
 5 | ;(global as any).Worker = MockWebWorker
 6 | 
 7 | // 模拟 Node.js 下的 worker_threads
 8 | jest.mock('worker_threads', () => ({
 9 |   Worker: jest.fn(),
10 | }))
11 | 
12 | describe('WorkerPoolForMd5s', () => {
13 |   test('create function should initialize pool correctly in Node environment', async () => {
14 |     const pool = await WorkerPoolForHash.create(4)
15 |     expect(pool.pool.length).toBe(4)
16 |     expect((await import('worker_threads')).Worker).toHaveBeenCalledTimes(4)
17 |   })
18 | })
19 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/browser/workerWrapper.spec.ts:
--------------------------------------------------------------------------------
 1 | import { StatusEnum, WorkerWrapper } from '../../src/entity'
 2 | import { MockWebWorker } from '../fixture/mockWebWorker'
 3 | 
 4 | // 在全局空间中声明这个类，以模拟在浏览器中的 Worker 行为
 5 | ;(global as any).Worker = MockWebWorker
 6 | 
 7 | describe('WorkerWrapper', () => {
 8 |   it('should handle messages correctly in browser environment', async () => {
 9 |     const webWorker = new Worker('')
10 |     const workerWrapper = new WorkerWrapper(webWorker)
11 |     const getFn = (param: ArrayBuffer) => param
12 |     const restoreFn: any = () => {}
13 |     const promise = workerWrapper.run(new ArrayBuffer(1), 0, getFn, restoreFn)
14 | 
15 |     await expect(promise).resolves.toBe('hash-string')
16 |     expect(workerWrapper.status).toBe(StatusEnum.WAITING)
17 |     expect(webWorker.terminate).toHaveBeenCalledTimes(0) // 根据需要测试 terminate 被调用的次数
18 |   })
19 | 
20 |   it('should call terminate on the worker', () => {
21 |     const mockTerminate = jest.fn()
22 |     const worker = new Worker('')
23 |     worker.terminate = mockTerminate
24 | 
25 |     const workerWrapper = new WorkerWrapper(worker)
26 |     workerWrapper.terminate()
27 | 
28 |     expect(mockTerminate).toHaveBeenCalled()
29 |   })
30 | })
31 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/fixture/mockBlob.ts:
--------------------------------------------------------------------------------
 1 | export class MockBlob extends Blob {
 2 |   constructor(...args: ConstructorParameters<typeof Blob>) {
 3 |     super(...args)
 4 |   }
 5 | 
 6 |   // 模拟 arrayBuffer 方法
 7 |   arrayBuffer(): Promise<ArrayBuffer> {
 8 |     return new Promise((resolve, reject) => {
 9 |       const reader: FileReader = new FileReader()
10 |       reader.onload = () => {
11 |         if (reader.result) {
12 |           resolve(reader.result as ArrayBuffer)
13 |         } else {
14 |           reject(new Error('ArrayBuffer is null'))
15 |         }
16 |       }
17 |       reader.onerror = () => {
18 |         reject(new Error('FileReader failed to read the Blob'))
19 |       }
20 |       reader.readAsArrayBuffer(this)
21 |     })
22 |   }
23 | }
24 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/fixture/mockFile.txt:
--------------------------------------------------------------------------------
 1 | Here is mockFile
 2 | Here is mockFile
 3 | Here is mockFile
 4 | Here is mockFile
 5 | Here is mockFile
 6 | Here is mockFile
 7 | Here is mockFile
 8 | Here is mockFile
 9 | Here is mockFile
10 | Here is mockFile
11 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/fixture/mockMiniSubject.ts:
--------------------------------------------------------------------------------
 1 | import { MiniSubject } from '../../src/utils'
 2 | 
 3 | export class MockMiniSubject<T> extends MiniSubject<T> {
 4 |   constructor(value: T) {
 5 |     super(value)
 6 |   }
 7 | 
 8 |   next(value: T) {
 9 |     this._value = value
10 |     this.subscribers.forEach((cb) => cb(value))
11 |   }
12 | }
13 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/fixture/mockWebWorker.ts:
--------------------------------------------------------------------------------
1 | export class MockWebWorker {
2 |   onmessage?: (event: any) => void
3 |   onerror?: (event: ErrorEvent) => void
4 |   postMessage = jest.fn().mockImplementation(() => {
5 |     this.onmessage && this.onmessage({ result: 'hash-string', chunk: new ArrayBuffer(1) })
6 |   })
7 |   terminate = jest.fn()
8 | }
9 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/fixture/mockWorkerPool.ts:
--------------------------------------------------------------------------------
 1 | import { WorkerPool } from '../../src/entity'
 2 | import { MockMiniSubject } from './mockMiniSubject'
 3 | import { MockWorkerWrapper } from './mockWorkerWrapper'
 4 | 
 5 | export class MockWorkerPool extends WorkerPool {
 6 |   constructor(maxWorkers = 4) {
 7 |     super(maxWorkers)
 8 |     this.curRunningCount = new MockMiniSubject(0)
 9 |     for (let i = 0; i < maxWorkers; i++) {
10 |       this.pool.push(new MockWorkerWrapper())
11 |     }
12 |   }
13 | }
14 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/fixture/mockWorkerWrapper.ts:
--------------------------------------------------------------------------------
 1 | import { StatusEnum, WorkerWrapper } from '../../src/entity'
 2 | 
 3 | export class MockWorkerWrapper extends WorkerWrapper {
 4 |   constructor() {
 5 |     super({
 6 |       terminate: () => {},
 7 |     } as Worker)
 8 |     this.status = StatusEnum.WAITING
 9 |   }
10 | 
11 |   // eslint-disable-next-line @typescript-eslint/no-unused-vars
12 |   run<T, U>(param: U, index: number, getFn: any, restoreFn: any) {
13 |     return Promise.resolve('result' as unknown as T)
14 |   }
15 | }
16 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/node/arrayUtils.spec.ts:
--------------------------------------------------------------------------------
  1 | import { getArrParts, getFileSliceLocations, readFileAsArrayBuffer } from '../../src/utils'
  2 | import fs from 'fs/promises'
  3 | import path from 'path'
  4 | import { describe } from 'node:test'
  5 | 
  6 | test('getArrParts should split array into parts of given size', () => {
  7 |   const input = [1, 2, 3, 4, 5, 6, 7]
  8 |   const size = 3
  9 |   const expectedOutput = [[1, 2, 3], [4, 5, 6], [7]]
 10 | 
 11 |   const result = getArrParts(input, size)
 12 | 
 13 |   expect(result).toEqual(expectedOutput)
 14 | })
 15 | 
 16 | test('getArrParts should handle empty array', () => {
 17 |   const input: [] = []
 18 |   const size = 3
 19 |   const expectedOutput: [] = []
 20 | 
 21 |   const result = getArrParts(input, size)
 22 | 
 23 |   expect(result).toEqual(expectedOutput)
 24 | })
 25 | 
 26 | test('getArrParts should handle size larger than array length', () => {
 27 |   const input = [1, 2, 3]
 28 |   const size = 5
 29 |   const expectedOutput = [[1, 2, 3]]
 30 | 
 31 |   const result = getArrParts(input, size)
 32 | 
 33 |   expect(result).toEqual(expectedOutput)
 34 | })
 35 | 
 36 | test('getArrParts should handle size of 1', () => {
 37 |   const input = [1, 2, 3]
 38 |   const size = 1
 39 |   const expectedOutput = [[1], [2], [3]]
 40 | 
 41 |   const result = getArrParts(input, size)
 42 | 
 43 |   expect(result).toEqual(expectedOutput)
 44 | })
 45 | 
 46 | test('getArrParts should handle size equal to array length', () => {
 47 |   const input = [1, 2, 3]
 48 |   const size = 3
 49 |   const expectedOutput = [[1, 2, 3]]
 50 | 
 51 |   const result = getArrParts(input, size)
 52 | 
 53 |   expect(result).toEqual(expectedOutput)
 54 | })
 55 | 
 56 | describe('getFileSliceLocations function slices file as expected', async () => {
 57 |   let sliceLocation: [number, number][]
 58 |   let endLocation: number
 59 |   const filePath = path.join(__dirname, './../fixture/mockFile.txt')
 60 | 
 61 |   // 基于你的测试文件和 baseSize 计算出预期的分片
 62 |   let expectedEndLocation: number
 63 | 
 64 |   // 在所有测试运行之前，执行一次异步操作。
 65 |   beforeAll(async () => {
 66 |     const baseSize = 1
 67 |     const result = await getFileSliceLocations(filePath, baseSize)
 68 |     const stats = await fs.stat(filePath)
 69 | 
 70 |     sliceLocation = result.sliceLocation
 71 |     endLocation = result.endLocation
 72 |     expectedEndLocation = stats.size
 73 |   })
 74 | 
 75 |   // 假设你知道 fileContent 的大小，可以根据它计算期望的 sliceLocation 值
 76 |   const expectedSliceLocation = [[0, 1048575]]
 77 | 
 78 |   it('sliceLocation should match expected value.', () => {
 79 |     expect(sliceLocation).toEqual(expectedSliceLocation)
 80 |   })
 81 | 
 82 |   it('endLocation should match expected value.', () => {
 83 |     expect(endLocation).toBe(expectedEndLocation)
 84 |   })
 85 | })
 86 | 
 87 | describe('readFileAsArrayBuffer reads specified range of file into ArrayBuffer', () => {
 88 |   const filePath = path.join(__dirname, './../fixture/mockFile.txt')
 89 |   const start = 0
 90 |   let end: number
 91 |   let arrayBuffer: ArrayBuffer
 92 |   let expectedLength: number
 93 | 
 94 |   beforeAll(async () => {
 95 |     arrayBuffer = await readFileAsArrayBuffer(filePath, start, end)
 96 |     const stats = await fs.stat(filePath)
 97 |     end = stats.size
 98 |     expectedLength = end - start
 99 |   })
100 | 
101 |   // ArrayBuffer 的 byteLength 应该与请求的字节长度一致
102 |   it(`ArrayBuffer should be expectedLength bytes long`, () => {
103 |     expect(arrayBuffer.byteLength).toBe(expectedLength)
104 |   })
105 | })
106 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/node/getRootHashByChunks.spec.ts:
--------------------------------------------------------------------------------
 1 | import { getRootHashByChunks } from '../../src/getRootHashByChunks'
 2 | 
 3 | describe('getRootHashByChunks', () => {
 4 |   it('should return the root hash', async () => {
 5 |     const hashList = ['a']
 6 |     const result = await getRootHashByChunks(hashList)
 7 |     expect(result).toBe('a')
 8 |   })
 9 | })
10 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/node/is.spec.ts:
--------------------------------------------------------------------------------
 1 | import { isBrowser, isBrowser2, isEmpty, isNode } from '../../src/utils'
 2 | 
 3 | test('isEmpty correctly identifies empty values', () => {
 4 |   expect(isEmpty(undefined)).toBe(true)
 5 |   expect(isEmpty(null)).toBe(true)
 6 |   expect(isEmpty('')).toBe(true)
 7 |   expect(isEmpty([])).toBe(true)
 8 |   expect(isEmpty(new Map())).toBe(true)
 9 |   expect(isEmpty(new Set())).toBe(true)
10 |   expect(isEmpty({})).toBe(true)
11 | })
12 | 
13 | // 测试 isEmpty 函数对非空值的正确判断
14 | test('isEmpty correctly identifies non-empty values', () => {
15 |   expect(isEmpty('text')).toBe(false)
16 |   expect(isEmpty([1, 2, 3])).toBe(false)
17 |   expect(isEmpty(new Map([['key', 'value']]))).toBe(false)
18 |   expect(isEmpty(new Set([1, 2, 3]))).toBe(false)
19 |   expect(isEmpty({ key: 'value' })).toBe(false)
20 | })
21 | 
22 | // 测试环境检测函数
23 | test('Environment detection functions', () => {
24 |   // 例如，以下假设测试运行在 Node.js 环境中
25 |   expect(isBrowser()).toBe(false)
26 |   expect(isBrowser2()).toBe(false)
27 |   expect(isNode()).toBe(true)
28 | })
29 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/node/merkleTree.spec.ts:
--------------------------------------------------------------------------------
 1 | import { md5 } from 'hash-wasm'
 2 | import { MerkleTree } from '../../src/entity'
 3 | 
 4 | async function createHash(data: string): Promise<string> {
 5 |   return md5(data)
 6 | }
 7 | 
 8 | // 测试空数组初始化应该抛出错误
 9 | test('init() with empty array should throw an error', async () => {
10 |   const tree = new MerkleTree()
11 | 
12 |   // 如果期望异步函数抛出错误，可以使用 expect(...).rejects.toThrow(...)
13 |   await expect(tree.init([])).rejects.toThrow('Empty Nodes')
14 | })
15 | 
16 | // 测试 MerkleTree.init 用字符串哈希列表初始化
17 | test('MerkleTree.init with string hash list', async () => {
18 |   // 创建一些测试用的哈希
19 |   const hashList = [
20 |     await createHash('data1'),
21 |     await createHash('data2'),
22 |     await createHash('data3'),
23 |     await createHash('data4'),
24 |   ]
25 | 
26 |   // 创建 MerkleTree 实例并初始化
27 |   const merkleTree = new MerkleTree()
28 |   await expect(merkleTree.init(hashList)).resolves.toBeUndefined() // 如果不期望具体的返回值，可以检查被调用函数是否成功被解决
29 | 
30 |   // 检查是否所有叶子节点都被正确创建
31 |   expect(merkleTree.leafs.length).toBe(hashList.length)
32 |   hashList.forEach((hash, index) => {
33 |     expect(merkleTree.leafs[index].h).toBe(hash)
34 |   })
35 | 
36 |   // 检查 root 是否被设置
37 |   expect(merkleTree.root.h).toBeTruthy()
38 | })
39 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/node/miniSubject.spec.ts:
--------------------------------------------------------------------------------
 1 | import { MiniSubject } from '../../src/utils'
 2 | 
 3 | test('MiniSubject initializes with a value and can get the value', () => {
 4 |   const initial = 10
 5 |   const subject = new MiniSubject(initial)
 6 |   expect(subject.value).toBe(initial) // 使用 expect().toBe() 来断言值
 7 | })
 8 | 
 9 | // 测试 MiniSubject 能够订阅并立即接收初始值
10 | test('MiniSubject can subscribe and receive the initial value immediately', () => {
11 |   const initial = 'test'
12 |   const subject = new MiniSubject(initial)
13 |   const mockCallback = jest.fn() // 使用 jest 的模拟函数
14 | 
15 |   subject.subscribe(mockCallback)
16 |   expect(mockCallback).toHaveBeenCalledWith(initial) // 确认回调函数被立即以初始值调用
17 | })
18 | 
19 | // 测试 MiniSubject 在调用 next 后能够更新所有订阅者
20 | test('MiniSubject updates all subscribers on next', () => {
21 |   const subject = new MiniSubject(0)
22 |   // 创建一个 Jest 模拟函数, 能够记录它被调用的情况, 包括调用了多少次, 以及每次调用时传入的参数等信息
23 |   const mockCallback1 = jest.fn()
24 |   const mockCallback2 = jest.fn()
25 | 
26 |   subject.subscribe(mockCallback1)
27 |   subject.next(42) // 更新值
28 |   subject.subscribe(mockCallback2)
29 | 
30 |   expect(mockCallback1.mock.calls.length).toBe(2)
31 |   expect(mockCallback1.mock.calls[0][0]).toBe(0)
32 |   expect(mockCallback1.mock.calls[1][0]).toBe(42)
33 | 
34 |   expect(mockCallback2.mock.calls.length).toBe(1)
35 | })
36 | 
37 | // 测试 MiniSubject 取消订阅后不会更新回调函数
38 | test('MiniSubject does not update unsubscribed callbacks', () => {
39 |   const subject = new MiniSubject('initial')
40 |   const mockCallback = jest.fn()
41 | 
42 |   const id = subject.subscribe(mockCallback)
43 |   subject.unsubscribe(id)
44 |   subject.next('updated')
45 | 
46 |   // 回调函数只应被调用一次（订阅时）
47 |   expect(mockCallback.mock.calls.length).toBe(1)
48 | })
49 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/node/workerPool.spec.ts:
--------------------------------------------------------------------------------
 1 | import { MockWorkerPool } from '../fixture/mockWorkerPool'
 2 | 
 3 | describe('WorkerPool', () => {
 4 |   // 测试 WorkerPool 是否能够执行任务并返回结果
 5 |   test('should execute tasks and return results', async () => {
 6 |     const workerPool = new MockWorkerPool(2)
 7 |     const params = [new ArrayBuffer(8), new ArrayBuffer(8)]
 8 | 
 9 |     const getFn = (param: ArrayBuffer) => param
10 |     const restoreFn: any = () => {}
11 | 
12 |     const results = await workerPool.exec<string, ArrayBuffer>(params, getFn, restoreFn)
13 | 
14 |     // 使用 Jest 的 toEqual 进行深度比较
15 |     expect(results).toEqual(['result', 'result'])
16 |   })
17 | 
18 |   // 测试 WorkerPool 是否能够正确地终止所有 workers
19 |   test('should terminate all workers', () => {
20 |     const workerPool = new MockWorkerPool(2)
21 |     // 使用 jest.spyOn 为每个 worker 的 terminate 方法设置监视
22 |     const terminateSpies = workerPool.pool.map((worker) => jest.spyOn(worker, 'terminate'))
23 | 
24 |     workerPool.terminate()
25 | 
26 |     // 检查每个 spy 是否被调用了一次
27 |     terminateSpies.forEach((spy) => {
28 |       expect(spy).toHaveBeenCalledTimes(1)
29 |     })
30 |   })
31 | 
32 |   test('should execute tasks and return results', async () => {
33 |     const workerPool = new MockWorkerPool(2)
34 |     const params = [new ArrayBuffer(8), new ArrayBuffer(8)]
35 | 
36 |     const getFn = (param: ArrayBuffer) => param
37 |     const restoreFn: any = () => {}
38 | 
39 |     const results = await workerPool.exec<string, ArrayBuffer>(params, getFn, restoreFn)
40 | 
41 |     expect(results).toEqual(['result', 'result'])
42 |   })
43 | 
44 |   test('should terminate all workers', () => {
45 |     const workerPool = new MockWorkerPool(2)
46 | 
47 |     // 使用 Jest 的 mock 函数来模拟 terminate 方法
48 |     workerPool.pool.forEach((worker) => {
49 |       worker.terminate = jest.fn()
50 |     })
51 | 
52 |     workerPool.terminate()
53 | 
54 |     // 检查每个 worker 的 terminate 方法是否被调用了一次
55 |     workerPool.pool.forEach((worker) => {
56 |       expect(worker.terminate).toHaveBeenCalledTimes(1)
57 |     })
58 |   })
59 | })
60 | 


--------------------------------------------------------------------------------
/packages/core/__tests__/node/workerWrapper.spec.ts:
--------------------------------------------------------------------------------
 1 | import { WorkerWrapper } from '../../src/entity'
 2 | import { Worker as NodeWorker } from 'worker_threads'
 3 | 
 4 | // 模拟 Node.js 的 'worker_threads' 模块
 5 | jest.mock('worker_threads', () => {
 6 |   // 设置 NodeWorker 实例的模拟行为
 7 |   const mockPostMessage = jest.fn()
 8 |   const mockTerminate = jest.fn()
 9 |   const mockOnMessage = jest.fn().mockImplementation((event, handler) => {
10 |     if (event === 'message') {
11 |       setTimeout(() => {
12 |         handler({ result: 'hash-string', chunk: new ArrayBuffer(1) })
13 |       }, 0)
14 |     }
15 |   })
16 | 
17 |   return {
18 |     Worker: jest.fn().mockImplementation(() => ({
19 |       postMessage: mockPostMessage,
20 |       on: mockOnMessage,
21 |       terminate: mockTerminate,
22 |       setMaxListeners: jest.fn(),
23 |     })),
24 |   }
25 | })
26 | 
27 | describe('WorkerWrapper', () => {
28 |   it('should send and process messages correctly in node environment', async () => {
29 |     const NodeWorkerMock = NodeWorker as unknown as jest.Mock<NodeWorker>
30 | 
31 |     const worker = new NodeWorkerMock()
32 |     const workerWrapper = new WorkerWrapper(worker)
33 | 
34 |     const getFn = (param: ArrayBuffer) => param
35 |     const restoreFn: any = () => {}
36 | 
37 |     const res = await workerWrapper.run<string, ArrayBuffer>(
38 |       new ArrayBuffer(1),
39 |       0,
40 |       getFn,
41 |       restoreFn,
42 |     )
43 | 
44 |     expect(res).toBe('hash-string')
45 |     expect(worker.terminate).toHaveBeenCalledTimes(0) // 根据需要测试 terminate 被调用的次数
46 |   })
47 | })
48 | 


--------------------------------------------------------------------------------
/packages/core/jest.config.ts:
--------------------------------------------------------------------------------
 1 | import type { Config } from 'jest'
 2 | const projectsConfigWrapper = (configs: Record<string, any>[]): any[] =>
 3 |   configs.map((config) => ({
 4 |     ...config,
 5 |     coveragePathIgnorePatterns: ['/__tests__/fixture/'],
 6 |     preset: 'ts-jest',
 7 |     transform: {
 8 |       '^.+\\.tsx?$': [
 9 |         // 为了解决报错: The 'import.meta' meta-property is only allowed when the '--module' option is 'es2020', 'esnext', or 'system'.
10 |         // make ts-jest happy ^_^
11 |         'ts-jest',
12 |         {
13 |           diagnostics: {
14 |             ignoreCodes: [1343],
15 |           },
16 |           astTransformers: {
17 |             before: [
18 |               {
19 |                 path: 'ts-jest-mock-import-meta',
20 |                 // 此处 url 可随便符合 url 格式的字符串, 因为在运行 Jest 时, Worker 会被 Mock Worker 换掉
21 |                 options: { metaObjectReplacement: { url: 'https://test.com' } },
22 |               },
23 |             ],
24 |           },
25 |         },
26 |       ],
27 |     },
28 |   }))
29 | 
30 | const config: Config = {
31 |   collectCoverage: true,
32 |   coverageReporters: ['lcov'],
33 |   projects: projectsConfigWrapper([
34 |     {
35 |       displayName: 'node',
36 |       testEnvironment: 'node',
37 |       testMatch: ['**/__tests__/node/**/*.spec.ts'],
38 |     },
39 |     {
40 |       displayName: 'browser',
41 |       testEnvironment: 'jsdom',
42 |       testMatch: ['**/__tests__/browser/**/*.spec.ts'],
43 |     },
44 |   ]),
45 | }
46 | 
47 | export default config
48 | 


--------------------------------------------------------------------------------
/packages/core/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "hash-worker",
 3 |   "version": "1.0.1",
 4 |   "description": "hash-worker is a tool for quickly calculating file's hash",
 5 |   "author": "https://github.com/Tkunl",
 6 |   "repository": "https://github.com/Tkunl/hash-worker",
 7 |   "type": "module",
 8 |   "main": "./dist/index.js",
 9 |   "module": "./dist/index.mjs",
10 |   "types": "./dist/index.d.ts",
11 |   "unpkg": "./dist/global.js",
12 |   "jsdelivr": "./dist/global.js",
13 |   "exports": {
14 |     ".": {
15 |       "types": "./dist/index.d.ts",
16 |       "require": "./dist/index.js",
17 |       "import": "./dist/index.mjs"
18 |     },
19 |     "./global": {
20 |       "types": "./dist/global.d.ts",
21 |       "require": "./dist/global.js",
22 |       "import": "./dist/global.js"
23 |     }
24 |   },
25 |   "files": [
26 |     "dist",
27 |     "README.md",
28 |     "README-zh.md",
29 |     "package.json",
30 |     "LICENSE"
31 |   ],
32 |   "scripts": {
33 |     "dev": "rollup --config rollup.config.ts --configPlugin swc3 --watch",
34 |     "build": "pnpm rm:dist && rollup --config rollup.config.ts --configPlugin swc3",
35 |     "test": "jest --coverage",
36 |     "rm:dist": "rimraf ./dist"
37 |   },
38 |   "keywords": [
39 |     "hash-worker",
40 |     "hash"
41 |   ],
42 |   "license": "MIT",
43 |   "dependencies": {
44 |     "hash-wasm": "^4.12.0"
45 |   },
46 |   "devDependencies": {
47 |     "ts-node": "^10.9.2"
48 |   }
49 | }
50 | 


--------------------------------------------------------------------------------
/packages/core/rollup.config.ts:
--------------------------------------------------------------------------------
 1 | import { defineConfig } from 'rollup'
 2 | import { dts } from 'rollup-plugin-dts'
 3 | import { nodeResolve } from '@rollup/plugin-node-resolve'
 4 | import { minify, swc } from 'rollup-plugin-swc3'
 5 | 
 6 | const bundleName = 'HashWorker'
 7 | 
 8 | // 一般来说现代项目不需要自行压缩这些 cjs/esm 模块，因为现代构建工具会自动处理
 9 | // 其次发包发布压缩的包意义在于减少安装大小，但是实际上这个行为可有可无
10 | // 关于 iife/umd 面向现代的前端提供 iife 就可以了。
11 | // 因此你不需要过多复杂的配置。
12 | 
13 | export default defineConfig([
14 |   // esm 产物
15 |   {
16 |     input: 'src/main.ts',
17 |     output: [
18 |       { file: 'dist/index.mjs', format: 'esm', exports: 'named' },
19 |       { file: 'dist/index.js', format: 'cjs', exports: 'named' },
20 |     ],
21 |     plugins: [
22 |       nodeResolve(),
23 |       swc({ sourceMaps: true }),
24 |       minify({ mangle: true, module: true, compress: true, sourceMap: true }),
25 |     ],
26 |     external: ['worker_threads'],
27 |   },
28 |   // esm 类型产物
29 |   {
30 |     input: 'src/main.ts',
31 |     output: { file: 'dist/index.d.ts' },
32 |     plugins: [dts()],
33 |     external: ['worker_threads'],
34 |   },
35 |   // iife 产物
36 |   {
37 |     input: 'src/main.ts',
38 |     output: { file: 'dist/global.js', format: 'iife', name: bundleName },
39 |     plugins: [
40 |       nodeResolve(),
41 |       swc({ sourceMaps: true }),
42 |       minify({ mangle: true, module: false, compress: true, sourceMap: true }),
43 |     ],
44 |     external: ['worker_threads'],
45 |   },
46 |   // iife 类型产物
47 |   {
48 |     input: 'src/iife.ts',
49 |     output: { file: 'dist/global.d.ts', format: 'es' },
50 |     plugins: [dts()],
51 |     external: ['worker_threads'],
52 |   },
53 |   // Worker
54 |   {
55 |     input: 'src/worker/hash.worker.ts',
56 |     output: { file: 'dist/worker/hash.worker.mjs', format: 'esm' },
57 |     plugins: [
58 |       nodeResolve(),
59 |       swc({ sourceMaps: true }),
60 |       minify({ mangle: true, module: true, compress: true }),
61 |     ],
62 |   },
63 | ])
64 | 


--------------------------------------------------------------------------------
/packages/core/src/entity/index.ts:
--------------------------------------------------------------------------------
1 | export * from './merkleTree'
2 | export * from './workerPool'
3 | export * from './workerWrapper'
4 | 


--------------------------------------------------------------------------------
/packages/core/src/entity/merkleTree.ts:
--------------------------------------------------------------------------------
 1 | import { md5 } from 'hash-wasm'
 2 | 
 3 | // 定义 Merkle 树节点的接口
 4 | interface IMerkleNode {
 5 |   h: string
 6 |   l: IMerkleNode | null
 7 |   r: IMerkleNode | null
 8 | }
 9 | 
10 | // 定义 Merkle 树的接口
11 | interface IMerkleTree {
12 |   root: IMerkleNode
13 |   leafs: IMerkleNode[]
14 |   // 你可以根据需要添加其他属性或方法，例如校验、添加和生成树等功能
15 | }
16 | 
17 | // Merkle 树节点的类实现
18 | export class MerkleNode implements IMerkleNode {
19 |   h: string
20 |   l: IMerkleNode | null
21 |   r: IMerkleNode | null
22 | 
23 |   constructor(hash: string, left: IMerkleNode | null = null, right: IMerkleNode | null = null) {
24 |     this.h = hash
25 |     this.l = left
26 |     this.r = right
27 |   }
28 | }
29 | 
30 | // Merkle 树的类实现
31 | export class MerkleTree implements IMerkleTree {
32 |   root: IMerkleNode = new MerkleNode('')
33 |   leafs: IMerkleNode[] = []
34 | 
35 |   async init(hashList: string[]): Promise<void>
36 |   async init(leafNodes: IMerkleNode[]): Promise<void>
37 |   async init(nodes: string[] | IMerkleNode[]): Promise<void> {
38 |     if (nodes.length === 0) {
39 |       throw new Error('Empty Nodes')
40 |     }
41 |     if (typeof nodes[0] === 'string') {
42 |       this.leafs = nodes.map((node) => new MerkleNode(node as string))
43 |     } else {
44 |       this.leafs = nodes as IMerkleNode[]
45 |     }
46 |     this.root = await this.buildTree()
47 |   }
48 | 
49 |   getRootHash() {
50 |     return this.root.h
51 |   }
52 | 
53 |   async buildTree(): Promise<IMerkleNode> {
54 |     // 实现构建 Merkle 树的逻辑。根据叶子节点创建父节点，一直到根节点。
55 |     let currentLevelNodes = this.leafs
56 |     while (currentLevelNodes.length > 1) {
57 |       const parentNodes: IMerkleNode[] = []
58 |       for (let i = 0; i < currentLevelNodes.length; i += 2) {
59 |         const left = currentLevelNodes[i]
60 |         const right = i + 1 < currentLevelNodes.length ? currentLevelNodes[i + 1] : null
61 |         // 具体的哈希计算方法
62 |         const parentHash = await this.calculateHash(left, right)
63 |         parentNodes.push(new MerkleNode(parentHash, left, right))
64 |       }
65 |       currentLevelNodes = parentNodes
66 |     }
67 | 
68 |     return currentLevelNodes[0] // 返回根节点
69 |   }
70 | 
71 |   private async calculateHash(left: IMerkleNode, right: IMerkleNode | null): Promise<string> {
72 |     return right ? md5(left.h + right.h) : left.h
73 |   }
74 | }
75 | 


--------------------------------------------------------------------------------
/packages/core/src/entity/workerPool.ts:
--------------------------------------------------------------------------------
 1 | import { StatusEnum, WorkerWrapper } from './workerWrapper'
 2 | import { MiniSubject } from '../utils'
 3 | import { getFn, restoreFn } from '../interface'
 4 | 
 5 | export abstract class WorkerPool {
 6 |   pool: WorkerWrapper[] = []
 7 |   maxWorkerCount: number
 8 |   curRunningCount = new MiniSubject(0)
 9 |   results: any[] = []
10 | 
11 |   protected constructor(maxWorkers: number) {
12 |     this.maxWorkerCount = maxWorkers
13 |   }
14 | 
15 |   exec<T, U>(params: U[], getFn: getFn<U>, restoreFn: restoreFn) {
16 |     this.results.length = 0
17 |     const workerParams = params.map((param, index) => ({ data: param, index }))
18 | 
19 |     return new Promise<T[]>((rs) => {
20 |       this.curRunningCount.subscribe((count) => {
21 |         if (count < this.maxWorkerCount && workerParams.length !== 0) {
22 |           // 当前能跑的任务数量
23 |           let curTaskCount = this.maxWorkerCount - count
24 |           if (curTaskCount > workerParams.length) {
25 |             curTaskCount = workerParams.length
26 |           }
27 | 
28 |           // 此时可以用来执行任务的 Worker
29 |           const canUseWorker: WorkerWrapper[] = []
30 |           for (const worker of this.pool) {
31 |             if (worker.status === StatusEnum.WAITING) {
32 |               canUseWorker.push(worker)
33 |               if (canUseWorker.length === curTaskCount) {
34 |                 break
35 |               }
36 |             }
37 |           }
38 | 
39 |           const paramsToRun = workerParams.splice(0, curTaskCount)
40 | 
41 |           // 更新当前正在跑起来的 worker 数量
42 |           this.curRunningCount.next(this.curRunningCount.value + curTaskCount)
43 |           canUseWorker.forEach((workerApp, index) => {
44 |             const param = paramsToRun[index]
45 |             workerApp
46 |               .run<T, U>(param.data, param.index, getFn, restoreFn)
47 |               .then((res) => {
48 |                 this.results[param.index] = res
49 |               })
50 |               .catch((e) => {
51 |                 this.results[param.index] = e
52 |               })
53 |               .finally(() => {
54 |                 this.curRunningCount.next(this.curRunningCount.value - 1)
55 |               })
56 |           })
57 |         }
58 | 
59 |         if (this.curRunningCount.value === 0 && workerParams.length === 0) {
60 |           rs(this.results as T[])
61 |         }
62 |       })
63 |     })
64 |   }
65 | 
66 |   terminate() {
67 |     this.pool.forEach((workerWrapper) => workerWrapper.terminate())
68 |   }
69 | }
70 | 


--------------------------------------------------------------------------------
/packages/core/src/entity/workerWrapper.ts:
--------------------------------------------------------------------------------
 1 | import { Worker as NodeWorker } from 'worker_threads'
 2 | import { isBrowser, isNode } from '../utils'
 3 | import { getFn, restoreFn, WorkerRes } from '../interface'
 4 | 
 5 | type Resolve<T = any> = (value: T | PromiseLike<T>) => void
 6 | type Reject = (reason?: any) => void
 7 | 
 8 | export enum StatusEnum {
 9 |   RUNNING = 'running',
10 |   WAITING = 'waiting',
11 | }
12 | 
13 | export class WorkerWrapper {
14 |   worker: Worker | NodeWorker
15 |   status: StatusEnum
16 | 
17 |   constructor(worker: Worker | NodeWorker) {
18 |     this.worker = worker
19 |     this.status = StatusEnum.WAITING
20 |   }
21 | 
22 |   run<T, U>(param: U, index: number, getFn: getFn<U>, restoreFn: restoreFn) {
23 |     this.status = StatusEnum.RUNNING
24 | 
25 |     const onMessage = (rs: Resolve) => (dataFromWorker: unknown) => {
26 |       let data: WorkerRes<string>
27 | 
28 |       if (isBrowser()) {
29 |         data = (dataFromWorker as { data: WorkerRes }).data
30 |       }
31 |       if (isNode()) {
32 |         data = dataFromWorker as WorkerRes
33 |       }
34 |       const { result, chunk } = data!
35 |       if (result && chunk) {
36 |         restoreFn({
37 |           buf: chunk,
38 |           index,
39 |         })
40 |         this.status = StatusEnum.WAITING
41 |         rs(result as T)
42 |       }
43 |     }
44 | 
45 |     const onError = (rj: Reject) => (ev: ErrorEvent) => {
46 |       this.status = StatusEnum.WAITING
47 |       rj(ev)
48 |     }
49 | 
50 |     if (isBrowser()) {
51 |       const worker = this.worker as Worker
52 |       return new Promise<T>((rs, rj) => {
53 |         worker.onmessage = onMessage(rs)
54 |         worker.onerror = onError(rj)
55 |         worker.postMessage(param, [getFn(param)])
56 |       })
57 |     }
58 | 
59 |     if (isNode()) {
60 |       const worker = this.worker as NodeWorker
61 |       return new Promise<T>((rs, rj) => {
62 |         // 处理 MaxListenersExceededWarning: Possible EventEmitter memory leak detected 警告
63 |         worker.setMaxListeners(1024)
64 |         worker.on('message', onMessage(rs))
65 |         worker.on('error', onError(rj))
66 |         worker.postMessage(param, [getFn(param)])
67 |       })
68 |     }
69 | 
70 |     throw new Error('Unsupported environment')
71 |   }
72 | 
73 |   terminate() {
74 |     this.worker.terminate()
75 |   }
76 | }
77 | 


--------------------------------------------------------------------------------
/packages/core/src/getFileHashChunks.ts:
--------------------------------------------------------------------------------
 1 | import { getFileMetadata, isBrowser, isNode } from './utils'
 2 | import { WorkerService } from './worker/workerService'
 3 | import { HashChksParam, HashChksRes } from './interface'
 4 | import { normalizeParam, processFileInBrowser, processFileInNode } from './helper'
 5 | 
 6 | let workerService: WorkerService | null = null
 7 | let curWorkerCount: number = 0
 8 | 
 9 | /**
10 |  * 将文件进行分片, 并获取分片后的 hashList
11 |  * @param param
12 |  */
13 | async function getFileHashChunks(param: HashChksParam): Promise<HashChksRes> {
14 |   const { config, file, filePath } = normalizeParam(param)
15 |   const { isCloseWorkerImmediately, isShowLog, workerCount } = config
16 | 
17 |   if (workerService === null || curWorkerCount !== workerCount) {
18 |     destroyWorkerPool()
19 |     workerService = new WorkerService(config.workerCount)
20 |   }
21 | 
22 |   const metadata = await getFileMetadata(file, filePath)
23 | 
24 |   let chunksBlob: Blob[] = []
25 |   let chunksHash: string[] = []
26 |   let fileHash = ''
27 | 
28 |   let beforeTime: number = 0
29 |   let overTime: number = 0
30 | 
31 |   isShowLog && (beforeTime = Date.now())
32 | 
33 |   if (isBrowser() && file) {
34 |     const res = await processFileInBrowser(file, config, workerService)
35 |     chunksBlob = res.chunksBlob
36 |     chunksHash = res.chunksHash
37 |     fileHash = res.fileHash
38 |   }
39 | 
40 |   if (isNode() && filePath) {
41 |     const res = await processFileInNode(filePath, config, workerService)
42 |     chunksHash = res.chunksHash
43 |     fileHash = res.fileHash
44 |   }
45 | 
46 |   isShowLog && (overTime = Date.now() - beforeTime)
47 |   isShowLog &&
48 |     console.log(
49 |       `get file hash in: ${overTime} ms by using ${config.workerCount} worker, speed: ${metadata.size / 1024 / (overTime / 1000)} Mb/s`,
50 |     )
51 | 
52 |   const res: HashChksRes = {
53 |     chunksHash,
54 |     merkleHash: fileHash,
55 |     metadata,
56 |   }
57 | 
58 |   if (isBrowser()) {
59 |     res.chunksBlob = chunksBlob
60 |   }
61 | 
62 |   isCloseWorkerImmediately && destroyWorkerPool()
63 |   return res
64 | }
65 | 
66 | function destroyWorkerPool() {
67 |   workerService && workerService.terminate()
68 |   workerService = null
69 |   curWorkerCount = 0
70 | }
71 | 
72 | export { getFileHashChunks, destroyWorkerPool }
73 | 


--------------------------------------------------------------------------------
/packages/core/src/getRootHashByChunks.ts:
--------------------------------------------------------------------------------
1 | import { MerkleTree } from './entity'
2 | 
3 | export async function getRootHashByChunks(hashList: string[]) {
4 |   const merkleTree = new MerkleTree()
5 |   await merkleTree.init(hashList)
6 |   return merkleTree.getRootHash()
7 | }
8 | 


--------------------------------------------------------------------------------
/packages/core/src/helper.ts:
--------------------------------------------------------------------------------
  1 | import { crc32, md5, xxhash64 } from 'hash-wasm'
  2 | import { getRootHashByChunks } from './getRootHashByChunks'
  3 | import { Config, HashChksParam, Strategy } from './interface'
  4 | import {
  5 |   getArrayBufFromBlobs,
  6 |   getArrParts,
  7 |   getFileSliceLocations,
  8 |   isBrowser,
  9 |   isNode,
 10 |   readFileAsArrayBuffer,
 11 |   sliceFile,
 12 | } from './utils'
 13 | import { WorkerService } from './worker/workerService'
 14 | 
 15 | const DEFAULT_MAX_WORKERS = 8
 16 | const BORDER_COUNT = 100
 17 | 
 18 | /**
 19 |  * 标准化参数
 20 |  * @param param
 21 |  */
 22 | export function normalizeParam(param: HashChksParam) {
 23 |   const env: 'node' | 'browser' = (() => {
 24 |     if (isNode()) return 'node'
 25 |     if (isBrowser()) return 'browser'
 26 |     throw new Error('Unsupported environment')
 27 |   })()
 28 | 
 29 |   const { chunkSize, workerCount, strategy, borderCount, isCloseWorkerImmediately, isShowLog } =
 30 |     param.config ?? {}
 31 | 
 32 |   const config = {
 33 |     // 默认 10MB 分片大小
 34 |     chunkSize: chunkSize ?? 10,
 35 |     // 默认使用 8个 Worker 线程
 36 |     workerCount: workerCount ?? DEFAULT_MAX_WORKERS,
 37 |     // 默认使用混合模式计算 hash
 38 |     strategy: strategy ?? Strategy.mixed,
 39 |     // 默认以 100 分片数量作为边界
 40 |     borderCount: borderCount ?? BORDER_COUNT,
 41 |     // 默认计算 hash 后立即关闭 worker
 42 |     isCloseWorkerImmediately: isCloseWorkerImmediately ?? true,
 43 |     // 是否显示速度 log
 44 |     isShowLog: isShowLog ?? false,
 45 |   }
 46 | 
 47 |   if (env === 'node') {
 48 |     if (!param.filePath) {
 49 |       throw new Error('The filePath attribute is required in node environment')
 50 |     }
 51 |     return {
 52 |       ...param,
 53 |       config,
 54 |       filePath: param.filePath,
 55 |     }
 56 |   }
 57 | 
 58 |   if (env === 'browser') {
 59 |     if (!param.file) {
 60 |       throw new Error('The file attribute is required in browser environment')
 61 |     }
 62 |     return {
 63 |       ...param,
 64 |       config,
 65 |       file: param.file,
 66 |     }
 67 |   }
 68 | 
 69 |   throw new Error('Unsupported environment')
 70 | }
 71 | 
 72 | /**
 73 |  * 计算单个文件分片的 Hash
 74 |  * @param strategy hash 策略
 75 |  * @param arrayBuffer 文件分片的 arrayBuffer
 76 |  */
 77 | export async function getChunksHashSingle(strategy: Strategy, arrayBuffer: ArrayBuffer) {
 78 |   const unit8Array = new Uint8Array(arrayBuffer)
 79 |   const getHashStrategy = (strategy: Strategy) => {
 80 |     if (strategy === Strategy.md5) return md5
 81 |     if (strategy === Strategy.crc32) return crc32
 82 |     if (strategy === Strategy.xxHash64) return xxhash64
 83 |     throw Error('Unknown strategy')
 84 |   }
 85 | 
 86 |   return [await getHashStrategy(strategy === Strategy.mixed ? Strategy.md5 : strategy)(unit8Array)]
 87 | }
 88 | 
 89 | /**
 90 |  * 计算多个文件分片的 Hash
 91 |  * @param strategy hash 策略
 92 |  * @param arrayBuffers 当前文件分片的 arrayBuffer 数组 (组内)
 93 |  * @param chunksCount 文件的全部分片数量
 94 |  * @param borderCount Strategy.mixed 时的边界个数
 95 |  * @param workerSvc WorkerService
 96 |  */
 97 | export async function getChunksHashMultiple(
 98 |   strategy: Strategy,
 99 |   arrayBuffers: ArrayBuffer[],
100 |   chunksCount: number,
101 |   borderCount: number,
102 |   workerSvc: WorkerService,
103 | ) {
104 |   const processor = {
105 |     [Strategy.xxHash64]: () => workerSvc.getXxHash64ForFiles(arrayBuffers),
106 |     [Strategy.md5]: () => workerSvc.getMD5ForFiles(arrayBuffers),
107 |     [Strategy.crc32]: () => workerSvc.getCRC32ForFiles(arrayBuffers),
108 |     [Strategy.mixed]: () =>
109 |       chunksCount <= borderCount
110 |         ? workerSvc.getMD5ForFiles(arrayBuffers)
111 |         : workerSvc.getCRC32ForFiles(arrayBuffers),
112 |   }
113 | 
114 |   return processor[strategy]()
115 | }
116 | 
117 | export async function processFileInBrowser(
118 |   file: File,
119 |   config: Required<Config>,
120 |   workerSvc: WorkerService,
121 |   _getChunksHashSingle = getChunksHashSingle,
122 |   _getChunksHashMultiple = getChunksHashMultiple,
123 | ) {
124 |   const { chunkSize, strategy, workerCount, borderCount } = config
125 | 
126 |   // 文件分片
127 |   const chunksBlob = sliceFile(file, chunkSize)
128 |   let chunksHash: string[] = []
129 | 
130 |   const singleChunkProcessor = async () => {
131 |     const arrayBuffer = await chunksBlob[0].arrayBuffer()
132 |     chunksHash = await _getChunksHashSingle(strategy, arrayBuffer)
133 |   }
134 | 
135 |   const multipleChunksProcessor = async () => {
136 |     let chunksBuf: ArrayBuffer[] = []
137 |     // 将文件分片进行分组, 组内任务并行执行, 组外任务串行执行
138 |     const chunksPart = getArrParts<Blob>(chunksBlob, workerCount)
139 |     const tasks = chunksPart.map((part) => async () => {
140 |       // 手动释放上一次用于计算 Hash 的 ArrayBuffer
141 |       chunksBuf.length = 0
142 |       chunksBuf = await getArrayBufFromBlobs(part)
143 |       // 执行不同的 hash 计算策略
144 |       return _getChunksHashMultiple(strategy, chunksBuf, chunksBlob.length, borderCount, workerSvc)
145 |     })
146 | 
147 |     for (const task of tasks) {
148 |       const result = await task()
149 |       chunksHash.push(...result)
150 |     }
151 |     chunksBuf && (chunksBuf.length = 0)
152 |   }
153 | 
154 |   chunksBlob.length === 1 ? await singleChunkProcessor() : await multipleChunksProcessor()
155 |   const fileHash = await getRootHashByChunks(chunksHash)
156 | 
157 |   return {
158 |     chunksBlob,
159 |     chunksHash,
160 |     fileHash,
161 |   }
162 | }
163 | 
164 | export async function processFileInNode(
165 |   filePath: string,
166 |   config: Required<Config>,
167 |   workerSvc: WorkerService,
168 |   _getChunksHashSingle = getChunksHashSingle,
169 |   _getChunksHashMultiple = getChunksHashMultiple,
170 | ) {
171 |   const { chunkSize, strategy, workerCount, borderCount } = config
172 | 
173 |   // 文件分片
174 |   const { sliceLocation, endLocation } = await getFileSliceLocations(filePath, chunkSize)
175 |   let chunksHash: string[] = []
176 | 
177 |   const singleChunkProcessor = async () => {
178 |     const arrayBuffer = await readFileAsArrayBuffer(filePath, 0, endLocation)
179 |     chunksHash = await _getChunksHashSingle(strategy, arrayBuffer)
180 |   }
181 | 
182 |   const multipleChunksProcessor = async () => {
183 |     let chunksBuf: ArrayBuffer[] = []
184 |     // 分组后的起始分割位置
185 |     const sliceLocationPart = getArrParts<[number, number]>(sliceLocation, workerCount)
186 |     const tasks = sliceLocationPart.map((partArr) => async () => {
187 |       // 手动释放上一次用于计算 Hash 的 ArrayBuffer
188 |       chunksBuf.length = 0
189 |       chunksBuf = await Promise.all(
190 |         partArr.map((part) => readFileAsArrayBuffer(filePath, part[0], part[1])),
191 |       )
192 |       // 执行不同的 hash 计算策略
193 |       return _getChunksHashMultiple(
194 |         strategy,
195 |         chunksBuf,
196 |         sliceLocation.length,
197 |         borderCount,
198 |         workerSvc,
199 |       )
200 |     })
201 | 
202 |     for (const task of tasks) {
203 |       const result = await task()
204 |       chunksHash.push(...result)
205 |     }
206 |     chunksBuf.length = 0
207 |   }
208 | 
209 |   sliceLocation.length === 1 ? await singleChunkProcessor() : await multipleChunksProcessor()
210 |   const fileHash = await getRootHashByChunks(chunksHash)
211 | 
212 |   return {
213 |     chunksHash,
214 |     fileHash,
215 |   }
216 | }
217 | 


--------------------------------------------------------------------------------
/packages/core/src/iife.ts:
--------------------------------------------------------------------------------
1 | // 这个文件是为了让你的 iife 有一个 namespace 的类型提示
2 | import * as HashWorker from './main'
3 | 
4 | export default HashWorker
5 | 


--------------------------------------------------------------------------------
/packages/core/src/interface/fileHashChunks.ts:
--------------------------------------------------------------------------------
 1 | import { FileMetaInfo } from './fileMetaInfo'
 2 | import { Strategy } from './strategy'
 3 | 
 4 | export interface Config {
 5 |   chunkSize?: number // 分片大小 MB
 6 |   workerCount?: number // worker 线程数量
 7 |   strategy?: Strategy // hash 计算策略
 8 |   borderCount?: number // 使用 'mixed' 时的分界点, 分片数量少于 borderCount 时使用 md5 作为 hash 算法, 否则使用 crc32
 9 |   isCloseWorkerImmediately?: boolean // 是否在计算 hash 后立即关闭 worker
10 |   isShowLog?: boolean // 是否显示 log
11 | }
12 | 
13 | interface BaseParam {
14 |   config?: Config
15 | }
16 | 
17 | interface BrowserEnvParam extends BaseParam {
18 |   file: File // 待计算 Hash 的文件 (浏览器环境)
19 |   filePath?: never // 当 file 存在时，filePath 不能存在
20 | }
21 | 
22 | interface NodeEnvParam extends BaseParam {
23 |   file?: never // 当 filePath 存在时，file 不能存在
24 |   filePath: string // 待计算 Hash 的文件的 URL (Node 环境)
25 | }
26 | 
27 | // 使用交叉类型确保 file 和 filePath 二者之一必须存在
28 | export type HashChksParam = BrowserEnvParam | NodeEnvParam
29 | 
30 | export interface HashChksRes {
31 |   chunksBlob?: Blob[] // 文件分片的 Blob[]
32 |   chunksHash: string[] // 文件分片的 Hash[]
33 |   merkleHash: string // 文件的 merkleHash
34 |   metadata: FileMetaInfo // 文件的 metadata
35 | }
36 | 


--------------------------------------------------------------------------------
/packages/core/src/interface/fileMetaInfo.ts:
--------------------------------------------------------------------------------
1 | export interface FileMetaInfo {
2 |   name: string // 文件名
3 |   size: number // 文件大小 KB
4 |   lastModified: number // 时间戳
5 |   type: string // 文件的后缀名
6 | }
7 | 


--------------------------------------------------------------------------------
/packages/core/src/interface/fnTypes.ts:
--------------------------------------------------------------------------------
1 | interface restoreFnOption {
2 |   bufs?: ArrayBuffer[]
3 |   buf: ArrayBuffer
4 |   index: number
5 | }
6 | 
7 | export type getFn<T> = (param: T) => ArrayBuffer
8 | export type restoreFn = (options: restoreFnOption) => void
9 | 


--------------------------------------------------------------------------------
/packages/core/src/interface/index.ts:
--------------------------------------------------------------------------------
1 | export type * from './fileHashChunks'
2 | export type * from './fileMetaInfo'
3 | export type * from './workerRes'
4 | export type * from './workerReq'
5 | export type * from './fnTypes'
6 | 
7 | export * from './strategy'
8 | 


--------------------------------------------------------------------------------
/packages/core/src/interface/strategy.ts:
--------------------------------------------------------------------------------
1 | export enum Strategy {
2 |   md5 = 'md5',
3 |   crc32 = 'crc32',
4 |   xxHash64 = 'xxHash64',
5 |   mixed = 'mixed',
6 | }
7 | 


--------------------------------------------------------------------------------
/packages/core/src/interface/workerReq.ts:
--------------------------------------------------------------------------------
1 | import { Strategy } from './strategy'
2 | 
3 | export interface WorkerReq {
4 |   chunk: ArrayBuffer
5 |   strategy: Strategy
6 | }
7 | 


--------------------------------------------------------------------------------
/packages/core/src/interface/workerRes.ts:
--------------------------------------------------------------------------------
1 | export interface WorkerRes<T = any> {
2 |   result: T
3 |   chunk: ArrayBuffer
4 | }
5 | 


--------------------------------------------------------------------------------
/packages/core/src/main.ts:
--------------------------------------------------------------------------------
1 | export * from './entity'
2 | export * from './interface'
3 | export * from './utils'
4 | export * from './getFileHashChunks'
5 | export * from './getRootHashByChunks'
6 | 


--------------------------------------------------------------------------------
/packages/core/src/utils/arrayUtils.ts:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * [1, 2, 3, 4] => [[1, 2], [3, 4]]
 3 |  * @param chunks 原始数组
 4 |  * @param size 分 part 大小
 5 |  */
 6 | export function getArrParts<T>(chunks: any, size: number) {
 7 |   const result: T[][] = []
 8 |   let tempPart: T[] = []
 9 |   chunks.forEach((chunk: T) => {
10 |     tempPart.push(chunk)
11 |     if (tempPart.length === size) {
12 |       result.push(tempPart)
13 |       tempPart = []
14 |     }
15 |   })
16 |   if (tempPart.length !== 0) result.push(tempPart)
17 |   return result
18 | }
19 | 


--------------------------------------------------------------------------------
/packages/core/src/utils/fileUtils.ts:
--------------------------------------------------------------------------------
  1 | import { FileMetaInfo } from '../interface'
  2 | import { isBrowser, isNode } from './is'
  3 | 
  4 | /**
  5 |  * 分割文件
  6 |  * @param file
  7 |  * @param baseSize 默认分块大小为 1MB
  8 |  */
  9 | export function sliceFile(file: File, baseSize = 1) {
 10 |   if (baseSize <= 0) throw Error('baseSize must be greater than 0')
 11 |   const chunkSize = Math.max(1, baseSize * 1048576) // 1MB = 1024 * 1024
 12 |   const chunks: Blob[] = []
 13 |   let startPos = 0
 14 |   while (startPos < file.size) {
 15 |     chunks.push(file.slice(startPos, startPos + chunkSize))
 16 |     startPos += chunkSize
 17 |   }
 18 |   return chunks
 19 | }
 20 | 
 21 | /**
 22 |  * 分割文件, 获取每个分片的起止位置
 23 |  * @param filePath
 24 |  * @param baseSize 默认分块大小为 1MB
 25 |  */
 26 | export async function getFileSliceLocations(filePath: string, baseSize = 1) {
 27 |   if (baseSize <= 0) throw Error('baseSize must be greater than 0')
 28 |   const fsp = await import('fs/promises')
 29 |   const chunkSize = Math.max(1, baseSize * 1048576) // 1MB = 1024 * 1024
 30 |   const stats = await fsp.stat(filePath)
 31 |   const end = stats.size // Bytes 字节
 32 |   const sliceLocation: [number, number][] = []
 33 |   for (let cur = 0; cur < end; cur += chunkSize) {
 34 |     sliceLocation.push([cur, cur + chunkSize - 1])
 35 |   }
 36 |   return { sliceLocation, endLocation: end }
 37 | }
 38 | 
 39 | /**
 40 |  * 将 File 转成 ArrayBuffer
 41 |  * 注意: Blob 无法直接移交到 Worker 中, 所以需要放到主线程中执行
 42 |  * @param chunks
 43 |  */
 44 | export async function getArrayBufFromBlobs(chunks: Blob[]): Promise<ArrayBuffer[]> {
 45 |   return Promise.all(chunks.map((chunk) => chunk.arrayBuffer()))
 46 | }
 47 | 
 48 | /**
 49 |  * 读取一个文件并将它转成 ArrayBuffer
 50 |  * @param path 文件路径
 51 |  * @param start 起始位置(字节)
 52 |  * @param end 结束位置(字节)
 53 |  */
 54 | export async function readFileAsArrayBuffer(path: string, start: number, end: number) {
 55 |   const fs = await import('fs')
 56 |   const readStream = fs.createReadStream(path, { start, end })
 57 |   const chunks: any[] = []
 58 |   return new Promise<ArrayBuffer>((rs, rj) => {
 59 |     readStream.on('data', (chunk) => {
 60 |       chunks.push(chunk) // 收集数据块
 61 |     })
 62 | 
 63 |     readStream.on('end', () => {
 64 |       const buf = Buffer.concat(chunks) // 合并所有数据块构成 Buffer
 65 |       const arrayBuf = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength)
 66 |       rs(arrayBuf as ArrayBuffer)
 67 |     })
 68 | 
 69 |     readStream.on('error', (e) => {
 70 |       rj(e)
 71 |     })
 72 |   })
 73 | }
 74 | 
 75 | /**
 76 |  * 获取文件元数据
 77 |  * @param file 文件
 78 |  * @param filePath 文件路径
 79 |  */
 80 | export async function getFileMetadata(file?: File, filePath?: string): Promise<FileMetaInfo> {
 81 |   if (file && isBrowser()) {
 82 |     let fileType: string | undefined = ''
 83 | 
 84 |     if (file.name.includes('.')) {
 85 |       fileType = file.name.split('.').pop()
 86 |       fileType = fileType !== void 0 ? '.' + fileType : ''
 87 |     }
 88 | 
 89 |     return {
 90 |       name: file.name,
 91 |       size: file.size / 1024,
 92 |       lastModified: file.lastModified,
 93 |       type: fileType,
 94 |     }
 95 |   }
 96 | 
 97 |   if (filePath && isNode()) {
 98 |     const fsp: typeof import('node:fs/promises') = await import('fs/promises')
 99 |     const path: typeof import('node:path') = await import('path')
100 |     const stats = await fsp.stat(filePath)
101 |     return {
102 |       name: path.basename(filePath),
103 |       size: stats.size / 1024,
104 |       lastModified: stats.mtime.getTime(),
105 |       type: path.extname(filePath),
106 |     }
107 |   }
108 |   throw new Error('Unsupported environment')
109 | }
110 | 


--------------------------------------------------------------------------------
/packages/core/src/utils/index.ts:
--------------------------------------------------------------------------------
 1 | import * as arr from './arrayUtils'
 2 | import * as fileUtils from './fileUtils'
 3 | import * as is from './is'
 4 | import * as miniSubject from './miniSubject'
 5 | import * as rand from './rand'
 6 | 
 7 | /**
 8 |  * 兼容 import utils from './utils'; utils.someFn() 写法
 9 |  */
10 | export default {
11 |   ...rand,
12 |   ...arr,
13 |   ...miniSubject,
14 |   ...fileUtils,
15 |   ...is,
16 | }
17 | 
18 | /**
19 |  * 兼容 import { someFunctionFromRand } from './utils' 写法
20 |  */
21 | export * from './arrayUtils'
22 | export * from './fileUtils'
23 | export * from './is'
24 | export * from './miniSubject'
25 | export * from './rand'
26 | 


--------------------------------------------------------------------------------
/packages/core/src/utils/is.ts:
--------------------------------------------------------------------------------
 1 | export function isEmpty(value: any) {
 2 |   if (value === void 0 || value === null || value === '') {
 3 |     return true
 4 |   }
 5 |   if (Array.isArray(value)) {
 6 |     return value.length === 0
 7 |   }
 8 |   if (value instanceof Map || value instanceof Set) {
 9 |     return value.size === 0
10 |   }
11 |   if (typeof value === 'object' && !Array.isArray(value) && !(value instanceof Date)) {
12 |     return Object.keys(value).length === 0
13 |   }
14 |   return false
15 | }
16 | 
17 | /**
18 |  * 判断当前运行环境是否是浏览器
19 |  * @returns {boolean} 如果是在浏览器环境中运行，返回 true；否则返回 false
20 |  */
21 | export function isBrowser(): boolean {
22 |   return typeof window !== 'undefined' && typeof window.document !== 'undefined'
23 | }
24 | 
25 | /**
26 |  * 判断当前运行环境是否是浏览器(Worker 中)
27 |  * @returns {boolean} 如果是在浏览器环境中运行，返回 true；否则返回 false
28 |  */
29 | export function isBrowser2(): boolean {
30 |   return typeof self !== 'undefined' && typeof self.postMessage === 'function'
31 | }
32 | 
33 | /**
34 |  * 判断当前运行环境是否是 Node.js
35 |  * @returns {boolean} 如果是在 Node.js 环境中运行，返回 true；否则返回 false
36 |  */
37 | export function isNode(): boolean {
38 |   return (
39 |     typeof global !== 'undefined' &&
40 |     typeof process !== 'undefined' &&
41 |     typeof process.versions !== 'undefined' &&
42 |     typeof process.versions.node !== 'undefined'
43 |   )
44 | }
45 | 


--------------------------------------------------------------------------------
/packages/core/src/utils/miniSubject.ts:
--------------------------------------------------------------------------------
 1 | import { generateUUID } from './rand'
 2 | 
 3 | type Cb<T> = (value: T) => void
 4 | 
 5 | export class MiniSubject<T> {
 6 |   protected _value: T
 7 |   protected subscribers: Map<string, Cb<T>> = new Map()
 8 | 
 9 |   constructor(value: T) {
10 |     this._value = value
11 |   }
12 | 
13 |   get value() {
14 |     return this._value
15 |   }
16 | 
17 |   next(value: T) {
18 |     this._value = value
19 |     this.subscribers.forEach((cb) => cb(value))
20 |   }
21 | 
22 |   subscribe(cb: Cb<T>) {
23 |     const id = generateUUID()
24 |     this.subscribers.set(id, cb)
25 |     cb(this.value)
26 |     return id
27 |   }
28 | 
29 |   unsubscribe(id: string) {
30 |     this.subscribers.delete(id)
31 |   }
32 | }
33 | 


--------------------------------------------------------------------------------
/packages/core/src/utils/rand.ts:
--------------------------------------------------------------------------------
1 | export function generateUUID(): string {
2 |   return 'xxxxxxxxxxxx4xxxyxxxxxxxxxxxxxxx'.replace(/[xy]/g, function (c) {
3 |     const r = (Math.random() * 16) | 0
4 |     const v = c === 'x' ? r : (r & 0x3) | 0x8
5 |     return v.toString(16)
6 |   })
7 | }
8 | 


--------------------------------------------------------------------------------
/packages/core/src/worker/hash.worker.ts:
--------------------------------------------------------------------------------
 1 | /// <reference lib="webworker" />
 2 | 
 3 | import { crc32, md5, xxhash64 } from 'hash-wasm'
 4 | import { Strategy, WorkerReq } from '../interface'
 5 | import { isBrowser2, isNode } from '../utils'
 6 | 
 7 | async function calculateHash(req: WorkerReq) {
 8 |   const { chunk: buf, strategy } = req
 9 | 
10 |   let hash = ''
11 |   // const u8 = new Uint8Array(buf)
12 |   if (strategy === Strategy.md5) hash = await md5(new Uint8Array(buf))
13 |   if (strategy === Strategy.crc32) hash = await crc32(new Uint8Array(buf))
14 |   if (strategy === Strategy.xxHash64) hash = await xxhash64(new Uint8Array(buf))
15 | 
16 |   return {
17 |     result: hash,
18 |     chunk: req,
19 |   }
20 | }
21 | 
22 | if (isBrowser2()) {
23 |   addEventListener('message', async ({ data }: { data: WorkerReq }) => {
24 |     const res = await calculateHash(data)
25 |     postMessage(res, [data.chunk])
26 |   })
27 | }
28 | 
29 | if (isNode()) {
30 |   ;(async () => {
31 |     const { parentPort } = await import('worker_threads')
32 |     parentPort &&
33 |       parentPort.on('message', async (data: WorkerReq) => {
34 |         const res = await calculateHash(data)
35 |         parentPort.postMessage(res, [data.chunk])
36 |       })
37 |   })()
38 | }
39 | 


--------------------------------------------------------------------------------
/packages/core/src/worker/workerPoolForHash.ts:
--------------------------------------------------------------------------------
 1 | import { WorkerPool } from '../entity'
 2 | import { WorkerWrapper } from '../entity'
 3 | import { isBrowser, isNode } from '../utils'
 4 | 
 5 | export class WorkerPoolForHash extends WorkerPool {
 6 |   constructor(maxWorkers: number) {
 7 |     super(maxWorkers)
 8 |   }
 9 | 
10 |   static async create(maxWorkers: number) {
11 |     const instance = new WorkerPoolForHash(maxWorkers)
12 |     const countArr = Array.from({ length: maxWorkers })
13 | 
14 |     if (isBrowser()) {
15 |       instance.pool = countArr.map(() => {
16 |         return new WorkerWrapper(
17 |           new Worker(new URL('./worker/hash.worker.mjs', import.meta.url), { type: 'module' }),
18 |         )
19 |       })
20 |     }
21 | 
22 |     if (isNode()) {
23 |       const { Worker: NodeWorker } = await import('worker_threads')
24 |       instance.pool = countArr.map(() => {
25 |         return new WorkerWrapper(
26 |           new NodeWorker(new URL('./worker/hash.worker.mjs', import.meta.url)),
27 |         )
28 |       })
29 |     }
30 | 
31 |     return instance
32 |   }
33 | }
34 | 


--------------------------------------------------------------------------------
/packages/core/src/worker/workerService.ts:
--------------------------------------------------------------------------------
 1 | import { WorkerPoolForHash } from './workerPoolForHash'
 2 | import { getFn, restoreFn, Strategy, WorkerReq } from '../interface'
 3 | 
 4 | export class WorkerService {
 5 |   MAX_WORKERS
 6 |   pool: WorkerPoolForHash | undefined
 7 | 
 8 |   constructor(maxWorkers: number) {
 9 |     this.MAX_WORKERS = maxWorkers
10 |   }
11 | 
12 |   private async getHashForFiles(chunks: ArrayBuffer[], strategy: Strategy) {
13 |     if (this.pool === undefined) {
14 |       this.pool = await WorkerPoolForHash.create(this.MAX_WORKERS)
15 |     }
16 |     const params: WorkerReq[] = chunks.map((chunk) => ({
17 |       chunk,
18 |       strategy,
19 |     }))
20 | 
21 |     const getFn: getFn<WorkerReq> = (param: WorkerReq) => param.chunk
22 |     const restoreFn: restoreFn = (options) => {
23 |       const { index, buf } = options
24 |       chunks[index] = buf
25 |     }
26 | 
27 |     return this.pool.exec<string, WorkerReq>(params, getFn, restoreFn)
28 |   }
29 | 
30 |   getMD5ForFiles(chunks: ArrayBuffer[]) {
31 |     return this.getHashForFiles(chunks, Strategy.md5)
32 |   }
33 | 
34 |   getCRC32ForFiles(chunks: ArrayBuffer[]) {
35 |     return this.getHashForFiles(chunks, Strategy.crc32)
36 |   }
37 | 
38 |   getXxHash64ForFiles(chunks: ArrayBuffer[]) {
39 |     return this.getHashForFiles(chunks, Strategy.xxHash64)
40 |   }
41 | 
42 |   terminate() {
43 |     this.pool && this.pool.terminate()
44 |     this.pool = undefined
45 |   }
46 | }
47 | 


--------------------------------------------------------------------------------
/packages/core/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 |   "extends": "../../tsconfig.json"
3 | }
4 | 


--------------------------------------------------------------------------------
/packages/playground/benchmark-demo/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "benchmark",
 3 |   "private": true,
 4 |   "type": "module",
 5 |   "license": "MIT",
 6 |   "scripts": {
 7 |     "play": "node dist/index.js",
 8 |     "build:benchmark-demo": "pnpm rm:dist && tsup --config tsup.config.ts",
 9 |     "rm:dist": "rimraf ./dist"
10 |   },
11 |   "dependencies": {
12 |     "hash-worker": "workspace:*",
13 |     "hash-worker-benchmark": "workspace:*"
14 |   }
15 | }
16 | 


--------------------------------------------------------------------------------
/packages/playground/benchmark-demo/src/index.ts:
--------------------------------------------------------------------------------
 1 | import { benchmark, BenchmarkOptions } from 'hash-worker-benchmark'
 2 | import { Strategy } from 'hash-worker'
 3 | 
 4 | const options: BenchmarkOptions = {
 5 |   sizeInMB: 100,
 6 |   workerCountTobeTest: [8, 8, 8],
 7 |   strategy: Strategy.xxHash64,
 8 | }
 9 | 
10 | benchmark(options).then(() => {})
11 | 


--------------------------------------------------------------------------------
/packages/playground/benchmark-demo/tsconfig.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "extends": "../../../tsconfig.json",
 3 |   "compilerOptions": {
 4 |     "moduleResolution": "node",
 5 |     "module": "ESNext",
 6 |     "outDir": "./dist", // 生成的 JavaScript 文件输出目录
 7 |     "rootDir": "./src", // TypeScript 源文件目录
 8 |     "sourceMap": false, // 不生成 .map 文件
 9 |     "declaration": false // 不生成 .d.ts 文件
10 |   },
11 |   "include": ["src/**/*.ts"]
12 | }
13 | 


--------------------------------------------------------------------------------
/packages/playground/benchmark-demo/tsup.config.ts:
--------------------------------------------------------------------------------
1 | import { defineConfig } from 'tsup'
2 | 
3 | export default defineConfig({
4 |   entry: ['src/index.ts'],
5 |   format: ['esm'],
6 | })
7 | 


--------------------------------------------------------------------------------
/packages/playground/iife-demo/index.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="en">
 3 | 
 4 | <head>
 5 |   <meta charset="UTF-8" />
 6 |   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
 7 |   <title>IIFE Demo</title>
 8 | </head>
 9 | 
10 | <body>
11 |   <script src="./global.js"></script>
12 |   <script src="./worker/hash.worker.mjs"></script>
13 |   <script>
14 |     let file
15 | 
16 |     function handleInputChange(e) {
17 |       file = e.target.files[0]
18 |     }
19 | 
20 |     function handleGetHash() {
21 |       const param = {
22 |         file,
23 |         config: {
24 |           workerCount: 8,
25 |           strategy: 'md5',
26 |         },
27 |       }
28 | 
29 |       HashWorker.getFileHashChunks(param).then((data) => {
30 |         console.log(data)
31 |         alert('Calculation complete, please check the console!')
32 |       })
33 |     }
34 |   </script>
35 | 
36 |   <div>
37 |     <h2>Hello</h2>
38 |     <div>
39 |       <p>如果你在使用 Chrome 浏览器, 并且在控制台日志中发现了报错输出</p>
40 |       <p>这可能是因为你开启了 Vue.js devtools 或 React Developer Tools 插件</p>
41 |       <p>关闭它们, 错误就会消失.</p>
42 |     </div>
43 |     <input type="file" onchange="handleInputChange(event)" />
44 |     <button onclick="handleGetHash()">get Hash</button>
45 |   </div>
46 | 
47 | </body>
48 | 
49 | </html>


--------------------------------------------------------------------------------
/packages/playground/iife-demo/package.json:
--------------------------------------------------------------------------------
1 | {
2 |   "name": "browser-demo",
3 |   "private": true,
4 |   "license": "MIT",
5 |   "scripts": {
6 |     "play": "node prepare.js && node server.js"
7 |   }
8 | }
9 | 


--------------------------------------------------------------------------------
/packages/playground/iife-demo/prepare.js:
--------------------------------------------------------------------------------
 1 | const path = require('path')
 2 | const { copyFiles } = require('../../../scripts/fileCopier')
 3 | 
 4 | const rootDir = process.cwd()
 5 | 
 6 | // 定义要复制的文件列表
 7 | const filesToCopy = [
 8 |   {
 9 |     src: path.resolve(rootDir, '../../core/dist/global.js'),
10 |     dest: path.resolve(rootDir, 'global.js'),
11 |   },
12 |   {
13 |     src: path.resolve(rootDir, '../../core/dist/worker/hash.worker.mjs'),
14 |     dest: path.resolve(rootDir, './worker/hash.worker.mjs'),
15 |   },
16 | ]
17 | 
18 | // 执行文件复制
19 | copyFiles(filesToCopy)
20 | 


--------------------------------------------------------------------------------
/packages/playground/iife-demo/server.js:
--------------------------------------------------------------------------------
 1 | const http = require('http')
 2 | const fs = require('fs')
 3 | const path = require('path')
 4 | 
 5 | const server = http.createServer((req, res) => {
 6 |   // 获取请求的文件路径
 7 |   let filePath = '.' + req.url
 8 |   if (filePath === './') {
 9 |     filePath = './index.html'
10 |   }
11 | 
12 |   // 获取文件扩展名
13 |   const extname = String(path.extname(filePath)).toLowerCase()
14 |   const mimeTypes = {
15 |     '.html': 'text/html',
16 |     '.js': 'application/javascript',
17 |     '.mjs': 'application/javascript',
18 |     '.css': 'text/css',
19 |     '.png': 'image/png',
20 |     '.jpg': 'image/jpg',
21 |     '.json': 'application/json',
22 |   }
23 | 
24 |   const contentType = mimeTypes[extname] || 'application/octet-stream'
25 | 
26 |   // 读取文件
27 |   fs.readFile(filePath, (error, content) => {
28 |     if (error) {
29 |       if (error.code === 'ENOENT') {
30 |         // 如果文件不存在，返回 404 页面
31 |         fs.readFile('./404.html', (error404, content404) => {
32 |           res.writeHead(404, { 'Content-Type': 'text/html' })
33 |           res.end(content404, 'utf-8')
34 |         })
35 |       } else {
36 |         // 其他错误，返回 500 页面
37 |         res.writeHead(500)
38 |         res.end(`Server Error: ${error.code}`)
39 |       }
40 |     } else {
41 |       // 成功读取文件，返回内容
42 |       res.writeHead(200, { 'Content-Type': contentType })
43 |       res.end(content, 'utf-8')
44 |     }
45 |   })
46 | })
47 | 
48 | const PORT = process.env.PORT || 8891
49 | server.listen(PORT, () => {
50 |   console.log(`Server running at http://localhost:${PORT}/`)
51 | })
52 | 


--------------------------------------------------------------------------------
/packages/playground/node-demo/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "node-demo",
 3 |   "private": true,
 4 |   "type": "module",
 5 |   "license": "MIT",
 6 |   "scripts": {
 7 |     "play": "node dist/index.js",
 8 |     "build:node-demo": "pnpm rm:dist && tsup --config tsup.config.ts",
 9 |     "rm:dist": "rimraf ./dist"
10 |   },
11 |   "dependencies": {
12 |     "hash-worker": "workspace:*",
13 |     "hash-worker-benchmark": "workspace:*"
14 |   }
15 | }
16 | 


--------------------------------------------------------------------------------
/packages/playground/node-demo/src/index.ts:
--------------------------------------------------------------------------------
 1 | import { getFileHashChunks, HashChksParam, Strategy } from 'hash-worker'
 2 | 
 3 | const param: HashChksParam = {
 4 |   filePath: 'filePath...',
 5 |   config: {
 6 |     strategy: Strategy.md5,
 7 |     workerCount: 8,
 8 |     isShowLog: true,
 9 |   },
10 | }
11 | 
12 | const beforeDate = Date.now()
13 | function main() {
14 |   getFileHashChunks(param).then((res: any) => {
15 |     console.log(res)
16 |   })
17 | }
18 | 
19 | main()
20 | 


--------------------------------------------------------------------------------
/packages/playground/node-demo/tsconfig.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "extends": "../../../tsconfig.json",
 3 |   "compilerOptions": {
 4 |     "moduleResolution": "node",
 5 |     "module": "ESNext",
 6 |     "outDir": "./dist", // 生成的 JavaScript 文件输出目录
 7 |     "rootDir": "./src", // TypeScript 源文件目录
 8 |     "sourceMap": false, // 不生成 .map 文件
 9 |     "declaration": false // 不生成 .d.ts 文件
10 |   },
11 |   "include": ["src/**/*.ts"]
12 | }
13 | 


--------------------------------------------------------------------------------
/packages/playground/node-demo/tsup.config.ts:
--------------------------------------------------------------------------------
1 | import { defineConfig } from 'tsup'
2 | 
3 | export default defineConfig({
4 |   entry: ['src/index.ts'],
5 |   format: ['esm'],
6 | })
7 | 


--------------------------------------------------------------------------------
/packages/playground/react-webpack-demo/.babelrc:
--------------------------------------------------------------------------------
1 | {
2 |   "presets": ["@babel/preset-env", "@babel/preset-react", "@babel/preset-typescript"]
3 | }
4 | 


--------------------------------------------------------------------------------
/packages/playground/react-webpack-demo/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "react-webpack-demo",
 3 |   "private": true,
 4 |   "license": "MIT",
 5 |   "scripts": {
 6 |     "play": "webpack serve --mode development"
 7 |   },
 8 |   "dependencies": {
 9 |     "react": "^18.2.0",
10 |     "react-dom": "^18.2.0",
11 |     "hash-worker": "workspace:*"
12 |   },
13 |   "devDependencies": {
14 |     "@babel/core": "^7.26.0",
15 |     "@babel/preset-env": "^7.26.0",
16 |     "@babel/preset-react": "^7.26.3",
17 |     "@babel/preset-typescript": "^7.26.0",
18 |     "@types/react": "^19.0.3",
19 |     "@types/react-dom": "^19.0.2",
20 |     "babel-loader": "^9.2.1",
21 |     "typescript": "^5.7.2",
22 |     "webpack": "^5.97.1",
23 |     "webpack-cli": "^6.0.1",
24 |     "webpack-dev-server": "^5.2.0"
25 |   }
26 | }
27 | 


--------------------------------------------------------------------------------
/packages/playground/react-webpack-demo/public/index.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="en">
 3 | <head>
 4 |   <meta charset="UTF-8" />
 5 |   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
 6 |   <title>React Webpack Demo</title>
 7 | </head>
 8 | <body>
 9 | <div id="app"></div>
10 | <script src="bundle.js"></script>
11 | </body>
12 | </html>
13 | 


--------------------------------------------------------------------------------
/packages/playground/react-webpack-demo/src/index.tsx:
--------------------------------------------------------------------------------
 1 | import React from 'react'
 2 | import ReactDOM from 'react-dom/client'
 3 | import { getFileHashChunks, HashChksRes, HashChksParam } from 'hash-worker'
 4 | 
 5 | function App() {
 6 |   let file: File
 7 | 
 8 |   function handleInputChange(e: any) {
 9 |     file = e.target.files[0]
10 |   }
11 | 
12 |   function handleGetHash() {
13 |     const param: HashChksParam = {
14 |       file: file!,
15 |       config: {
16 |         workerCount: 8,
17 |       },
18 |     }
19 | 
20 |     getFileHashChunks(param).then((data: HashChksRes) => {
21 |       console.log(data)
22 |       alert('Calculation complete, please check the console!')
23 |     })
24 |   }
25 | 
26 |   return (
27 |     <>
28 |       <div>Hello</div>
29 |       <input type="file" onChange={handleInputChange} />
30 |       <button onClick={handleGetHash}>get Hash</button>
31 |     </>
32 |   )
33 | }
34 | 
35 | ReactDOM.createRoot(document.querySelector('#app')!).render(
36 |   <React.StrictMode>
37 |     <App />
38 |   </React.StrictMode>,
39 | )
40 | 


--------------------------------------------------------------------------------
/packages/playground/react-webpack-demo/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 |   "extends": "../../../tsconfig.json",
3 |   "compilerOptions": {
4 |     "jsx": "react",
5 |     "types": ["react", "react-dom"]
6 |   }
7 | }
8 | 


--------------------------------------------------------------------------------
/packages/playground/react-webpack-demo/webpack.config.js:
--------------------------------------------------------------------------------
 1 | const path = require('path')
 2 | 
 3 | module.exports = {
 4 |   entry: './src/index.tsx',
 5 |   output: {
 6 |     filename: 'bundle.js',
 7 |     path: path.resolve(__dirname, 'dist'),
 8 |     publicPath: '/',
 9 |   },
10 |   resolve: {
11 |     extensions: ['.ts', '.tsx', '.js'],
12 |     fallback: {
13 |       fs: false,
14 |       path: false,
15 |       'fs/promises': false,
16 |       worker_threads: false,
17 |     },
18 |   },
19 |   module: {
20 |     rules: [
21 |       {
22 |         test: /\.(ts|tsx)$/,
23 |         use: 'babel-loader',
24 |         exclude: /node_modules/,
25 |       },
26 |     ],
27 |   },
28 |   externals: {
29 |     fs: 'commonjs fs',
30 |     path: 'commonjs path',
31 |     'fs/promises': 'commonjs fs/promises',
32 |     worker_threads: 'commonjs worker_threads',
33 |   },
34 |   devServer: {
35 |     static: {
36 |       directory: path.join(__dirname, 'public'),
37 |     },
38 |     compress: true,
39 |     port: 8890,
40 |     historyApiFallback: true,
41 |   },
42 | }
43 | 


--------------------------------------------------------------------------------
/packages/playground/vue-vite-demo/index.html:
--------------------------------------------------------------------------------
 1 | <!doctype html>
 2 | <html lang="en">
 3 | <head>
 4 |   <meta charset="UTF-8" />
 5 |   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
 6 |   <title>Vue Vite Demo</title>
 7 | </head>
 8 | <body>
 9 | <div id="app"></div>
10 | <script type="module" src="/src/main.ts"></script>
11 | </body>
12 | </html>
13 | 


--------------------------------------------------------------------------------
/packages/playground/vue-vite-demo/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "vue-demo",
 3 |   "private": true,
 4 |   "type": "module",
 5 |   "license": "MIT",
 6 |   "scripts": {
 7 |     "play": "vite --port 8888"
 8 |   },
 9 |   "dependencies": {
10 |     "hash-worker": "workspace:*",
11 |     "hash-worker-benchmark": "workspace:*",
12 |     "vue": "^3.5.13"
13 |   },
14 |   "devDependencies": {
15 |     "@vitejs/plugin-vue": "^5.2.1",
16 |     "vite": "^6.0.7"
17 |   }
18 | }
19 | 


--------------------------------------------------------------------------------
/packages/playground/vue-vite-demo/src/App.vue:
--------------------------------------------------------------------------------
 1 | <script lang="ts" setup>
 2 | import { useFileHashInfo } from './hooks/useFileHashInfo'
 3 | import { benchmark, BenchmarkOptions } from 'hash-worker-benchmark'
 4 | import { Strategy } from 'hash-worker'
 5 | 
 6 | const { handleInputChange, handleGetHash, handleDestroyWorkerPool } = useFileHashInfo()
 7 | 
 8 | function handleDoBenchmark() {
 9 |   const options: BenchmarkOptions = {
10 |     strategy: Strategy.md5
11 |   }
12 |   benchmark(options).then(() => {
13 |   })
14 | }
15 | 
16 | </script>
17 | 
18 | <template>
19 |   <div>Hello</div>
20 |   <input type="file" @change="handleInputChange" />
21 | 
22 |   <div>
23 |     <button @click="handleGetHash">get Hash</button>
24 |     <button @click="handleDestroyWorkerPool">destroy pool</button>
25 |     <button @click="handleDoBenchmark">Benchmark</button>
26 |   </div>
27 | </template>
28 | 


--------------------------------------------------------------------------------
/packages/playground/vue-vite-demo/src/hooks/useFileHashInfo.ts:
--------------------------------------------------------------------------------
 1 | import { ref } from 'vue'
 2 | import {
 3 |   destroyWorkerPool,
 4 |   getFileHashChunks,
 5 |   HashChksParam,
 6 |   HashChksRes,
 7 |   Strategy,
 8 | } from 'hash-worker'
 9 | 
10 | export function useFileHashInfo() {
11 |   const file = ref<File>()
12 | 
13 |   function handleInputChange(e: any) {
14 |     file.value = e.target.files[0]
15 |   }
16 | 
17 |   function handleGetHash() {
18 |     const param: HashChksParam = {
19 |       file: file.value!,
20 |       config: {
21 |         workerCount: 8,
22 |         strategy: Strategy.md5,
23 |       },
24 |     }
25 | 
26 |     getFileHashChunks(param).then((res: HashChksRes) => {
27 |       console.log(res)
28 |       alert('Calculation complete, please check the console!')
29 |     })
30 |   }
31 | 
32 |   function handleDestroyWorkerPool() {
33 |     destroyWorkerPool()
34 |   }
35 | 
36 |   return {
37 |     handleInputChange,
38 |     handleGetHash,
39 |     handleDestroyWorkerPool,
40 |   }
41 | }
42 | 


--------------------------------------------------------------------------------
/packages/playground/vue-vite-demo/src/main.ts:
--------------------------------------------------------------------------------
1 | import { createApp } from 'vue'
2 | import App from './App.vue'
3 | 
4 | createApp(App).mount('#app')
5 | 


--------------------------------------------------------------------------------
/packages/playground/vue-vite-demo/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 |   "extends": "../../../tsconfig.json",
3 |   "compilerOptions": {
4 |     "types": ["vite/client"]
5 |   }
6 | }
7 | 


--------------------------------------------------------------------------------
/packages/playground/vue-vite-demo/vite.config.ts:
--------------------------------------------------------------------------------
 1 | import { defineConfig } from 'vite'
 2 | import vue from '@vitejs/plugin-vue'
 3 | 
 4 | export default defineConfig({
 5 |   plugins: [vue()],
 6 |   optimizeDeps: {
 7 |     exclude: ['hash-worker'],
 8 |   },
 9 | })
10 | 


--------------------------------------------------------------------------------
/pnpm-workspace.yaml:
--------------------------------------------------------------------------------
1 | packages:
2 |   - "packages/core"
3 |   - "packages/benchmark"
4 |   - "packages/playground/*"
5 | 


--------------------------------------------------------------------------------
/scripts/clear.js:
--------------------------------------------------------------------------------
 1 | const rimraf = require('rimraf')
 2 | const path = require('path')
 3 | 
 4 | const nodeModulesDir = [
 5 |   '',
 6 |   'packages/benchmark/',
 7 |   'packages/core/',
 8 |   'packages/playground/benchmark-demo/',
 9 |   'packages/playground/node-demo/',
10 |   'packages/playground/react-webpack-demo/',
11 |   'packages/playground/vue-vite-demo/',
12 | ].map((dir) => dir + 'node_modules')
13 | 
14 | const distToBeBundled = [
15 |   'packages/benchmark/',
16 |   'packages/core/',
17 |   'packages/playground/benchmark-demo/',
18 |   'packages/playground/node-demo/',
19 | ]
20 | 
21 | const distDir = distToBeBundled.map((dir) => dir + 'dist')
22 | const turboCacheDir = distToBeBundled.map((dir) => dir + '.turbo')
23 | const iifeDemoDeps = [
24 |   'packages/playground/iife-demo/global.js',
25 |   'packages/playground/iife-demo/worker',
26 | ]
27 | 
28 | const coverageDir = ['packages/core/coverage']
29 | 
30 | // 主函数来删除所有路径，并处理错误
31 | function removePaths(paths) {
32 |   paths.forEach((path) => {
33 |     try {
34 |       rimraf.sync(path)
35 |       console.log(`Successfully deleted: ${path}`)
36 |     } catch (err) {
37 |       console.error(`Failed to delete: ${path}, Error: ${err.message}`)
38 |     }
39 |   })
40 | 
41 |   console.log('All deletion attempts have been processed.')
42 | }
43 | 
44 | function processArgs() {
45 |   const args = process.argv.slice(2)
46 |   let pattern = ''
47 | 
48 |   args.forEach((arg) => {
49 |     const [key, value] = arg.split('=')
50 |     if (key === '--pattern') {
51 |       pattern = value
52 |     }
53 |   })
54 |   return pattern
55 | }
56 | 
57 | ;(() => {
58 |   const startTime = Date.now() // 记录开始时间
59 |   const pattern = processArgs() // 获取执行参数
60 | 
61 |   // 定义目录映射
62 |   const dirMap = {
63 |     node_modules: nodeModulesDir,
64 |     dist: distDir,
65 |     cache: turboCacheDir,
66 |     coverage: coverageDir,
67 |   }
68 | 
69 |   let pathsToDelete = []
70 | 
71 |   if (pattern === 'all') {
72 |     pathsToDelete = [
73 |       ...nodeModulesDir,
74 |       ...distDir,
75 |       ...turboCacheDir,
76 |       ...iifeDemoDeps,
77 |       ...coverageDir,
78 |     ]
79 |   } else if (dirMap[pattern]) {
80 |     pathsToDelete = dirMap[pattern]
81 |   }
82 | 
83 |   // 解析路径并删除
84 |   pathsToDelete = pathsToDelete.map((p) => path.resolve(process.cwd(), p))
85 | 
86 |   removePaths(pathsToDelete)
87 | 
88 |   const endTime = Date.now() // 记录结束时间
89 |   const timeTaken = endTime - startTime // 计算总耗时
90 |   console.log(`Total time taken: ${timeTaken}ms`)
91 | })()
92 | 


--------------------------------------------------------------------------------
/scripts/fileCopier.js:
--------------------------------------------------------------------------------
 1 | const fs = require('fs')
 2 | const path = require('path')
 3 | 
 4 | /**
 5 |  * 复制文件的函数
 6 |  * @param {string} src - 源文件路径
 7 |  * @param {string} dest - 目标文件路径
 8 |  */
 9 | function copyFile(src, dest) {
10 |   // 如果目标文件已存在，则先删除
11 |   if (fs.existsSync(dest)) {
12 |     try {
13 |       fs.unlinkSync(dest)
14 |       console.log(`Deleted existing file at ${dest}`)
15 |     } catch (err) {
16 |       console.error(`Error deleting file at ${dest}:`, err)
17 |       return
18 |     }
19 |   }
20 | 
21 |   const readStream = fs.createReadStream(src)
22 |   const writeStream = fs.createWriteStream(dest)
23 | 
24 |   readStream.on('error', (err) => {
25 |     console.error(`Error reading file from ${src}:`, err)
26 |   })
27 | 
28 |   writeStream.on('error', (err) => {
29 |     console.error(`Error writing file to ${dest}:`, err)
30 |   })
31 | 
32 |   writeStream.on('finish', () => {
33 |     console.log(`Successfully copied ${src} to ${dest}`)
34 |   })
35 | 
36 |   readStream.pipe(writeStream)
37 | }
38 | 
39 | /**
40 |  * 复制多个文件的函数
41 |  * @param {Array<{src: string, dest: string}>} files - 包含源文件和目标文件路径的对象数组
42 |  */
43 | function copyFiles(files) {
44 |   files.forEach(({ src, dest }) => {
45 |     // 确保目标目录存在
46 |     const destDir = path.dirname(dest)
47 |     if (!fs.existsSync(destDir)) {
48 |       fs.mkdirSync(destDir, { recursive: true })
49 |     }
50 | 
51 |     copyFile(src, dest)
52 |   })
53 | }
54 | 
55 | // 使用 module.exports 导出 copyFiles 函数
56 | module.exports = { copyFiles }
57 | 


--------------------------------------------------------------------------------
/scripts/syncReadme.js:
--------------------------------------------------------------------------------
 1 | const path = require('path')
 2 | const { copyFiles } = require('./fileCopier')
 3 | 
 4 | // 获取项目的根目录路径
 5 | const rootDir = process.cwd()
 6 | 
 7 | // 定义要复制的文件列表
 8 | const filesToCopy = [
 9 |   {
10 |     src: path.resolve(rootDir, 'README.md'),
11 |     dest: path.resolve(rootDir, 'packages/core/README.md'),
12 |   },
13 |   {
14 |     src: path.resolve(rootDir, 'README-zh.md'),
15 |     dest: path.resolve(rootDir, 'packages/core/README-zh.md'),
16 |   },
17 | ]
18 | 
19 | // 执行文件复制
20 | copyFiles(filesToCopy)
21 | 


--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "compilerOptions": {
 3 |     "target": "es2022",
 4 |     "module": "ESNext",
 5 |     "sourceMap": true,
 6 |     "declaration": true, // 是否包含 d.ts 文件
 7 |     "strict": true, // Ts 的严格模式
 8 |     "esModuleInterop": true, // 可以使用 es6 的导入语法导入 cjs 模块
 9 |     "forceConsistentCasingInFileNames": true, // 确保文件大小写一致来确保引用的一致性
10 |     "skipLibCheck": true, // 跳过第三方库的 d.ts 类型文件检查
11 |     "lib": ["esnext", "dom"], // 指定编译过程中需要包括的库文件列表, 这些库文件是声明 js 运行时和 dom 中可用的全局变量的类型
12 |     "moduleResolution": "Bundler" // 定义依赖解析策略, 默认是 node 模块解析策略
13 |   }
14 | }
15 | 


--------------------------------------------------------------------------------
/turbo.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "$schema": "https://turbo.build/schema.json",
 3 |   "tasks": {
 4 |     "build": {
 5 |       "dependsOn": ["^build"],
 6 |       "outputs": ["dist/**"],
 7 |       "outputLogs": "new-only"
 8 |     },
 9 |     "build:benchmark": {
10 |       "dependsOn": ["^build", "^build:benchmark"],
11 |       "outputs": ["dist/**"],
12 |       "outputLogs": "new-only"
13 |     },
14 |     "build:benchmark-demo": {
15 |       "dependsOn": ["^build:benchmark", "^build:benchmark-demo"],
16 |       "outputs": ["dist/**"],
17 |       "outputLogs": "new-only"
18 |     },
19 |     "build:node-demo": {
20 |       "dependsOn": ["^build", "^build:node-demo"],
21 |       "outputs": ["dist/**"],
22 |       "outputLogs": "new-only"
23 |     },
24 |     "build:all": {
25 |       "dependsOn": ["build", "build:benchmark", "build:benchmark-demo", "build:node-demo"],
26 |       "outputs": ["dist/**"],
27 |       "outputLogs": "new-only"
28 |     }
29 |   }
30 | }
31 | 


--------------------------------------------------------------------------------