├── .editorconfig ├── .github └── workflows │ ├── nodejs.yml │ └── publish.yml ├── .gitignore ├── .swcrc ├── .yarn └── releases │ └── yarn-4.5.1.cjs ├── .yarnrc.yml ├── CHANGELOG.md ├── Dockerfile ├── LICENSE.md ├── README.md ├── bench ├── index.js ├── package.json └── yarn.lock ├── eslint.config.mjs ├── jest.config.js ├── package.json ├── rollup.config.js ├── src ├── index.js ├── index.workerd.js ├── xxhash.js ├── xxhash.wasm └── xxhash.wat ├── test ├── index.test.js └── setup.js ├── types.d.ts └── yarn.lock /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | charset = utf-8 6 | trim_trailing_whitespace = true 7 | insert_final_newline = true 8 | indent_style = space 9 | indent_size = 2 10 | -------------------------------------------------------------------------------- /.github/workflows/nodejs.yml: -------------------------------------------------------------------------------- 1 | name: Node.js 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | 9 | strategy: 10 | matrix: 11 | node-version: [18.x, 20.x, 22.x, 23.x] 12 | 13 | steps: 14 | - uses: actions/checkout@v4 15 | - name: Node.js ${{ matrix.node-version }} 16 | uses: actions/setup-node@v4 17 | with: 18 | node-version: ${{ matrix.node-version }} 19 | - name: Dependencies 20 | run: | 21 | sudo apt-get update 22 | sudo apt-get install binaryen 23 | yarn 24 | - name: Build 25 | run: yarn build 26 | - name: Lint 27 | run: yarn lint 28 | - name: Test Coverage 29 | run: yarn test-coverage 30 | - name: Bundle Size 31 | run: yarn size 32 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish Package 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | publish: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | - name: Node.js 13 | uses: actions/setup-node@v4 14 | with: 15 | node-version: 22.x 16 | registry-url: https://registry.npmjs.org 17 | - name: Dependencies 18 | run: | 19 | sudo apt-get update 20 | sudo apt-get install binaryen 21 | yarn 22 | - name: Build 23 | run: yarn build 24 | - name: Publish 25 | env: 26 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} 27 | run: npm publish 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | coverage 2 | cjs 3 | esm 4 | umd 5 | workerd 6 | node_modules 7 | logs 8 | *.log 9 | npm-debug.log* 10 | 11 | # Yarn 12 | # From: https://yarnpkg.com/getting-started/qa#which-files-should-be-gitignored 13 | # Don't care about zero-install, that was the default (or only way) at the first release of yarn v2, 14 | # but I never liked it and they now realised that it's probably not great to have the cache 15 | # committed in the Git repo, so get rid of all of that. 16 | .pnp.* 17 | .yarn/* 18 | !.yarn/patches 19 | !.yarn/plugins 20 | !.yarn/releases 21 | !.yarn/sdks 22 | !.yarn/versions 23 | -------------------------------------------------------------------------------- /.swcrc: -------------------------------------------------------------------------------- 1 | { 2 | "exclude": "node_modules/**", 3 | "minify": true, 4 | "sourceMaps": true, 5 | "jsc": { 6 | "minify": { 7 | "sourceMap": true, 8 | "compress": { 9 | "const_to_let": false 10 | }, 11 | "mangle": { 12 | "toplevel": true 13 | } 14 | } 15 | }, 16 | "env": { 17 | "targets": { 18 | "chrome": 85, 19 | "edge": 79, 20 | "firefox": 79, 21 | "safari": 15, 22 | "node": 18 23 | }, 24 | "exclude": [ 25 | "transform-parameters" 26 | ] 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /.yarnrc.yml: -------------------------------------------------------------------------------- 1 | compressionLevel: mixed 2 | 3 | enableGlobalCache: false 4 | 5 | nodeLinker: node-modules 6 | 7 | yarnPath: .yarn/releases/yarn-4.5.1.cjs 8 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | This changelog keeps all release notes in one place and mirrors the release 4 | notes from the [GitHub releases][github-releases], except for older versions, 5 | where no GitHub releases had been created. 6 | 7 | ## v1.1.0 8 | 9 | ### Cloudflare Workers Support 10 | 11 | [Cloudflare Workers](https://developers.cloudflare.com/workers/) disallow loading WebAssembly modules from an 12 | `ArrayBuffer` for security reasons. 13 | 14 | They support [conditional import](https://developers.cloudflare.com/workers/wrangler/bundling/#conditional-exports) from 15 | the `workerd` field in `package.json`, therefore `xxhash-wasm` now includes an additional `workerd` package that 16 | includes the WASM in a separate file. (#51) 17 | 18 | You can install it with npm as usual and import it with: 19 | 20 | ```javascript 21 | import xxhash from "xxhash-wasm"; 22 | ``` 23 | 24 | ### Performance 25 | 26 | - Set state with `.subarray()` over `.slice()` to avoid unnecessary copy that will be discarded right afterwards (#52) 27 | 28 | ## v1.0.2 29 | 30 | - Add support for Typescript's `nodenext` module resolution (#33) 31 | 32 | ## v1.0.1 33 | 34 | - Export data types separately + fixed bigint data type (#28) 35 | 36 | ## v1.0.0 37 | 38 | This big release includes an up to a 3-4x performance improvement in most cases and a new streaming API similar to Node's built-in `crypto` API. To fully utilise the performance improvements, there are some breaking changes in the API and newer engine requirements. 39 | 40 | ### 3-4x Performance improvements 41 | 42 | To achieve these substantial performance improvements, a handful of new features have been used, which are fairly recent additions to the browsers, Node and the WebAssembly specification. 43 | These include the following: 44 | 45 | 1. [`BigInt`][bigint-mdn] support in WebAssembly 46 | 2. Bulk memory operations in WebAssembly 47 | 3. [`TextEncoder.encodeInto`][textencoder-encodeinto-mdn] 48 | 49 | Taking all of these requirements into account, `v1.0.0` should be compatible with: 50 | 51 | - Chrome >= 85 52 | - Edge >= 79 53 | - Firefox >= 79 54 | - Safari >= 15.0 55 | - Node >= 15.0 56 | 57 | If support for an older engine is required, `xxhash-wasm@0.4.2` is available with much broader engine support, but 3-4x slower hashing performance. 58 | 59 | Besides the features regarding memory optimisations for WebAssembly, the biggest addition is the use of [`BigInt`][bigint-mdn], which avoids the whole workaround that was previously used in order to represent `u64` integers in JavaScript. 60 | That makes everything a lot simpler and faster, but that also brings some breaking changes of the 64-bit API. 61 | 62 | The [`TextEncoder.encodeInto`][textencoder-encodeinto-mdn] allows to encode the string as UTF-8 bytes directly into the WebAssembly memory, meaning that if you have the string and hash it directly, it will be faster than encoding it yourself and then using the `Raw` API. 63 | *If possible, defer the encoding of the string to the hashing, unless you need to use the encoded string (bytes) for other purposes as well, or you are creating the bytes differently (e.g. different encoding), in which case it's much more efficient to use the `h**Raw` APIs instead of having to unnecessarily convert them to a string first.* 64 | 65 | ### Streaming API 66 | 67 | The streaming API allows to build up the input that is being hashed in an iterative manner, which is particularly helpful for larger inputs which are collected over time instead of having it all at once in memory. 68 | It is kept in line with Node's `crypto.createHash`, hence the streams are initialised with `create32`/`create64` and then `.update(string | Uint8Array)` is used to add an input, which can either be a `string` or a `Uint8Array`, and finally `.digest()` needs to be called to finalise the hash. 69 | 70 | ```javascript 71 | const { create32, create64 } = await xxhash(); 72 | 73 | // 32-bit version 74 | create32() 75 | .update("some data") 76 | // update accepts either a string or Uint8Array 77 | .update(Uint8Array.from([1, 2, 3])) 78 | .digest(); // 955607085 79 | 80 | // 64-bit version 81 | create64() 82 | .update("some data") 83 | // update accepts either a string or Uint8Array 84 | .update(Uint8Array.from([1, 2, 3])) 85 | .digest(); // 883044157688673477n 86 | ``` 87 | 88 | ### Breaking Changes 89 | 90 | #### 64-bit seed as [`BigInt`][bigint-mdn] 91 | 92 | 64-bit hash APIs now use [`BigInt`][bigint-mdn], where the `seed` is now a single [`BigInt`][bigint-mdn] instead of being split into the two halves `seedHigh` and `seedLow`. 93 | This makes it much simpler to use and avoids any workarounds for previous limitations. 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 109 | 118 | 119 | 120 |
0.4.21.0.0
103 | 104 | ```typescript 105 | h64(input: string, [seedHigh: u32, seedLow: u32]): string 106 | h64Raw(input: Uint8Array, [seedHigh: u32, seedLow: u32]): Uint8Array 107 | ``` 108 | 110 | 111 | ```typescript 112 | h64(input: string, [seed: BigInt]): BigInt 113 | h64ToString(input: string, [seed: BigInt]): string 114 | h64Raw(input: Uint8Array, [seed: BigInt]): BigInt 115 | ``` 116 | 117 |
121 | 122 | #### `h32`/`h64` return numbers instead of strings 123 | 124 | The hashes are numbers but were previously converted to a string of their a zero-padded hex string representations, mainly to keep the 32-bit in line with the 64-bit version, which could not be expressed by a single number without [`BigInt`][bigint-mdn]. 125 | This overhead is unnecessary for many applications and therefore the performance suffers. Now `h32` returns a `number` and `h64` a [`BigInt`][bigint-mdn]. 126 | For convenience, `h32ToString` and `h64ToString` have been added to get the hash as a string, which can also be achieved by converting them manually, e.g. `hash64.toString(16).padStart(16, "0")`. 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 142 | 154 | 155 | 156 |
0.4.21.0.0
136 | 137 | ```typescript 138 | h32(input: string, [seed: u32]): string 139 | h64(input: string, [seedHigh: u32, seedLow: u32]): string 140 | ``` 141 | 143 | 144 | ```typescript 145 | h32(input: string, [seed: u32]): number 146 | h64(input: string, [seed: BigInt]): BigInt 147 | 148 | // New *ToString methods for convenience and to get old behaviour 149 | h32ToString(input: string, [seed: u32]): string 150 | h64ToString(input: string, [seed: BigInt]): string 151 | ``` 152 | 153 |
157 | 158 | [bigint-mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt 159 | [textencoder-encodeinto-mdn]: https://developer.mozilla.org/en-US/docs/Web/API/TextEncoder/encodeInto 160 | 161 | ## v0.4.2 162 | 163 | - Fix 64-bit hex representation when second part has leading zeros (#23) 164 | 165 | ## v0.4.1 166 | 167 | - Initialise `TextEncoder` lazily 168 | 169 | ## v0.4.0 170 | 171 | - TypeScript definitions 172 | - `h32Raw` and `h64Raw` APIs for use with `Uint8Array` 173 | 174 | ## v0.3.1 175 | 176 | WebAssembly is optimised by binaryen: 177 | 178 | - Faster 179 | - Smaller 180 | 181 | ## v0.3.0 182 | 183 | New API to avoid reinitialising WASM instances 184 | 185 | ## v0.2.0 186 | 187 | Include a CommonJS bundle for Node.js 188 | 189 | 190 | [github-releases]: https://github.com/jungomi/xxhash-wasm/releases 191 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu 2 | 3 | LABEL maintainer "Michael Jungo " 4 | 5 | RUN apt-get update && apt-get install -y build-essential git cmake clang python3 6 | RUN git clone https://github.com/WebAssembly/binaryen && \ 7 | cd binaryen && \ 8 | cmake . && make install 9 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | ===================== 3 | 4 | Copyright © 2017 Michael Jungo 5 | 6 | Permission is hereby granted, free of charge, to any person 7 | obtaining a copy of this software and associated documentation 8 | files (the “Software”), to deal in the Software without 9 | restriction, including without limitation the rights to use, 10 | copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the 12 | Software is furnished to do so, subject to the following 13 | conditions: 14 | 15 | The above copyright notice and this permission notice shall be 16 | included in all copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, 19 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 20 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 21 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 22 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 23 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 24 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 25 | OTHER DEALINGS IN THE SOFTWARE. 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # xxhash-wasm 2 | 3 | [![Node.js][actions-nodejs-badge]][actions-nodejs-link] 4 | [![npm][npm-badge]][npm-link] 5 | 6 | A WebAssembly implementation of [xxHash][xxhash], a fast non-cryptographic hash 7 | algorithm. It can be called seamlessly from JavaScript. You can use it like any 8 | other JavaScript library but still get the benefits of WebAssembly, no special 9 | setup needed. 10 | 11 | ## Table of Contents 12 | 13 | 14 | 15 | * [Installation](#installation) 16 | * [From npm](#from-npm) 17 | * [From Unpkg](#from-unpkg) 18 | * [ES Modules](#es-modules) 19 | * [UMD build](#umd-build) 20 | * [Cloudflare Workers](#cloudflare-workers) 21 | * [Usage](#usage) 22 | * [Streaming Example](#streaming-example) 23 | * [Node](#node) 24 | * [Performance](#performance) 25 | * [Engine Requirements](#engine-requirements) 26 | * [API](#api) 27 | * [h32](#h32) 28 | * [h64](#h64) 29 | * [Streaming](#streaming) 30 | * [Comparison to xxhashjs](#comparison-to-xxhashjs) 31 | * [Benchmarks](#benchmarks) 32 | * [Bundle size](#bundle-size) 33 | 34 | 35 | 36 | ## Installation 37 | 38 | ### From npm 39 | 40 | ```sh 41 | npm install --save xxhash-wasm 42 | ``` 43 | 44 | Or with Yarn: 45 | 46 | ```sh 47 | yarn add xxhash-wasm 48 | ``` 49 | 50 | ### From [Unpkg][unpkg] 51 | 52 | #### ES Modules 53 | 54 | ```html 55 | 58 | ``` 59 | 60 | #### UMD build 61 | 62 | ```html 63 | 64 | ``` 65 | 66 | The global `xxhash` will be available. 67 | 68 | ### Cloudflare Workers 69 | 70 | If you are using [Cloudflare Workers](https://developers.cloudflare.com/workers/) (workerd) you can use the installed 71 | npm package as is. The `xxhash-wasm` package is compatible with Cloudflare Workers. 72 | 73 | ```javascript 74 | import xxhash from "xxhash-wasm"; 75 | ``` 76 | 77 | Importing it will pick the correct file base on the [conditional 78 | import](https://developers.cloudflare.com/workers/wrangler/bundling/#conditional-exports) 79 | from the package.json. 80 | 81 | ## Usage 82 | 83 | The WebAssembly is contained in the JavaScript bundle, so you don't need to 84 | manually fetch it and create a new WebAssembly instance. 85 | 86 | ```javascript 87 | import xxhash from "xxhash-wasm"; 88 | 89 | // Creates the WebAssembly instance. 90 | xxhash().then(hasher => { 91 | const input = "The string that is being hashed"; 92 | 93 | // 32-bit version 94 | hasher.h32(input); // 3998627172 (decimal representation) 95 | // For convenience, get hash as string of its zero-padded hex representation 96 | hasher.h32ToString(input); // "ee563564" 97 | 98 | // 64-bit version 99 | hasher.h64(input); // 5776724552493396044n (BigInt) 100 | // For convenience, get hash as string of its zero-padded hex representation 101 | hasher.h64ToString(input); // "502b0c5fc4a5704c" 102 | }); 103 | ``` 104 | 105 | Or with `async`/`await` and destructuring: 106 | 107 | ```javascript 108 | // Creates the WebAssembly instance. 109 | const { h32, h64 } = await xxhash(); 110 | 111 | const input = "The string that is being hashed"; 112 | // 32-bit version 113 | h32(input); // 3998627172 (decimal representation) 114 | // 64-bit version 115 | h64(input); // 5776724552493396044n (BigInt) 116 | ``` 117 | 118 | ### Streaming Example 119 | 120 | `xxhash-wasm` supports a `crypto`-like streaming api, useful for avoiding memory 121 | consumption when hashing large amounts of data: 122 | 123 | ```javascript 124 | const { create32, create64 } = await xxhash(); 125 | 126 | // 32-bit version 127 | create32() 128 | .update("some data") 129 | // update accepts either a string or Uint8Array 130 | .update(Uint8Array.from([1, 2, 3])) 131 | .digest(); // 955607085 132 | 133 | // 64-bit version 134 | create64() 135 | .update("some data") 136 | // update accepts either a string or Uint8Array 137 | .update(Uint8Array.from([1, 2, 3])) 138 | .digest(); // 883044157688673477n 139 | ``` 140 | 141 | ### Node 142 | 143 | It doesn't matter whether you are using CommonJS or ES Modules in Node 144 | (e.g. with `"type": "module"` in `package.json` or using the explicit file 145 | extensions `.cjs` or `.mjs` respectively), importing `xxhash-wasm` will always 146 | load the corresponding module, as both bundles are provided and specified in 147 | the `exports` field of its `package.json`, therefore the appropriate one will 148 | automatically be selected. 149 | 150 | **Using ES Modules** 151 | 152 | ```javascript 153 | import xxhash from "xxhash-wasm"; 154 | ``` 155 | 156 | **Using CommonJS** 157 | 158 | ```javascript 159 | const xxhash = require("xxhash-wasm"); 160 | ``` 161 | 162 | ## Performance 163 | 164 | For performance sensitive applications, `xxhash-wasm` provides the `h**` and 165 | `h**Raw` APIs, which return raw numeric hash results rather than zero-padded hex 166 | strings. The overhead of the string conversion in the `h**ToString` APIs can be 167 | as much as 20% of overall runtime when hashing small byte-size inputs, and the 168 | string result is often inconsequential (for example when simply checking if the 169 | the resulting hashes are the same). When necessary, getting a zero-padded hex 170 | string from the provided `number` or [`BigInt`][bigint-mdn] results is easily 171 | achieved via `result.toString(16).padStart(16, "0")` and the `h**ToString` APIs 172 | are purely for convenience. 173 | 174 | The `h**`, `h**ToString`, and streaming APIs make use of 175 | [`TextEncoder.encodeInto`][textencoder-encodeinto-mdn] to directly encode 176 | strings as a stream of UTF-8 bytes into the WebAssembly memory buffer, meaning 177 | that for string-hashing purposes, these APIs will be significantly faster than 178 | converting the string to bytes externally and using the `Raw` API. That said, 179 | for large strings it may be beneficial to consider the streaming API or another 180 | approach to encoding, as `encodeInto` is forced to allocate 3-times the string 181 | length to account for the chance the input string contains high-byte-count 182 | code units. 183 | 184 | *If possible, defer the encoding of the string to the hashing, unless you need 185 | to use the encoded string (bytes) for other purposes as well, or you are 186 | creating the bytes differently (e.g. different encoding), in which case it's 187 | much more efficient to use the `h**Raw` APIs instead of having to unnecessarily 188 | convert them to a string first.* 189 | 190 | ### Engine Requirements 191 | 192 | In an effort to make this library as performant as possible, it uses several 193 | recent additions to browsers, Node and the WebAssembly specification. 194 | Notably, these include: 195 | 196 | 1. [`BigInt`][bigint-mdn] support in WebAssembly 197 | 2. Bulk memory operations in WebAssembly 198 | 3. [`TextEncoder.encodeInto`][textencoder-encodeinto-mdn] 199 | 200 | Taking all of these requirements into account, `xxhash-wasm` should be 201 | compatible with: 202 | 203 | * Chrome >= 85 204 | * Edge >= 79 205 | * Firefox >= 79 206 | * Safari >= 15.0 207 | * Node >= 15.0 208 | 209 | If support for an older engine is required, `xxhash-wasm@0.4.2` is available 210 | with much broader engine support, but 3-4x slower hashing performance. 211 | 212 | ## API 213 | 214 | ```js 215 | const { 216 | h32, 217 | h32ToString, 218 | h32Raw, 219 | create32, 220 | h64, 221 | h64ToString, 222 | h64Raw, 223 | create64, 224 | } = await xxhash(); 225 | ``` 226 | 227 | Create a WebAssembly instance. 228 | 229 | ### h32 230 | 231 | ```typescript 232 | h32(input: string, [seed: u32]): number 233 | ``` 234 | 235 | Generate a 32-bit hash of the UTF-8 encoded bytes of `input`. The optional 236 | `seed` is a `u32` and any number greater than the maximum (`0xffffffff`) is 237 | wrapped, which means that `0xffffffff + 1 = 0`. 238 | 239 | Returns a `u32` `number` containing the hash value. 240 | 241 | ```typescript 242 | h32ToString(input: string, [seed: u32]): string 243 | ``` 244 | 245 | Same as `h32`, but returning a zero-padded hex string. 246 | 247 | ```typescript 248 | h32Raw(input: Uint8Array, [seed: u32]): number 249 | ``` 250 | 251 | Same as `h32` but with a `Uint8Array` as input instead of a `string`. 252 | 253 | ### h64 254 | 255 | ```typescript 256 | h64(input: string, [seed: bigint]): bigint 257 | ``` 258 | 259 | Generate a 64-bit hash of the UTF-8 encoded bytes of `input`. The optional 260 | `seed` is a `u64` provided as a [BigInt][bigint-mdn]. 261 | 262 | Returns a `u64` `bigint` containing the hash value. 263 | 264 | ```typescript 265 | h64ToString(input: string, [seed: bigint]): string 266 | ``` 267 | 268 | Same as `h64`, but returning a zero-padded hex string. 269 | 270 | ```typescript 271 | h64Raw(input: Uint8Array, [seed: bigint]): bigint 272 | ``` 273 | 274 | Same as `h64` but with a `Uint8Array` as input instead of a `string`. 275 | 276 | ### Streaming 277 | 278 | ```typescript 279 | type XXHash { 280 | update(input: string | Uint8Array): XXHash; 281 | digest(): T 282 | } 283 | ``` 284 | 285 | The streaming API mirrors Node's built-in `crypto.createHash`, providing 286 | `update` and `digest` methods to add data to the hash and compute the final hash 287 | value, respectively. 288 | 289 | ```typescript 290 | create32([seed: number]): XXHash 291 | ``` 292 | 293 | Create a 32-bit hash for streaming applications. 294 | 295 | ```typescript 296 | create64([seed: bigint]): XXHash 297 | ``` 298 | 299 | Create a 64-bit hash for streaming applications. 300 | 301 | ## Comparison to [xxhashjs][xxhashjs] 302 | 303 | [`xxhashjs`][xxhashjs] is implemented in pure JavaScript and because JavaScript 304 | is lacking support for 64-bit integers, it uses a workaround with 305 | [`cuint`][cuint]. Not only is that a big performance hit, but it also increases 306 | the bundle size by quite a bit when it's used in the browser. 307 | 308 | This library (`xxhash-wasm`) has the big advantage that WebAssembly supports 309 | `u64` and also some instructions (e.g. `rotl`), which would otherwise have 310 | to be emulated. However, The downside is that you have to initialise 311 | a WebAssembly instance, which takes a little over 2ms in Node and about 1ms in 312 | the browser. But once the instance is created, it can be used without any 313 | further overhead. For the benchmarks below, the instantiation is done before the 314 | benchmark and therefore it's excluded from the results, since it wouldn't make 315 | sense to always create a new WebAssembly instance. 316 | 317 | ### Benchmarks 318 | 319 | Benchmarks are using [Benchmark.js][benchmarkjs] with random strings of 320 | different lengths. *Higher is better* 321 | 322 | | String length | xxhashjs 32-bit | xxhashjs 64-bit | xxhash-wasm 32-bit | xxhash-wasm 64-bit | 323 | | ------------------------: | ------------------ | ------------------ | ----------------------- | ----------------------- | 324 | | 1 byte | 513,517 ops/sec | 11,896 ops/sec | ***5,752,446 ops/sec*** | 4,438,501 ops/sec | 325 | | 10 bytes | 552,133 ops/sec | 12,953 ops/sec | ***6,240,640 ops/sec*** | 4,855,340 ops/sec | 326 | | 100 bytes | 425,277 ops/sec | 10,838 ops/sec | ***5,470,011 ops/sec*** | 4,314,904 ops/sec | 327 | | 1,000 bytes | 102,165 ops/sec | 6,697 ops/sec | 3,283,526 ops/sec | ***3,332,556 ops/sec*** | 328 | | 10,000 bytes | 13,010 ops/sec | 1,452 ops/sec | 589,068 ops/sec | ***940,350 ops/sec*** | 329 | | 100,000 bytes | 477 ops/sec | 146 ops/sec | 61,824 ops/sec | ***98,959 ops/sec*** | 330 | | 1,000,000 bytes | 36.40 ops/sec | 12.93 ops/sec | 5,122 ops/sec | ***8,632 ops/sec*** | 331 | | 10,000,000 bytes | 3.12 ops/sec | 1.19 ops/sec | 326 ops/sec | ***444 ops/sec*** | 332 | | 100,000,000 bytes | 0.31 ops/sec | 0.13 ops/sec | 27.84 ops/sec | ***34.56 ops/sec*** | 333 | 334 | `xxhash-wasm` outperforms `xxhashjs` significantly, the 32-bit is up to 90 times 335 | faster (generally increases as the size of the input grows), and the 64-bit is 336 | up to 350 times faster (generally increases as the size of the input grows). 337 | 338 | The 64-bit version is the faster algorithm but there is a small degree of 339 | overhead involved in using BigInts, and so it retains a performance advantage 340 | over all lengths over xxhashjs and the 32-bit algorithm above ~1000 bytes. 341 | 342 | `xxhash-wasm` also significantly outperforms Node's built-in hash algorithms, 343 | making it suitable for use in a wide variety of situations, where 344 | non-cryptographic hashes are acceptable. Benchmarks from an x64 MacBook Pro 345 | running Node 17.3: 346 | 347 | | String length | Node `crypto` md5 | Node `crypto` sha1 | xxhash-wasm 64-bit | 348 | | ------------------------: | ------------------ | ------------------ | ----------------------- | 349 | | 1 byte | 342,924 ops/sec | 352,825 ops/sec | ***4,438,501 ops/sec*** | 350 | | 10 bytes | 356,596 ops/sec | 352,209 ops/sec | ***4,855,340 ops/sec*** | 351 | | 100 bytes | 354,898 ops/sec | 355,024 ops/sec | ***4,314,904 ops/sec*** | 352 | | 1,000 bytes | 249,242 ops/sec | 271,383 ops/sec | ***3,332,556 ops/sec*** | 353 | | 10,000 bytes | 62,896 ops/sec | 80,986 ops/sec | ***940,350 ops/sec*** | 354 | | 100,000 bytes | 7,316 ops/sec | 10,198 ops/sec | ***98,959 ops/sec*** | 355 | | 1,000,000 bytes | 698 ops/sec | 966 ops/sec | ***8,632 ops/sec*** | 356 | | 10,000,000 bytes | 58.98 ops/sec | 79.78 ops/sec | ***444 ops/sec*** | 357 | | 100,000,000 bytes | 6.30 ops/sec | 8.20 ops/sec | ***34.56 ops/sec*** | 358 | 359 | If suitable for your use case, the `Raw` API offers significant throughput 360 | improvements over the string-hashing API, particularly for smaller inputs, 361 | assuming that you have access to the `Uint8Array` already (see also the 362 | [Performance section](#performance)): 363 | 364 | | String length | xxhash-wasm 64-bit Raw | xxhash-wasm 64-bit | 365 | | ------------------------: | ----------------------- | ------------------- | 366 | | 1 byte | ***9,342,811 ops/sec*** | 4,438,501 ops/sec | 367 | | 10 bytes | ***9,668,989 ops/sec*** | 4,855,340 ops/sec | 368 | | 100 bytes | ***8,775,845 ops/sec*** | 4,314,904 ops/sec | 369 | | 1,000 bytes | ***5,541,403 ops/sec*** | 3,332,556 ops/sec | 370 | | 10,000 bytes | ***1,079,866 ops/sec*** | 940,350 ops/sec | 371 | | 100,000 bytes | ***113,350 ops/sec*** | 98,959 ops/sec | 372 | | 1,000,000 bytes | ***9,779 ops/sec*** | 8,632 ops/sec | 373 | | 10,000,000 bytes | ***563 ops/sec*** | 444 ops/sec | 374 | | 100,000,000 bytes | ***43.77 ops/sec*** | 34.56 ops/sec | 375 | 376 | ### Bundle size 377 | 378 | Both libraries can be used in the browser and they provide a UMD bundle. The 379 | bundles are self-contained, that means they can be included and used without 380 | having to add any other dependencies. The table shows the bundle size of the 381 | minified versions. *Lower is better*. 382 | 383 | | | xxhashjs | xxhash-wasm | 384 | | -------------- | ---------- | ------------- | 385 | | Bundle size | 41.5kB | ***11.4kB*** | 386 | | Gzipped Size | 10.3kB | ***2.3kB*** | 387 | 388 | [actions-nodejs-badge]: https://github.com/jungomi/xxhash-wasm/actions/workflows/nodejs.yml/badge.svg 389 | [actions-nodejs-link]: https://github.com/jungomi/xxhash-wasm/actions/workflows/nodejs.yml 390 | [benchmarkjs]: https://benchmarkjs.com/ 391 | [bigint-mdn]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/BigInt 392 | [cuint]: https://github.com/pierrec/js-cuint 393 | [npm-badge]: https://img.shields.io/npm/v/xxhash-wasm.svg?style=flat-square 394 | [npm-link]: https://www.npmjs.com/package/xxhash-wasm 395 | [release-notes-v1.0.0]: https://github.com/jungomi/xxhash-wasm/releases/tag/v1.0.0 396 | [textencoder-encodeinto-mdn]: https://developer.mozilla.org/en-US/docs/Web/API/TextEncoder/encodeInto 397 | [travis]: https://travis-ci.org/jungomi/xxhash-wasm 398 | [travis-badge]: https://img.shields.io/travis/jungomi/xxhash-wasm/master.svg?style=flat-square 399 | [unpkg]: https://unpkg.com/ 400 | [xxhash]: https://github.com/Cyan4973/xxHash 401 | [xxhashjs]: https://github.com/pierrec/js-xxhash 402 | -------------------------------------------------------------------------------- /bench/index.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | const Benchmark = require("benchmark"); 3 | const xxhash = require("xxhash-wasm"); 4 | const XXH = require("xxhashjs"); 5 | 6 | // This is the highest utf-8 character that uses only one byte. A string will be 7 | // randomly generated and this makes the number of bytes consitent/predictable. 8 | const highestSingleByteChar = 0x7f; 9 | 10 | function randomString(numBytes) { 11 | // If number of bytes is too high it will result in a stackoverflow. 12 | // To circumvent that the string is generated in chunks. 13 | const strings = []; 14 | const numChunks = Math.ceil(numBytes / 1e5); 15 | for (let i = 1; i <= numChunks; i++) { 16 | const bytes = 17 | i === numChunks && numBytes % 1e5 !== 0 ? numBytes % 1e5 : 1e5; 18 | const codePoints = Array(bytes) 19 | .fill() 20 | .map(() => Math.floor(Math.random() * (highestSingleByteChar + 1))); 21 | strings.push(String.fromCodePoint(...codePoints)); 22 | } 23 | return "".concat(...strings); 24 | } 25 | 26 | const handlers = { 27 | onCycle(event) { 28 | console.log(String(event.target)); 29 | }, 30 | onComplete() { 31 | const fastest = this.filter("fastest").map("name"); 32 | console.log(`Benchmark ${this.name} - Fastest is ${fastest}`); 33 | }, 34 | }; 35 | 36 | const seed = 0; 37 | const seedBigInt = 0n; 38 | 39 | async function runBench() { 40 | console.time("wasm setup"); 41 | const { h32, h64 } = await xxhash(); 42 | console.timeEnd("wasm setup"); 43 | 44 | for (let i = 1; i <= 1e8; i *= 10) { 45 | const suite = new Benchmark.Suite(`${i} bytes`, handlers); 46 | const input = randomString(i); 47 | 48 | suite 49 | .add("xxhashjs#h32", () => { 50 | XXH.h32(input, seed).toString(16); 51 | }) 52 | .add("xxhashjs#h64", () => { 53 | XXH.h64(input, seed).toString(16); 54 | }) 55 | .add("xxhash-wasm#h32", () => { 56 | h32(input, seed); 57 | }) 58 | .add("xxhash-wasm#h64", () => { 59 | h64(input, seedBigInt); 60 | }) 61 | .run(); 62 | } 63 | } 64 | 65 | runBench(); 66 | -------------------------------------------------------------------------------- /bench/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "xxhash-wasm-bench", 3 | "private": true, 4 | "author": "Michael Jungo ", 5 | "license": "MIT", 6 | "scripts": { 7 | "start": "node --max-old-space-size=4096 index.js" 8 | }, 9 | "dependencies": { 10 | "benchmark": "^2.1.4", 11 | "xxhash-wasm": "^1.0.1", 12 | "xxhashjs": "^0.2.2" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /bench/yarn.lock: -------------------------------------------------------------------------------- 1 | # This file is generated by running "yarn install" inside your project. 2 | # Manual changes might be lost - proceed with caution! 3 | 4 | __metadata: 5 | version: 5 6 | cacheKey: 8 7 | 8 | "benchmark@npm:^2.1.4": 9 | version: 2.1.4 10 | resolution: "benchmark@npm:2.1.4" 11 | dependencies: 12 | lodash: ^4.17.4 13 | platform: ^1.3.3 14 | checksum: aa466561d4f2b0a2419a3069b8f90fd35ffacf26849697eea9de525ecfbd10b44da11070cc51c88d772076db8cb2415641b493de7d6c024fdf8551019c6fcf1c 15 | languageName: node 16 | linkType: hard 17 | 18 | "cuint@npm:^0.2.2": 19 | version: 0.2.2 20 | resolution: "cuint@npm:0.2.2" 21 | checksum: b8127a93a7f16ce120ffcb22108014327c9808b258ee20e7dbb4c6740d7cb0f0c12d18a054eb716b0f2470090666abaae8a082d3cd5ef0e94fa447dd155842c4 22 | languageName: node 23 | linkType: hard 24 | 25 | "lodash@npm:^4.17.4": 26 | version: 4.17.21 27 | resolution: "lodash@npm:4.17.21" 28 | checksum: eb835a2e51d381e561e508ce932ea50a8e5a68f4ebdd771ea240d3048244a8d13658acbd502cd4829768c56f2e16bdd4340b9ea141297d472517b83868e677f7 29 | languageName: node 30 | linkType: hard 31 | 32 | "platform@npm:^1.3.3": 33 | version: 1.3.4 34 | resolution: "platform@npm:1.3.4" 35 | checksum: aadbdbda8475bcfe4b491f42bf0c4a5295244f446c42ca85a9036bb69bf48ccd4beeddb6908e4dc162acb735889311d6c5949784c44c6df6d8beb781344300bc 36 | languageName: node 37 | linkType: hard 38 | 39 | "xxhash-wasm-bench@workspace:.": 40 | version: 0.0.0-use.local 41 | resolution: "xxhash-wasm-bench@workspace:." 42 | dependencies: 43 | benchmark: ^2.1.4 44 | xxhash-wasm: ^1.0.1 45 | xxhashjs: ^0.2.2 46 | languageName: unknown 47 | linkType: soft 48 | 49 | "xxhash-wasm@npm:^1.0.1": 50 | version: 1.0.1 51 | resolution: "xxhash-wasm@npm:1.0.1" 52 | checksum: beb7677772724508c6ffde7924c0c8b3b879337d1d1598563af09e87befc6a335015bec0fa41873cc27d4745f32dea19d20f9c606ea71d29e1c949e8a13d5c24 53 | languageName: node 54 | linkType: hard 55 | 56 | "xxhashjs@npm:^0.2.2": 57 | version: 0.2.2 58 | resolution: "xxhashjs@npm:0.2.2" 59 | dependencies: 60 | cuint: ^0.2.2 61 | checksum: cf6baf05bafe5651dbf108008bafdb1ebe972f65228633f00b56c49d7a1e614a821fe3345c4eb27462994c7c954d982eae05871be6a48146f30803dd87f3c3b6 62 | languageName: node 63 | linkType: hard 64 | -------------------------------------------------------------------------------- /eslint.config.mjs: -------------------------------------------------------------------------------- 1 | import prettier from "eslint-plugin-prettier"; 2 | import globals from "globals"; 3 | import babelParser from "@babel/eslint-parser"; 4 | import path from "node:path"; 5 | import { fileURLToPath } from "node:url"; 6 | import js from "@eslint/js"; 7 | import { FlatCompat } from "@eslint/eslintrc"; 8 | 9 | const __filename = fileURLToPath(import.meta.url); 10 | const __dirname = path.dirname(__filename); 11 | const compat = new FlatCompat({ 12 | baseDirectory: __dirname, 13 | recommendedConfig: js.configs.recommended, 14 | allConfig: js.configs.all, 15 | }); 16 | 17 | export default [ 18 | { 19 | ignores: [ 20 | "bench/node_modules", 21 | "**/coverage", 22 | "**/cjs", 23 | "**/esm", 24 | "**/umd", 25 | "**/workerd", 26 | ], 27 | }, 28 | ...compat.extends("eslint:recommended", "prettier"), 29 | { 30 | plugins: { 31 | prettier, 32 | }, 33 | 34 | languageOptions: { 35 | globals: { 36 | ...globals.browser, 37 | ...globals.node, 38 | BigInt: true, 39 | WebAssembly: true, 40 | }, 41 | 42 | parser: babelParser, 43 | ecmaVersion: 6, 44 | sourceType: "module", 45 | 46 | parserOptions: { 47 | ecmaFeatures: { 48 | jsx: true, 49 | }, 50 | 51 | requireConfigFile: false, 52 | }, 53 | }, 54 | 55 | rules: { 56 | "prettier/prettier": "error", 57 | "brace-style": ["error", "1tbs"], 58 | "no-unused-vars": "warn", 59 | "no-var": "error", 60 | "prefer-const": "warn", 61 | }, 62 | }, 63 | ]; 64 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | setupFiles: ["/test/setup.js"], 3 | transform: { 4 | "^.+\\.(t|j)sx?$": "@swc/jest", 5 | }, 6 | }; 7 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "xxhash-wasm", 3 | "version": "1.1.0", 4 | "description": "A WebAssembly implementation of xxHash", 5 | "type": "module", 6 | "main": "./cjs/xxhash-wasm.cjs", 7 | "module": "./esm/xxhash-wasm.js", 8 | "exports": { 9 | "types": "./types.d.ts", 10 | "workerd": "./workerd/xxhash-wasm.js", 11 | "import": "./esm/xxhash-wasm.js", 12 | "require": "./cjs/xxhash-wasm.cjs" 13 | }, 14 | "types": "./types.d.ts", 15 | "author": "Michael Jungo ", 16 | "license": "MIT", 17 | "repository": { 18 | "type": "git", 19 | "url": "https://github.com/jungomi/xxhash-wasm.git" 20 | }, 21 | "files": [ 22 | "cjs", 23 | "esm", 24 | "umd", 25 | "workerd", 26 | "types.d.ts" 27 | ], 28 | "keywords": [ 29 | "xxhash", 30 | "hash", 31 | "wasm", 32 | "webassembly" 33 | ], 34 | "scripts": { 35 | "build": "yarn run build-wasm && yarn run build-js", 36 | "build-js": "rollup -c", 37 | "build-wasm": "wasm-opt --enable-bulk-memory -O4 src/xxhash.wat -o src/xxhash.wasm", 38 | "clean": "rimraf coverage cjs esm umd", 39 | "fix": "eslint . --fix", 40 | "lint": "eslint .", 41 | "size": "bundlewatch", 42 | "test": "jest", 43 | "test-update": "jest --updateSnapshot", 44 | "test-coverage": "jest --coverage", 45 | "prebuild": "yarn run clean", 46 | "prepublish": "yarn run build" 47 | }, 48 | "devDependencies": { 49 | "@babel/core": "^7.26.0", 50 | "@babel/eslint-parser": "^7.25.9", 51 | "@eslint/eslintrc": "^3.2.0", 52 | "@eslint/js": "^9.15.0", 53 | "@swc/core": "^1.9.2", 54 | "@swc/jest": "^0.2.37", 55 | "bundlewatch": "^0.4.0", 56 | "eslint": "^9.15.0", 57 | "eslint-config-prettier": "^9.1.0", 58 | "eslint-plugin-prettier": "^5.2.1", 59 | "globals": "^15.12.0", 60 | "jest": "^29.7.0", 61 | "jest-t-assert": "^0.3.0", 62 | "node-gyp": "^10.2.0", 63 | "prettier": "^3.3.3", 64 | "rimraf": "^6.0.1", 65 | "rollup": "^4.27.3", 66 | "rollup-plugin-copy": "^3.5.0", 67 | "rollup-plugin-node-resolve": "^5.2.0", 68 | "rollup-plugin-replace": "^2.2.0", 69 | "rollup-plugin-swc3": "^0.12.1" 70 | }, 71 | "bundlewatch": { 72 | "files": [ 73 | { 74 | "path": "./cjs/xxhash-wasm.cjs", 75 | "maxSize": "2.2kb" 76 | }, 77 | { 78 | "path": "./esm/xxhash-wasm.js", 79 | "maxSize": "2.2kb" 80 | }, 81 | { 82 | "path": "./umd/xxhash-wasm.js", 83 | "maxSize": "2.3kb" 84 | } 85 | ] 86 | }, 87 | "packageManager": "yarn@4.5.1" 88 | } 89 | -------------------------------------------------------------------------------- /rollup.config.js: -------------------------------------------------------------------------------- 1 | import { readFileSync } from "fs"; 2 | import { dirname, resolve } from "path"; 3 | import copy from "rollup-plugin-copy"; 4 | import nodeResolve from "rollup-plugin-node-resolve"; 5 | import replace from "rollup-plugin-replace"; 6 | import { swc } from "rollup-plugin-swc3"; 7 | import { fileURLToPath } from "url"; 8 | 9 | const __filename = fileURLToPath(import.meta.url); 10 | const __dirname = dirname(__filename); 11 | 12 | const wasmBytes = Array.from( 13 | readFileSync(resolve(__dirname, "src/xxhash.wasm")), 14 | ); 15 | 16 | const output = [ 17 | { 18 | file: "cjs/xxhash-wasm.cjs", 19 | format: "cjs", 20 | sourcemap: true, 21 | exports: "default", 22 | }, 23 | { file: "esm/xxhash-wasm.js", format: "es", sourcemap: true }, 24 | { 25 | file: "umd/xxhash-wasm.js", 26 | format: "umd", 27 | name: "xxhash", 28 | sourcemap: true, 29 | }, 30 | ]; 31 | const replacements = { 32 | WASM_PRECOMPILED_BYTES: JSON.stringify(wasmBytes), 33 | }; 34 | 35 | const swc_config = JSON.parse( 36 | readFileSync(resolve(__dirname, ".swcrc"), "utf-8"), 37 | ); 38 | 39 | const plugins = [ 40 | replace(replacements), 41 | // The config is necessary, because the plugin overwrites some of the settings, 42 | // instead of just falling back to .swcrc 43 | swc(swc_config), 44 | nodeResolve(), 45 | ]; 46 | 47 | export default [ 48 | { 49 | input: "src/index.js", 50 | output, 51 | plugins, 52 | }, 53 | { 54 | input: "src/index.workerd.js", 55 | output: { file: "workerd/xxhash-wasm.js", format: "es", sourcemap: true }, 56 | external: /\.wasm$/, 57 | plugins: [ 58 | ...plugins, 59 | copy({ 60 | targets: [{ src: "src/xxhash.wasm", dest: "workerd" }], 61 | }), 62 | ], 63 | }, 64 | ]; 65 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import { xxhash } from "./xxhash"; 2 | 3 | // The .wasm is filled in by the build process, so the user doesn't need to load 4 | // xxhash.wasm by themselves because it's part of the bundle. Otherwise it 5 | // couldn't be distributed easily as the user would need to host xxhash.wasm 6 | // and then fetch it, to be able to use it. 7 | // eslint-disable-next-line no-undef 8 | const wasmBytes = new Uint8Array(WASM_PRECOMPILED_BYTES); 9 | 10 | export default async function () { 11 | return xxhash((await WebAssembly.instantiate(wasmBytes)).instance); 12 | } 13 | -------------------------------------------------------------------------------- /src/index.workerd.js: -------------------------------------------------------------------------------- 1 | import { xxhash } from "./xxhash"; 2 | 3 | // In CloudFlare workerd we must use import to prevent code injection. 4 | import module from "./xxhash.wasm"; 5 | 6 | export default async function () { 7 | return xxhash(await WebAssembly.instantiate(module)); 8 | } 9 | -------------------------------------------------------------------------------- /src/xxhash.js: -------------------------------------------------------------------------------- 1 | const u32_BYTES = 4; 2 | const u64_BYTES = 8; 3 | 4 | // The xxh32 hash state struct: 5 | const XXH32_STATE_SIZE_BYTES = 6 | u32_BYTES + // total_len 7 | u32_BYTES + // large_len 8 | u32_BYTES * 4 + // Accumulator lanes 9 | u32_BYTES * 4 + // Internal buffer 10 | u32_BYTES + // memsize 11 | u32_BYTES; // reserved 12 | 13 | // The xxh64 hash state struct: 14 | const XXH64_STATE_SIZE_BYTES = 15 | u64_BYTES + // total_len 16 | u64_BYTES * 4 + // Accumulator lanes 17 | u64_BYTES * 4 + // Internal buffer 18 | u32_BYTES + // memsize 19 | u32_BYTES + // reserved32 20 | u64_BYTES; // reserved64 21 | 22 | export function xxhash(instance) { 23 | const { 24 | exports: { 25 | mem, 26 | xxh32, 27 | xxh64, 28 | init32, 29 | update32, 30 | digest32, 31 | init64, 32 | update64, 33 | digest64, 34 | }, 35 | } = instance; 36 | 37 | let memory = new Uint8Array(mem.buffer); 38 | // Grow the wasm linear memory to accommodate length + offset bytes 39 | function growMemory(length, offset) { 40 | if (mem.buffer.byteLength < length + offset) { 41 | const extraPages = Math.ceil( 42 | // Wasm pages are spec'd to 64K 43 | (length + offset - mem.buffer.byteLength) / (64 * 1024), 44 | ); 45 | mem.grow(extraPages); 46 | // After growing, the original memory's ArrayBuffer is detached, so we'll 47 | // need to replace our view over it with a new one over the new backing 48 | // ArrayBuffer. 49 | memory = new Uint8Array(mem.buffer); 50 | } 51 | } 52 | 53 | // The h32 and h64 streaming hash APIs are identical, so we can implement 54 | // them both by way of a templated call to this generalized function. 55 | function create(size, seed, init, update, digest, finalize) { 56 | // Ensure that we've actually got enough space in the wasm memory to store 57 | // the state blob for this hasher. 58 | growMemory(size); 59 | 60 | // We'll hold our hashing state in this closure. 61 | const state = new Uint8Array(size); 62 | memory.set(state); 63 | init(0, seed); 64 | 65 | // Each time we interact with wasm, it may have mutated our state so we'll 66 | // need to read it back into our closed copy. 67 | state.set(memory.subarray(0, size)); 68 | 69 | return { 70 | update(input) { 71 | memory.set(state); 72 | let length; 73 | if (typeof input === "string") { 74 | growMemory(input.length * 3, size); 75 | length = encoder.encodeInto(input, memory.subarray(size)).written; 76 | } else { 77 | // The only other valid input type is a Uint8Array 78 | growMemory(input.byteLength, size); 79 | memory.set(input, size); 80 | length = input.byteLength; 81 | } 82 | update(0, size, length); 83 | state.set(memory.subarray(0, size)); 84 | return this; 85 | }, 86 | digest() { 87 | memory.set(state); 88 | return finalize(digest(0)); 89 | }, 90 | }; 91 | } 92 | 93 | // Logical shift right makes it an u32, otherwise it's interpreted as an i32. 94 | function forceUnsigned32(i) { 95 | return i >>> 0; 96 | } 97 | 98 | // BigInts are arbitrary precision and signed, so to get the "correct" u64 99 | // value from the return, we'll need to force that interpretation. 100 | const u64Max = 2n ** 64n - 1n; 101 | function forceUnsigned64(i) { 102 | return i & u64Max; 103 | } 104 | 105 | const encoder = new TextEncoder(); 106 | const defaultSeed = 0; 107 | const defaultBigSeed = 0n; 108 | 109 | function h32(str, seed = defaultSeed) { 110 | // https://developer.mozilla.org/en-US/docs/Web/API/TextEncoder/encodeInto#buffer_sizing 111 | // By sizing the buffer to 3 * string-length we guarantee that the buffer 112 | // will be appropriately sized for the utf-8 encoding of the string. 113 | growMemory(str.length * 3, 0); 114 | return forceUnsigned32( 115 | xxh32(0, encoder.encodeInto(str, memory).written, seed), 116 | ); 117 | } 118 | 119 | function h64(str, seed = defaultBigSeed) { 120 | growMemory(str.length * 3, 0); 121 | return forceUnsigned64( 122 | xxh64(0, encoder.encodeInto(str, memory).written, seed), 123 | ); 124 | } 125 | 126 | return { 127 | h32, 128 | h32ToString(str, seed = defaultSeed) { 129 | return h32(str, seed).toString(16).padStart(8, "0"); 130 | }, 131 | h32Raw(inputBuffer, seed = defaultSeed) { 132 | growMemory(inputBuffer.byteLength, 0); 133 | memory.set(inputBuffer); 134 | return forceUnsigned32(xxh32(0, inputBuffer.byteLength, seed)); 135 | }, 136 | create32(seed = defaultSeed) { 137 | return create( 138 | XXH32_STATE_SIZE_BYTES, 139 | seed, 140 | init32, 141 | update32, 142 | digest32, 143 | forceUnsigned32, 144 | ); 145 | }, 146 | h64, 147 | h64ToString(str, seed = defaultBigSeed) { 148 | return h64(str, seed).toString(16).padStart(16, "0"); 149 | }, 150 | h64Raw(inputBuffer, seed = defaultBigSeed) { 151 | growMemory(inputBuffer.byteLength, 0); 152 | memory.set(inputBuffer); 153 | return forceUnsigned64(xxh64(0, inputBuffer.byteLength, seed)); 154 | }, 155 | create64(seed = defaultBigSeed) { 156 | return create( 157 | XXH64_STATE_SIZE_BYTES, 158 | seed, 159 | init64, 160 | update64, 161 | digest64, 162 | forceUnsigned64, 163 | ); 164 | }, 165 | }; 166 | } 167 | -------------------------------------------------------------------------------- /src/xxhash.wasm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jungomi/xxhash-wasm/5923f26411ed763044bed17a1fec33fee74e47a0/src/xxhash.wasm -------------------------------------------------------------------------------- /src/xxhash.wat: -------------------------------------------------------------------------------- 1 | (module 2 | (memory (export "mem") 1) 3 | 4 | (global $PRIME32_1 i32 (i32.const 2654435761)) 5 | (global $PRIME32_2 i32 (i32.const 2246822519)) 6 | (global $PRIME32_3 i32 (i32.const 3266489917)) 7 | (global $PRIME32_4 i32 (i32.const 668265263)) 8 | (global $PRIME32_5 i32 (i32.const 374761393)) 9 | 10 | (global $PRIME64_1 i64 (i64.const 11400714785074694791)) 11 | (global $PRIME64_2 i64 (i64.const 14029467366897019727)) 12 | (global $PRIME64_3 i64 (i64.const 1609587929392839161)) 13 | (global $PRIME64_4 i64 (i64.const 9650029242287828579)) 14 | (global $PRIME64_5 i64 (i64.const 2870177450012600261)) 15 | 16 | ;; State offsets for XXH32 state structs 17 | (global $SO_32_TOTAL_LEN i32 (i32.const 0)) 18 | (global $SO_32_LARGE_LEN i32 (i32.const 4)) 19 | (global $SO_32_V1 i32 (i32.const 8)) 20 | (global $SO_32_V2 i32 (i32.const 12)) 21 | (global $SO_32_V3 i32 (i32.const 16)) 22 | (global $SO_32_V4 i32 (i32.const 20)) 23 | (global $SO_32_MEM32 i32 (i32.const 24)) 24 | (global $SO_32_MEMSIZE i32 (i32.const 40)) 25 | 26 | ;; State offsets for XXH64 state structs 27 | (global $SO_64_TOTAL_LEN i32 (i32.const 0)) 28 | (global $SO_64_V1 i32 (i32.const 8)) 29 | (global $SO_64_V2 i32 (i32.const 16)) 30 | (global $SO_64_V3 i32 (i32.const 24)) 31 | (global $SO_64_V4 i32 (i32.const 32)) 32 | (global $SO_64_MEM64 i32 (i32.const 40)) 33 | (global $SO_64_MEMSIZE i32 (i32.const 72)) 34 | 35 | (func (export "xxh32") (param $ptr i32) (param $len i32) (param $seed i32) (result i32) 36 | (local $h32 i32) 37 | (local $end i32) 38 | (local $limit i32) 39 | (local $v1 i32) 40 | (local $v2 i32) 41 | (local $v3 i32) 42 | (local $v4 i32) 43 | (local.set $end (i32.add (local.get $ptr) (local.get $len))) 44 | (if 45 | (i32.ge_u (local.get $len) (i32.const 16)) 46 | (block 47 | (local.set $limit (i32.sub (local.get $end) (i32.const 16))) 48 | (local.set $v1 (i32.add (i32.add (local.get $seed) (global.get $PRIME32_1)) (global.get $PRIME32_2))) 49 | (local.set $v2 (i32.add (local.get $seed) (global.get $PRIME32_2))) 50 | (local.set $v3 (i32.add (local.get $seed) (i32.const 0))) 51 | (local.set $v4 (i32.sub (local.get $seed) (global.get $PRIME32_1))) 52 | ;; For every chunk of 4 words, so 4 * 32bits = 16 bytes 53 | (loop $4words-loop 54 | (local.set $v1 (call $round32 (local.get $v1) (i32.load (local.get $ptr)))) 55 | (local.set $ptr (i32.add (local.get $ptr) (i32.const 4))) 56 | (local.set $v2 (call $round32 (local.get $v2) (i32.load (local.get $ptr)))) 57 | (local.set $ptr (i32.add (local.get $ptr) (i32.const 4))) 58 | (local.set $v3 (call $round32 (local.get $v3) (i32.load (local.get $ptr)))) 59 | (local.set $ptr (i32.add (local.get $ptr) (i32.const 4))) 60 | (local.set $v4 (call $round32 (local.get $v4) (i32.load (local.get $ptr)))) 61 | (local.set $ptr (i32.add (local.get $ptr) (i32.const 4))) 62 | (br_if $4words-loop (i32.le_u (local.get $ptr) (local.get $limit)))) 63 | (local.set $h32 (i32.add 64 | (i32.rotl (local.get $v1) (i32.const 1)) 65 | (i32.add 66 | (i32.rotl (local.get $v2) (i32.const 7)) 67 | (i32.add 68 | (i32.rotl (local.get $v3) (i32.const 12)) 69 | (i32.rotl (local.get $v4) (i32.const 18))))))) 70 | ;; else block, when input is smaller than 16 bytes 71 | (local.set $h32 (i32.add (local.get $seed) (global.get $PRIME32_5)))) 72 | (local.set $h32 (i32.add (local.get $h32) (local.get $len))) 73 | (call $finalize32 (local.get $h32) (local.get $ptr) (i32.and (local.get $len) (i32.const 15)))) 74 | 75 | (func $finalize32 (param $h32 i32) (param $ptr i32) (param $len i32) (result i32) 76 | (local $end i32) 77 | (local.set $end (i32.add (local.get $ptr) (local.get $len))) 78 | ;; For the remaining words not covered above, either 0, 1, 2 or 3 79 | (block $exit-remaining-words 80 | (loop $remaining-words-loop 81 | (br_if $exit-remaining-words (i32.gt_u (i32.add (local.get $ptr) (i32.const 4)) (local.get $end))) 82 | (local.set $h32 (i32.add (local.get $h32) (i32.mul (i32.load (local.get $ptr)) (global.get $PRIME32_3)))) 83 | (local.set $h32 (i32.mul (i32.rotl (local.get $h32) (i32.const 17)) (global.get $PRIME32_4))) 84 | (local.set $ptr (i32.add (local.get $ptr) (i32.const 4))) 85 | (br $remaining-words-loop))) 86 | ;; For the remaining bytes that didn't make a whole word, 87 | ;; either 0, 1, 2 or 3 bytes, as 4bytes = 32bits = 1 word. 88 | (block $exit-remaining-bytes 89 | (loop $remaining-bytes-loop 90 | (br_if $exit-remaining-bytes (i32.ge_u (local.get $ptr) (local.get $end))) 91 | (local.set $h32 (i32.add (local.get $h32) (i32.mul (i32.load8_u (local.get $ptr)) (global.get $PRIME32_5)))) 92 | (local.set $h32 (i32.mul (i32.rotl (local.get $h32) (i32.const 11)) (global.get $PRIME32_1))) 93 | (local.set $ptr (i32.add (local.get $ptr) (i32.const 1))) 94 | (br $remaining-bytes-loop))) 95 | (local.set $h32 (i32.xor (local.get $h32) (i32.shr_u (local.get $h32) (i32.const 15)))) 96 | (local.set $h32 (i32.mul (local.get $h32) (global.get $PRIME32_2))) 97 | (local.set $h32 (i32.xor (local.get $h32) (i32.shr_u (local.get $h32) (i32.const 13)))) 98 | (local.set $h32 (i32.mul (local.get $h32) (global.get $PRIME32_3))) 99 | (local.set $h32 (i32.xor (local.get $h32) (i32.shr_u (local.get $h32) (i32.const 16)))) 100 | (local.get $h32)) 101 | 102 | (func $round32 (param $seed i32) (param $value i32) (result i32) 103 | (local.set $seed (i32.add (local.get $seed) (i32.mul (local.get $value) (global.get $PRIME32_2)))) 104 | (local.set $seed (i32.rotl (local.get $seed) (i32.const 13))) 105 | (local.set $seed (i32.mul (local.get $seed) (global.get $PRIME32_1))) 106 | (local.get $seed)) 107 | 108 | ;; Initialize a XXH32 state struct 109 | (func (export "init32") (param $statePtr i32) (param $seed i32) 110 | (i32.store 111 | (i32.add (local.get $statePtr) (global.get $SO_32_V1)) 112 | (i32.add (i32.add (local.get $seed) (global.get $PRIME32_1)) (global.get $PRIME32_2))) 113 | (i32.store 114 | (i32.add (local.get $statePtr) (global.get $SO_32_V2)) 115 | (i32.add (local.get $seed) (global.get $PRIME32_2))) 116 | (i32.store 117 | (i32.add (local.get $statePtr) (global.get $SO_32_V3)) 118 | (local.get $seed)) 119 | (i32.store 120 | (i32.add (local.get $statePtr) (global.get $SO_32_V4)) 121 | (i32.sub (local.get $seed) (global.get $PRIME32_1)))) 122 | 123 | ;; Update a XXH32 State struct with the provided input bytes 124 | (func (export "update32") (param $statePtr i32) (param $inputPtr i32) (param $len i32) 125 | (local $end i32) 126 | (local $limit i32) 127 | (local $mem32Ptr i32) 128 | (local $initial-memsize i32) 129 | (local $largeLenPtr i32) 130 | (local $totalLenPtr i32) 131 | (local $v1 i32) 132 | (local $v2 i32) 133 | (local $v3 i32) 134 | (local $v4 i32) 135 | (local.set $end (i32.add (local.get $inputPtr) (local.get $len))) 136 | (local.set $mem32Ptr (i32.add (local.get $statePtr) (global.get $SO_32_MEM32))) 137 | (local.set $largeLenPtr (i32.add (local.get $statePtr) (global.get $SO_32_LARGE_LEN))) 138 | (local.set $totalLenPtr (i32.add (local.get $statePtr) (global.get $SO_32_TOTAL_LEN))) 139 | (local.set $initial-memsize (i32.load (i32.add (local.get $statePtr) (global.get $SO_32_MEMSIZE)))) 140 | (i32.store (local.get $totalLenPtr) (i32.add (i32.load (local.get $totalLenPtr)) (local.get $len))) 141 | (i32.store 142 | (local.get $largeLenPtr) 143 | (i32.or 144 | (i32.load (local.get $largeLenPtr)) 145 | (i32.or 146 | (i32.ge_u (local.get $len) (i32.const 16)) 147 | (i32.ge_u (i32.load (local.get $totalLenPtr)) (i32.const 16))))) 148 | (if (i32.lt_u 149 | (i32.add (local.get $len) (local.get $initial-memsize)) 150 | (i32.const 16)) 151 | (block 152 | (memory.copy 153 | (i32.add (local.get $mem32Ptr) (local.get $initial-memsize)) 154 | (local.get $inputPtr) 155 | (local.get $len)) 156 | (i32.store 157 | (i32.add (local.get $statePtr) (global.get $SO_32_MEMSIZE)) 158 | (i32.add (local.get $initial-memsize) (local.get $len))) 159 | (return))) 160 | (if (i32.ne (local.get $initial-memsize) (i32.const 0)) 161 | (block 162 | (memory.copy 163 | (i32.add (local.get $mem32Ptr) (local.get $initial-memsize)) 164 | (local.get $inputPtr) 165 | (i32.sub (i32.const 16) (local.get $initial-memsize))) 166 | (i32.store 167 | (i32.add (local.get $statePtr) (global.get $SO_32_V1)) 168 | (call $round32 169 | (i32.load (i32.add (local.get $statePtr) (global.get $SO_32_V1))) 170 | (i32.load (local.get $mem32Ptr)))) 171 | (i32.store 172 | (i32.add (local.get $statePtr) (global.get $SO_32_V2)) 173 | (call $round32 174 | (i32.load (i32.add (local.get $statePtr) (global.get $SO_32_V2))) 175 | (i32.load (i32.add (local.get $mem32Ptr) (i32.const 4))))) 176 | (i32.store 177 | (i32.add (local.get $statePtr) (global.get $SO_32_V3)) 178 | (call $round32 179 | (i32.load (i32.add (local.get $statePtr) (global.get $SO_32_V3))) 180 | (i32.load (i32.add (local.get $mem32Ptr) (i32.const 8))))) 181 | (i32.store 182 | (i32.add (local.get $statePtr) (global.get $SO_32_V4)) 183 | (call $round32 184 | (i32.load (i32.add (local.get $statePtr) (global.get $SO_32_V4))) 185 | (i32.load (i32.add (local.get $mem32Ptr) (i32.const 12))))) 186 | (local.set $inputPtr (i32.add (local.get $inputPtr) (i32.sub (i32.const 16) (local.get $initial-memsize)))) 187 | (i32.store (i32.add (local.get $statePtr) (global.get $SO_32_MEMSIZE)) (i32.const 0)))) 188 | (if (i32.le_u (local.get $inputPtr) (i32.sub (local.get $end) (i32.const 16))) 189 | (block 190 | (local.set $limit (i32.sub (local.get $end) (i32.const 16))) 191 | (local.set $v1 (i32.load (i32.add (local.get $statePtr) (global.get $SO_32_V1)))) 192 | (local.set $v2 (i32.load (i32.add (local.get $statePtr) (global.get $SO_32_V2)))) 193 | (local.set $v3 (i32.load (i32.add (local.get $statePtr) (global.get $SO_32_V3)))) 194 | (local.set $v4 (i32.load (i32.add (local.get $statePtr) (global.get $SO_32_V4)))) 195 | (loop $update-loop 196 | (local.set $v1 (call $round32 (local.get $v1) (i32.load (local.get $inputPtr)))) 197 | (local.set $inputPtr (i32.add (local.get $inputPtr) (i32.const 4))) 198 | (local.set $v2 (call $round32 (local.get $v2) (i32.load (local.get $inputPtr)))) 199 | (local.set $inputPtr (i32.add (local.get $inputPtr) (i32.const 4))) 200 | (local.set $v3 (call $round32 (local.get $v3) (i32.load (local.get $inputPtr)))) 201 | (local.set $inputPtr (i32.add (local.get $inputPtr) (i32.const 4))) 202 | (local.set $v4 (call $round32 (local.get $v4) (i32.load (local.get $inputPtr)))) 203 | (local.set $inputPtr (i32.add (local.get $inputPtr) (i32.const 4))) 204 | (br_if $update-loop (i32.le_u (local.get $inputPtr) (local.get $limit)))) 205 | (i32.store (i32.add (local.get $statePtr) (global.get $SO_32_V1)) (local.get $v1)) 206 | (i32.store (i32.add (local.get $statePtr) (global.get $SO_32_V2)) (local.get $v2)) 207 | (i32.store (i32.add (local.get $statePtr) (global.get $SO_32_V3)) (local.get $v3)) 208 | (i32.store (i32.add (local.get $statePtr) (global.get $SO_32_V4)) (local.get $v4)))) 209 | (if (i32.lt_u (local.get $inputPtr) (local.get $end)) 210 | (block 211 | (memory.copy 212 | (local.get $mem32Ptr) 213 | (local.get $inputPtr) 214 | (i32.sub (local.get $end) (local.get $inputPtr))) 215 | (i32.store (i32.add (local.get $statePtr) (global.get $SO_32_MEMSIZE)) 216 | (i32.sub (local.get $end) (local.get $inputPtr)))))) 217 | 218 | ;; Digest an XXH32 State struct into a hash value 219 | (func (export "digest32") (param $ptr i32) (result i32) 220 | (local $h32 i32) 221 | (local $v1 i32) 222 | (local $v2 i32) 223 | (local $v3 i32) 224 | (local $v4 i32) 225 | (local.set $v3 (i32.load (i32.add (local.get $ptr) (global.get $SO_32_V3)))) 226 | (if (i32.ne (i32.load (i32.add (local.get $ptr) (global.get $SO_32_LARGE_LEN))) (i32.const 0)) 227 | (block 228 | (local.set $v1 (i32.load (i32.add (local.get $ptr) (global.get $SO_32_V1)))) 229 | (local.set $v2 (i32.load (i32.add (local.get $ptr) (global.get $SO_32_V2)))) 230 | (local.set $v4 (i32.load (i32.add (local.get $ptr) (global.get $SO_32_V4)))) 231 | (local.set $h32 (i32.add 232 | (i32.rotl (local.get $v1) (i32.const 1)) 233 | (i32.add 234 | (i32.rotl (local.get $v2) (i32.const 7)) 235 | (i32.add 236 | (i32.rotl (local.get $v3) (i32.const 12)) 237 | (i32.rotl (local.get $v4) (i32.const 18))))))) 238 | (local.set $h32 (i32.add (local.get $v3) (global.get $PRIME32_5)))) 239 | (local.set $h32 (i32.add 240 | (local.get $h32) 241 | (i32.load (i32.add (local.get $ptr) (global.get $SO_32_TOTAL_LEN))))) 242 | (call $finalize32 243 | (local.get $h32) 244 | (i32.add (local.get $ptr) (global.get $SO_32_MEM32)) 245 | (i32.load (i32.add (local.get $ptr) (global.get $SO_32_MEMSIZE))))) 246 | 247 | ;; This is the actual WebAssembly implementation for one-shot XXH64. 248 | ;; $ptr indicates the beginning of the memory where the to-be-hashed data is stored. 249 | ;; $len is the length of the data. 250 | ;; $seed is the seed to be used in the hash invocation 251 | (func (export "xxh64") (param $ptr i32) (param $len i32) (param $seed i64) (result i64) 252 | (local $h64 i64) 253 | (local $end i32) 254 | (local $limit i32) 255 | (local $v1 i64) 256 | (local $v2 i64) 257 | (local $v3 i64) 258 | (local $v4 i64) 259 | (local.set $end (i32.add (local.get $ptr) (local.get $len))) 260 | (if 261 | (i32.ge_u (local.get $len) (i32.const 32)) 262 | (block 263 | (local.set $limit (i32.sub (local.get $end) (i32.const 32))) 264 | (local.set $v1 (i64.add (i64.add (local.get $seed) (global.get $PRIME64_1)) (global.get $PRIME64_2))) 265 | (local.set $v2 (i64.add (local.get $seed) (global.get $PRIME64_2))) 266 | (local.set $v3 (i64.add (local.get $seed) (i64.const 0))) 267 | (local.set $v4 (i64.sub (local.get $seed) (global.get $PRIME64_1))) 268 | ;; For every chunk of 4 words, so 4 * 64bits = 32 bytes 269 | (loop $4words-loop 270 | (local.set $v1 (call $round64 (local.get $v1) (i64.load (local.get $ptr)))) 271 | (local.set $ptr (i32.add (local.get $ptr) (i32.const 8))) 272 | (local.set $v2 (call $round64 (local.get $v2) (i64.load (local.get $ptr)))) 273 | (local.set $ptr (i32.add (local.get $ptr) (i32.const 8))) 274 | (local.set $v3 (call $round64 (local.get $v3) (i64.load (local.get $ptr)))) 275 | (local.set $ptr (i32.add (local.get $ptr) (i32.const 8))) 276 | (local.set $v4 (call $round64 (local.get $v4) (i64.load (local.get $ptr)))) 277 | (local.set $ptr (i32.add (local.get $ptr) (i32.const 8))) 278 | (br_if $4words-loop (i32.le_u (local.get $ptr) (local.get $limit)))) 279 | (local.set $h64 (i64.add 280 | (i64.rotl (local.get $v1) (i64.const 1)) 281 | (i64.add 282 | (i64.rotl (local.get $v2) (i64.const 7)) 283 | (i64.add 284 | (i64.rotl (local.get $v3) (i64.const 12)) 285 | (i64.rotl (local.get $v4) (i64.const 18)))))) 286 | (local.set $h64 (call $merge-round64 (local.get $h64) (local.get $v1))) 287 | (local.set $h64 (call $merge-round64 (local.get $h64) (local.get $v2))) 288 | (local.set $h64 (call $merge-round64 (local.get $h64) (local.get $v3))) 289 | (local.set $h64 (call $merge-round64 (local.get $h64) (local.get $v4)))) 290 | ;; else block, when input is smaller than 32 bytes 291 | (local.set $h64 (i64.add (local.get $seed) (global.get $PRIME64_5)))) 292 | (local.set $h64 (i64.add (local.get $h64) (i64.extend_i32_u (local.get $len)))) 293 | (call $finalize64 (local.get $h64) (local.get $ptr) (i32.and (local.get $len) (i32.const 31)))) 294 | 295 | (func $finalize64 (param $h64 i64) (param $ptr i32) (param $len i32) (result i64) 296 | (local $end i32) 297 | (local.set $end (i32.add (local.get $ptr) (local.get $len))) 298 | ;; For the remaining words not covered above, either 0, 1, 2 or 3 299 | (block $exit-remaining-words 300 | (loop $remaining-words-loop 301 | (br_if $exit-remaining-words (i32.gt_u (i32.add (local.get $ptr) (i32.const 8)) (local.get $end))) 302 | (local.set $h64 (i64.xor (local.get $h64) (call $round64 (i64.const 0) (i64.load (local.get $ptr))))) 303 | (local.set $h64 (i64.add 304 | (i64.mul 305 | (i64.rotl (local.get $h64) (i64.const 27)) 306 | (global.get $PRIME64_1)) 307 | (global.get $PRIME64_4))) 308 | (local.set $ptr (i32.add (local.get $ptr) (i32.const 8))) 309 | (br $remaining-words-loop))) 310 | ;; For the remaining half word. That is when there are more than 32bits 311 | ;; remaining which didn't make a whole word. 312 | (if 313 | (i32.le_u (i32.add (local.get $ptr) (i32.const 4)) (local.get $end)) 314 | (block 315 | (local.set $h64 (i64.xor (local.get $h64) (i64.mul (i64.load32_u (local.get $ptr)) (global.get $PRIME64_1)))) 316 | (local.set $h64 (i64.add 317 | (i64.mul 318 | (i64.rotl (local.get $h64) (i64.const 23)) 319 | (global.get $PRIME64_2)) 320 | (global.get $PRIME64_3))) 321 | (local.set $ptr (i32.add (local.get $ptr) (i32.const 4))))) 322 | ;; For the remaining bytes that didn't make a half a word (32bits), 323 | ;; either 0, 1, 2 or 3 bytes, as 4bytes = 32bits = 1/2 word. 324 | (block $exit-remaining-bytes 325 | (loop $remaining-bytes-loop 326 | (br_if $exit-remaining-bytes (i32.ge_u (local.get $ptr) (local.get $end))) 327 | (local.set $h64 (i64.xor (local.get $h64) (i64.mul (i64.load8_u (local.get $ptr)) (global.get $PRIME64_5)))) 328 | (local.set $h64 (i64.mul (i64.rotl (local.get $h64) (i64.const 11)) (global.get $PRIME64_1))) 329 | (local.set $ptr (i32.add (local.get $ptr) (i32.const 1))) 330 | (br $remaining-bytes-loop))) 331 | (local.set $h64 (i64.xor (local.get $h64) (i64.shr_u (local.get $h64) (i64.const 33)))) 332 | (local.set $h64 (i64.mul (local.get $h64) (global.get $PRIME64_2))) 333 | (local.set $h64 (i64.xor (local.get $h64) (i64.shr_u (local.get $h64) (i64.const 29)))) 334 | (local.set $h64 (i64.mul (local.get $h64) (global.get $PRIME64_3))) 335 | (local.set $h64 (i64.xor (local.get $h64) (i64.shr_u (local.get $h64) (i64.const 32)))) 336 | (local.get $h64)) 337 | 338 | (func $round64 (param $acc i64) (param $value i64) (result i64) 339 | (local.set $acc (i64.add (local.get $acc) (i64.mul (local.get $value) (global.get $PRIME64_2)))) 340 | (local.set $acc (i64.rotl (local.get $acc) (i64.const 31))) 341 | (local.set $acc (i64.mul (local.get $acc) (global.get $PRIME64_1))) 342 | (local.get $acc)) 343 | 344 | (func $merge-round64 (param $acc i64) (param $value i64) (result i64) 345 | (local.set $value (call $round64 (i64.const 0) (local.get $value))) 346 | (local.set $acc (i64.xor (local.get $acc) (local.get $value))) 347 | (local.set $acc (i64.add (i64.mul (local.get $acc) (global.get $PRIME64_1)) (global.get $PRIME64_4))) 348 | (local.get $acc)) 349 | 350 | ;; Initialize an XXH64 State Struct 351 | (func (export "init64") (param $statePtr i32) (param $seed i64) 352 | (i64.store 353 | (i32.add (local.get $statePtr) (global.get $SO_64_V1)) 354 | (i64.add (i64.add (local.get $seed) (global.get $PRIME64_1)) (global.get $PRIME64_2))) 355 | (i64.store 356 | (i32.add (local.get $statePtr) (global.get $SO_64_V2)) 357 | (i64.add (local.get $seed) (global.get $PRIME64_2))) 358 | (i64.store 359 | (i32.add (local.get $statePtr) (global.get $SO_64_V3)) 360 | (local.get $seed)) 361 | (i64.store 362 | (i32.add (local.get $statePtr) (global.get $SO_64_V4)) 363 | (i64.sub (local.get $seed) (global.get $PRIME64_1)))) 364 | 365 | ;; Update an XXH64 State Struct with an array of input bytes 366 | (func (export "update64") (param $statePtr i32) (param $inputPtr i32) (param $len i32) 367 | (local $end i32) 368 | (local $limit i32) 369 | (local $mem64Ptr i32) 370 | (local $initial-memsize i32) 371 | (local $v1 i64) 372 | (local $v2 i64) 373 | (local $v3 i64) 374 | (local $v4 i64) 375 | (local.set $end (i32.add (local.get $inputPtr) (local.get $len))) 376 | (local.set $mem64Ptr (i32.add (local.get $statePtr) (global.get $SO_64_MEM64))) 377 | (local.set $initial-memsize (i32.load (i32.add (local.get $statePtr) (global.get $SO_64_MEMSIZE)))) 378 | (i64.store (i32.add (local.get $statePtr) (global.get $SO_64_TOTAL_LEN)) 379 | (i64.add 380 | (i64.load (i32.add (local.get $statePtr) (global.get $SO_64_TOTAL_LEN))) 381 | (i64.extend_i32_u (local.get $len)))) 382 | (if (i32.lt_u 383 | (i32.add (local.get $len) (local.get $initial-memsize)) 384 | (i32.const 32)) 385 | (block 386 | (memory.copy 387 | (i32.add (local.get $mem64Ptr) (local.get $initial-memsize)) 388 | (local.get $inputPtr) 389 | (local.get $len)) 390 | (i32.store 391 | (i32.add (local.get $statePtr) (global.get $SO_64_MEMSIZE)) 392 | (i32.add (local.get $initial-memsize) (local.get $len))) 393 | (return))) 394 | (if (i32.ne (local.get $initial-memsize) (i32.const 0)) 395 | (block 396 | (memory.copy 397 | (i32.add (local.get $mem64Ptr) (local.get $initial-memsize)) 398 | (local.get $inputPtr) 399 | (i32.sub (i32.const 32) (local.get $initial-memsize))) 400 | (i64.store 401 | (i32.add (local.get $statePtr) (global.get $SO_64_V1)) 402 | (call $round64 403 | (i64.load (i32.add (local.get $statePtr) (global.get $SO_64_V1))) 404 | (i64.load (local.get $mem64Ptr)))) 405 | (i64.store 406 | (i32.add (local.get $statePtr) (global.get $SO_64_V2)) 407 | (call $round64 408 | (i64.load (i32.add (local.get $statePtr) (global.get $SO_64_V2))) 409 | (i64.load (i32.add (local.get $mem64Ptr) (i32.const 8))))) 410 | (i64.store 411 | (i32.add (local.get $statePtr) (global.get $SO_64_V3)) 412 | (call $round64 413 | (i64.load (i32.add (local.get $statePtr) (global.get $SO_64_V3))) 414 | (i64.load (i32.add (local.get $mem64Ptr) (i32.const 16))))) 415 | (i64.store 416 | (i32.add (local.get $statePtr) (global.get $SO_64_V4)) 417 | (call $round64 418 | (i64.load (i32.add (local.get $statePtr) (global.get $SO_64_V4))) 419 | (i64.load (i32.add (local.get $mem64Ptr) (i32.const 24))))) 420 | (local.set $inputPtr (i32.add (local.get $inputPtr) (i32.sub (i32.const 32) (local.get $initial-memsize)))) 421 | (i32.store (i32.add (local.get $statePtr) (global.get $SO_64_MEMSIZE)) (i32.const 0)))) 422 | (if (i32.le_u (i32.add (local.get $inputPtr) (i32.const 32)) (local.get $end)) 423 | (block 424 | (local.set $limit (i32.sub (local.get $end) (i32.const 32))) 425 | (local.set $v1 (i64.load (i32.add (local.get $statePtr) (global.get $SO_64_V1)))) 426 | (local.set $v2 (i64.load (i32.add (local.get $statePtr) (global.get $SO_64_V2)))) 427 | (local.set $v3 (i64.load (i32.add (local.get $statePtr) (global.get $SO_64_V3)))) 428 | (local.set $v4 (i64.load (i32.add (local.get $statePtr) (global.get $SO_64_V4)))) 429 | (loop $update-loop 430 | (local.set $v1 (call $round64 (local.get $v1) (i64.load (local.get $inputPtr)))) 431 | (local.set $inputPtr (i32.add (local.get $inputPtr) (i32.const 8))) 432 | (local.set $v2 (call $round64 (local.get $v2) (i64.load (local.get $inputPtr)))) 433 | (local.set $inputPtr (i32.add (local.get $inputPtr) (i32.const 8))) 434 | (local.set $v3 (call $round64 (local.get $v3) (i64.load (local.get $inputPtr)))) 435 | (local.set $inputPtr (i32.add (local.get $inputPtr) (i32.const 8))) 436 | (local.set $v4 (call $round64 (local.get $v4) (i64.load (local.get $inputPtr)))) 437 | (local.set $inputPtr (i32.add (local.get $inputPtr) (i32.const 8))) 438 | (br_if $update-loop (i32.le_u (local.get $inputPtr) (local.get $limit)))) 439 | (i64.store (i32.add (local.get $statePtr) (global.get $SO_64_V1)) (local.get $v1)) 440 | (i64.store (i32.add (local.get $statePtr) (global.get $SO_64_V2)) (local.get $v2)) 441 | (i64.store (i32.add (local.get $statePtr) (global.get $SO_64_V3)) (local.get $v3)) 442 | (i64.store (i32.add (local.get $statePtr) (global.get $SO_64_V4)) (local.get $v4)))) 443 | (if (i32.lt_u (local.get $inputPtr) (local.get $end)) 444 | (block 445 | (memory.copy 446 | (local.get $mem64Ptr) 447 | (local.get $inputPtr) 448 | (i32.sub (local.get $end) (local.get $inputPtr))) 449 | (i32.store 450 | (i32.add (local.get $statePtr) (global.get $SO_64_MEMSIZE)) 451 | (i32.sub (local.get $end) (local.get $inputPtr)))))) 452 | 453 | ;; Digest an XXH64 State struct into a hash value 454 | (func (export "digest64") (param $ptr i32) (result i64) 455 | (local $h64 i64) 456 | (local $v1 i64) 457 | (local $v2 i64) 458 | (local $v3 i64) 459 | (local $v4 i64) 460 | (local $total_len i64) 461 | (local.set $total_len (i64.load (i32.add (local.get $ptr) (global.get $SO_64_TOTAL_LEN)))) 462 | (local.set $v3 (i64.load (i32.add (local.get $ptr) (global.get $SO_64_V3)))) 463 | (if (i64.ge_u (local.get $total_len) (i64.const 32)) 464 | (block 465 | (local.set $v1 (i64.load (i32.add (local.get $ptr) (global.get $SO_64_V1)))) 466 | (local.set $v2 (i64.load (i32.add (local.get $ptr) (global.get $SO_64_V2)))) 467 | (local.set $v4 (i64.load (i32.add (local.get $ptr) (global.get $SO_64_V4)))) 468 | (local.set $h64 (i64.add 469 | (i64.add 470 | (i64.rotl (local.get $v1) (i64.const 1)) 471 | (i64.rotl (local.get $v2) (i64.const 7))) 472 | (i64.add 473 | (i64.rotl (local.get $v3) (i64.const 12)) 474 | (i64.rotl (local.get $v4) (i64.const 18))))) 475 | (local.set $h64 (call $merge-round64 (local.get $h64) (local.get $v1))) 476 | (local.set $h64 (call $merge-round64 (local.get $h64) (local.get $v2))) 477 | (local.set $h64 (call $merge-round64 (local.get $h64) (local.get $v3))) 478 | (local.set $h64 (call $merge-round64 (local.get $h64) (local.get $v4)))) 479 | (local.set $h64 (i64.add (local.get $v3) (global.get $PRIME64_5)))) 480 | (local.set $h64 (i64.add (local.get $h64) (local.get $total_len))) 481 | (call $finalize64 482 | (local.get $h64) 483 | (i32.add (local.get $ptr) (global.get $SO_64_MEM64)) 484 | (i32.wrap_i64 (i64.and (local.get $total_len) (i64.const 31)))))) 485 | 486 | 487 | -------------------------------------------------------------------------------- /test/index.test.js: -------------------------------------------------------------------------------- 1 | import test from "jest-t-assert"; 2 | import xxhash from "../src"; 3 | 4 | // The test cases were taken from tests of other implementation and the 5 | // resulting hashes have been generated by running another implementation. 6 | // All cases used the seed 0. 7 | const testCases = [ 8 | { input: "", h32: "02cc5d05", h64: "ef46db3751d8e999" }, 9 | { input: "a", h32: "550d7456", h64: "d24ec4f1a98c6e5b" }, 10 | { input: "as", h32: "9d5a0464", h64: "1c330fb2d66be179" }, 11 | { input: "asd", h32: "3d83552b", h64: "631c37ce72a97393" }, 12 | { input: "asdf", h32: "5e702c32", h64: "415872f599cea71e" }, 13 | { input: "abc", h32: "32d153ff", h64: "44bc2cf5ad770999" }, 14 | { input: "abcd", h32: "a3643705", h64: "de0327b0d25d92cc" }, 15 | { input: "0.7278296545100061", h32: "432c173f", h64: "596877150e8ee48c" }, 16 | { 17 | input: "Call me Ishmael. Some years ago--never mind how long precisely-", 18 | h32: "6f320359", 19 | h64: "02a2e85470d6fd96", 20 | }, 21 | { 22 | input: 23 | "The quick brown fox jumps over the lazy dog http://i.imgur.com/VHQXScB.gif", 24 | h32: "5ce7b616", 25 | h64: "93267f9820452ead", 26 | }, 27 | { input: "heiå", h32: "db5abccc", h64: "b9d3d990d2001a1a" }, 28 | { input: "κόσμε", h32: "d855f606", h64: "a0488960c70d8772" }, 29 | ]; 30 | 31 | for (const testCase of testCases) { 32 | test(`h32 of ${testCase.input}`, async (t) => { 33 | const hasher = await xxhash(); 34 | const h32 = hasher.h32(testCase.input).toString(16).padStart(8, "0"); 35 | t.is(h32, testCase.h32); 36 | }); 37 | 38 | test(`h32ToString of ${testCase.input}`, async (t) => { 39 | const hasher = await xxhash(); 40 | const h32 = hasher.h32ToString(testCase.input); 41 | t.is(h32, testCase.h32); 42 | }); 43 | 44 | test(`h32Raw of ${testCase.input}`, async (t) => { 45 | const encoder = new TextEncoder(); 46 | const hasher = await xxhash(); 47 | const h32 = hasher 48 | .h32Raw(encoder.encode(testCase.input)) 49 | .toString(16) 50 | .padStart(8, "0"); 51 | t.is(h32, testCase.h32); 52 | }); 53 | 54 | test(`streamed h32 of ${testCase.input}`, async (t) => { 55 | const hasher = await xxhash(); 56 | const h32 = hasher 57 | .create32() 58 | .update(testCase.input) 59 | .digest() 60 | .toString(16) 61 | .padStart(8, "0"); 62 | t.is(h32, testCase.h32); 63 | }); 64 | 65 | test(`h64 of ${testCase.input}`, async (t) => { 66 | const hasher = await xxhash(); 67 | const h64 = hasher.h64(testCase.input); 68 | t.is(h64, BigInt(`0x${testCase.h64}`)); 69 | }); 70 | 71 | test(`h64ToString of ${testCase.input}`, async (t) => { 72 | const hasher = await xxhash(); 73 | const h64 = hasher.h64ToString(testCase.input); 74 | t.is(h64, testCase.h64); 75 | }); 76 | 77 | test(`h64Raw of ${testCase.input}`, async (t) => { 78 | const encoder = new TextEncoder(); 79 | const hasher = await xxhash(); 80 | const h64 = hasher.h64Raw(encoder.encode(testCase.input)); 81 | t.is(h64, BigInt(`0x${testCase.h64}`)); 82 | }); 83 | 84 | test(`streamed h64 of ${testCase.input}`, async (t) => { 85 | const hasher = await xxhash(); 86 | const h64 = hasher 87 | .create64() 88 | .update(testCase.input) 89 | .digest() 90 | .toString(16) 91 | .padStart(16, "0"); 92 | t.is(h64, testCase.h64); 93 | }); 94 | } 95 | 96 | test("h32 with different seeds produces different hashes", async (t) => { 97 | const hasher = await xxhash(); 98 | const input = "different seeds"; 99 | const h320 = hasher.h32(input, 0); 100 | const h32abcd = hasher.h32(input, 0xabcd); 101 | t.not(h320, h32abcd); 102 | }); 103 | 104 | test("h64 with different seeds produces different hashes", async (t) => { 105 | const hasher = await xxhash(); 106 | const input = "different seeds"; 107 | const h640 = hasher.h64(input, 0n); 108 | const h64lowAbcd = hasher.h64(input, BigInt(0xabcd)); 109 | const h64highAbcd = hasher.h64(input, BigInt(0xabcd) << (32n + BigInt(0))); 110 | t.not(h640, h64lowAbcd); 111 | t.not(h640, h64highAbcd); 112 | t.not(h64lowAbcd, h64highAbcd); 113 | }); 114 | 115 | test("a string greater than the initial memory size works", async (t) => { 116 | const hasher = await xxhash(); 117 | const bytesPerPage = 64 * 1024; 118 | const input = "z".repeat(bytesPerPage + 1); 119 | const h32 = hasher.h32ToString(input, 0); 120 | const h64 = hasher.h64ToString(input, 0n); 121 | t.is(h32, "7871ee9b"); 122 | t.is(h64, "68278ba56dc14510"); 123 | }); 124 | 125 | test("streamed h32 with multiple inputs produces same hash", async (t) => { 126 | const hasher = await xxhash(); 127 | const { update, digest } = hasher.create32(0); 128 | update("hello"); 129 | update("world"); 130 | t.is(digest(), hasher.h32("helloworld")); 131 | }); 132 | 133 | test("streamed h64 with multiple inputs produces same hash", async (t) => { 134 | const hasher = await xxhash(); 135 | const { update, digest } = hasher.create64(0n); 136 | update("hello"); 137 | update("world"); 138 | t.is(digest(), hasher.h64("helloworld")); 139 | }); 140 | 141 | test("streamed h32 with buffer input produces same hash", async (t) => { 142 | const input = Buffer.from("helloworld"); 143 | const hasher = await xxhash(); 144 | t.is(hasher.create32(0).update(input).digest(), hasher.h32Raw(input)); 145 | }); 146 | 147 | test("streamed h64 with buffer input produces same hash", async (t) => { 148 | const input = Buffer.from("helloworld"); 149 | const hasher = await xxhash(); 150 | t.is(hasher.create64(0n).update(input).digest(), hasher.h64Raw(input)); 151 | }); 152 | -------------------------------------------------------------------------------- /test/setup.js: -------------------------------------------------------------------------------- 1 | // The build system includes the xxhash.wasm. Just loading the source will not 2 | // work because WASM_PRECOMPILED_BYTES is not defined. To make it work in the 3 | // tests, xxhash.wasm is read from the file system and assigned to global. 4 | const { readFileSync } = require("fs"); 5 | const { resolve } = require("path"); 6 | 7 | const wasmBytes = readFileSync(resolve(__dirname, "../src/xxhash.wasm")); 8 | 9 | global.WASM_PRECOMPILED_BYTES = Array.from(wasmBytes); 10 | -------------------------------------------------------------------------------- /types.d.ts: -------------------------------------------------------------------------------- 1 | export type XXHash = { 2 | update(input: string | Uint8Array): XXHash; 3 | digest(): T; 4 | } 5 | 6 | export type XXHashAPI = { 7 | h32(input: string, seed?: number): number; 8 | h32ToString(input: string, seed?: number): string; 9 | h32Raw(inputBuffer: Uint8Array, seed?: number): number; 10 | create32(seed?: number): XXHash; 11 | h64(input: string, seed?: bigint): bigint; 12 | h64ToString(input: string, seed?: bigint): string; 13 | h64Raw(inputBuffer: Uint8Array, seed?: bigint): bigint; 14 | create64(seed?: bigint): XXHash; 15 | }; 16 | 17 | declare module "xxhash-wasm" { 18 | export default function xxhash(): Promise; 19 | } 20 | --------------------------------------------------------------------------------