├── .github └── workflows │ └── test-node.yml ├── .gitignore ├── LICENSE ├── README.md ├── index.js ├── package.json └── test.js /.github/workflows/test-node.yml: -------------------------------------------------------------------------------- 1 | name: Build Status 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | branches: 8 | - master 9 | jobs: 10 | build: 11 | strategy: 12 | matrix: 13 | node-version: [lts/*] 14 | os: [ubuntu-latest, macos-latest, windows-latest] 15 | runs-on: ${{ matrix.os }} 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Use Node.js ${{ matrix.node-version }} 19 | uses: actions/setup-node@v2 20 | with: 21 | node-version: ${{ matrix.node-version }} 22 | - run: npm install 23 | - run: npm test 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | sandbox.js 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2020 Mathias Buus 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # big-sparse-array 2 | 3 | A sparse array optimised for low memory whilst still being fast. 4 | 5 | ``` 6 | npm install big-sparse-array 7 | ``` 8 | 9 | Uses a shallow tree structure with a branching factor of 4096 10 | to index a series of small arrays that try to compress as much as possible 11 | to reduce the memory overhead needed. 12 | 13 | Similar to a Map, except it is faster, but might use a bit more memory, ymmv. 14 | 15 | ## Usage 16 | 17 | ``` js 18 | const BigSparseArray = require('big-sparse-array') 19 | 20 | const b = new BigSparseArray() 21 | 22 | b.set(42422242525, true) 23 | b.get(42422242525) // returns true 24 | b.get(111111111) // returns undefined 25 | ``` 26 | 27 | ## API 28 | 29 | #### `const b = new BigSparseArray()` 30 | 31 | Make a new sparse array. 32 | 33 | #### `b.set(index, value)` 34 | 35 | Insert a new value at an index. `index` must be a integer. 36 | 37 | #### `value = b.get(index)` 38 | 39 | Get a value out. Returns `undefined` if the value could not be found. 40 | 41 | ## License 42 | 43 | MIT 44 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const FACTOR = new Uint16Array(8) 2 | 3 | function factor4096 (i, n) { 4 | while (n > 0) { 5 | const f = i & 4095 6 | FACTOR[--n] = f 7 | i = (i - f) / 4096 8 | } 9 | return FACTOR 10 | } 11 | 12 | module.exports = class BigSparseArray { 13 | constructor () { 14 | this.tiny = new TinyArray() 15 | this.maxLength = 4096 16 | this.factor = 1 17 | } 18 | 19 | set (index, val) { 20 | if (val !== undefined) { 21 | while (index >= this.maxLength) { 22 | this.maxLength *= 4096 23 | this.factor++ 24 | if (!this.tiny.isEmptyish()) { 25 | const t = new TinyArray() 26 | t.set(0, this.tiny) 27 | this.tiny = t 28 | } 29 | } 30 | } 31 | 32 | const f = factor4096(index, this.factor) 33 | const last = this.factor - 1 34 | 35 | let tiny = this.tiny 36 | for (let i = 0; i < last; i++) { 37 | const next = tiny.get(f[i]) 38 | if (next === undefined) { 39 | if (val === undefined) return 40 | tiny = tiny.set(f[i], new TinyArray()) 41 | } else { 42 | tiny = next 43 | } 44 | } 45 | 46 | return tiny.set(f[last], val) 47 | } 48 | 49 | get (index) { 50 | if (index >= this.maxLength) return 51 | 52 | const f = factor4096(index, this.factor) 53 | const last = this.factor - 1 54 | 55 | let tiny = this.tiny 56 | for (let i = 0; i < last; i++) { 57 | tiny = tiny.get(f[i]) 58 | if (tiny === undefined) return 59 | } 60 | 61 | return tiny.get(f[last]) 62 | } 63 | } 64 | 65 | class TinyArray { 66 | constructor () { 67 | this.s = 0 68 | this.b = new Array(1) 69 | this.f = new Uint16Array(1) 70 | } 71 | 72 | isEmptyish () { 73 | return this.b.length === 1 && this.b[0] === undefined 74 | } 75 | 76 | get (i) { 77 | if (this.s === 12) return this.b[i] 78 | const f = i >>> this.s 79 | const r = i & (this.b.length - 1) 80 | return this.f[r] === f ? this.b[r] : undefined 81 | } 82 | 83 | set (i, v) { 84 | while (this.s !== 12) { 85 | const f = i >>> this.s 86 | const r = i & (this.b.length - 1) 87 | const o = this.b[r] 88 | 89 | if (o === undefined || f === this.f[r]) { 90 | this.b[r] = v 91 | this.f[r] = f 92 | return v 93 | } 94 | 95 | this.grow() 96 | } 97 | 98 | this.b[i] = v 99 | return v 100 | } 101 | 102 | grow () { 103 | const os = this.s 104 | const ob = this.b 105 | const of = this.f 106 | 107 | this.s += 4 108 | this.b = new Array(this.b.length << 4) 109 | this.f = this.s === 12 ? null : new Uint8Array(this.b.length) 110 | 111 | const m = this.b.length - 1 112 | 113 | for (let or = 0; or < ob.length; or++) { 114 | if (ob[or] === undefined) continue 115 | 116 | const i = of[or] << os | or 117 | const f = i >>> this.s 118 | const r = i & m 119 | 120 | this.b[r] = ob[or] 121 | if (this.s !== 12) this.f[r] = f 122 | } 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "big-sparse-array", 3 | "version": "1.0.3", 4 | "description": "A sparse array optimised for low memory whilst still being fast", 5 | "main": "index.js", 6 | "dependencies": {}, 7 | "devDependencies": { 8 | "brittle": "^3.1.1", 9 | "standard": "^16.0.3" 10 | }, 11 | "scripts": { 12 | "test": "standard && brittle test.js" 13 | }, 14 | "repository": { 15 | "type": "git", 16 | "url": "https://github.com/mafintosh/big-sparse-array.git" 17 | }, 18 | "author": "Mathias Buus (@mafintosh)", 19 | "license": "MIT", 20 | "bugs": { 21 | "url": "https://github.com/mafintosh/big-sparse-array/issues" 22 | }, 23 | "homepage": "https://github.com/mafintosh/big-sparse-array" 24 | } 25 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | const test = require('brittle') 2 | const BigSparseArray = require('./') 3 | 4 | test('basic', function (t) { 5 | const b = new BigSparseArray() 6 | 7 | b.set(42, true) 8 | t.is(b.get(42), true) 9 | 10 | b.set(42, 42) 11 | t.is(b.get(42), 42) 12 | 13 | b.set(42424242424242, 'big') 14 | t.is(b.get(42424242424242), 'big') 15 | t.is(b.get(42424242424243), undefined) 16 | t.is(b.get(42), 42) 17 | }) 18 | 19 | test('grow', function (t) { 20 | const b = new BigSparseArray() 21 | 22 | for (let i = 0; i < 10000; i++) { 23 | b.set(i, i) 24 | } 25 | 26 | let missing = 10000 27 | 28 | for (let i = 0; i < 10000; i++) { 29 | if (b.get(i) === i) missing-- 30 | } 31 | 32 | t.is(missing, 0) 33 | }) 34 | 35 | test('bounds', function (t) { 36 | const b = new BigSparseArray() 37 | 38 | b.set(0, true) 39 | 40 | t.is(b.get(4096), undefined) 41 | }) 42 | --------------------------------------------------------------------------------