├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── index.js ├── package.json └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - '4' 4 | - '6' 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Mathias Buus 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # memory-pager 2 | 3 | Access memory using small fixed sized buffers instead of allocating a huge buffer. 4 | Useful if you are implementing sparse data structures (such as large bitfield). 5 | 6 | ![travis](https://travis-ci.org/mafintosh/memory-pager.svg?branch=master) 7 | 8 | ``` 9 | npm install memory-pager 10 | ``` 11 | 12 | ## Usage 13 | 14 | ``` js 15 | var pager = require('paged-memory') 16 | 17 | var pages = pager(1024) // use 1kb per page 18 | 19 | var page = pages.get(10) // get page #10 20 | 21 | console.log(page.offset) // 10240 22 | console.log(page.buffer) // a blank 1kb buffer 23 | ``` 24 | 25 | ## API 26 | 27 | #### `var pages = pager(pageSize)` 28 | 29 | Create a new pager. `pageSize` defaults to `1024`. 30 | 31 | #### `var page = pages.get(pageNumber, [noAllocate])` 32 | 33 | Get a page. The page will be allocated at first access. 34 | 35 | Optionally you can set the `noAllocate` flag which will make the 36 | method return undefined if no page has been allocated already 37 | 38 | A page looks like this 39 | 40 | ``` js 41 | { 42 | offset: byteOffset, 43 | buffer: bufferWithPageSize 44 | } 45 | ``` 46 | 47 | #### `pages.set(pageNumber, buffer)` 48 | 49 | Explicitly set the buffer for a page. 50 | 51 | #### `pages.updated(page)` 52 | 53 | Mark a page as updated. 54 | 55 | #### `pages.lastUpdate()` 56 | 57 | Get the last page that was updated. 58 | 59 | #### `var buf = pages.toBuffer()` 60 | 61 | Concat all pages allocated pages into a single buffer 62 | 63 | ## License 64 | 65 | MIT 66 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | module.exports = Pager 2 | 3 | function Pager (pageSize, opts) { 4 | if (!(this instanceof Pager)) return new Pager(pageSize, opts) 5 | 6 | this.length = 0 7 | this.updates = [] 8 | this.path = new Uint16Array(4) 9 | this.pages = new Array(32768) 10 | this.maxPages = this.pages.length 11 | this.level = 0 12 | this.pageSize = pageSize || 1024 13 | this.deduplicate = opts ? opts.deduplicate : null 14 | this.zeros = this.deduplicate ? alloc(this.deduplicate.length) : null 15 | } 16 | 17 | Pager.prototype.updated = function (page) { 18 | while (this.deduplicate && page.buffer[page.deduplicate] === this.deduplicate[page.deduplicate]) { 19 | page.deduplicate++ 20 | if (page.deduplicate === this.deduplicate.length) { 21 | page.deduplicate = 0 22 | if (page.buffer.equals && page.buffer.equals(this.deduplicate)) page.buffer = this.deduplicate 23 | break 24 | } 25 | } 26 | if (page.updated || !this.updates) return 27 | page.updated = true 28 | this.updates.push(page) 29 | } 30 | 31 | Pager.prototype.lastUpdate = function () { 32 | if (!this.updates || !this.updates.length) return null 33 | var page = this.updates.pop() 34 | page.updated = false 35 | return page 36 | } 37 | 38 | Pager.prototype._array = function (i, noAllocate) { 39 | if (i >= this.maxPages) { 40 | if (noAllocate) return 41 | grow(this, i) 42 | } 43 | 44 | factor(i, this.path) 45 | 46 | var arr = this.pages 47 | 48 | for (var j = this.level; j > 0; j--) { 49 | var p = this.path[j] 50 | var next = arr[p] 51 | 52 | if (!next) { 53 | if (noAllocate) return 54 | next = arr[p] = new Array(32768) 55 | } 56 | 57 | arr = next 58 | } 59 | 60 | return arr 61 | } 62 | 63 | Pager.prototype.get = function (i, noAllocate) { 64 | var arr = this._array(i, noAllocate) 65 | var first = this.path[0] 66 | var page = arr && arr[first] 67 | 68 | if (!page && !noAllocate) { 69 | page = arr[first] = new Page(i, alloc(this.pageSize)) 70 | if (i >= this.length) this.length = i + 1 71 | } 72 | 73 | if (page && page.buffer === this.deduplicate && this.deduplicate && !noAllocate) { 74 | page.buffer = copy(page.buffer) 75 | page.deduplicate = 0 76 | } 77 | 78 | return page 79 | } 80 | 81 | Pager.prototype.set = function (i, buf) { 82 | var arr = this._array(i, false) 83 | var first = this.path[0] 84 | 85 | if (i >= this.length) this.length = i + 1 86 | 87 | if (!buf || (this.zeros && buf.equals && buf.equals(this.zeros))) { 88 | arr[first] = undefined 89 | return 90 | } 91 | 92 | if (this.deduplicate && buf.equals && buf.equals(this.deduplicate)) { 93 | buf = this.deduplicate 94 | } 95 | 96 | var page = arr[first] 97 | var b = truncate(buf, this.pageSize) 98 | 99 | if (page) page.buffer = b 100 | else arr[first] = new Page(i, b) 101 | } 102 | 103 | Pager.prototype.toBuffer = function () { 104 | var list = new Array(this.length) 105 | var empty = alloc(this.pageSize) 106 | var ptr = 0 107 | 108 | while (ptr < list.length) { 109 | var arr = this._array(ptr, true) 110 | for (var i = 0; i < 32768 && ptr < list.length; i++) { 111 | list[ptr++] = (arr && arr[i]) ? arr[i].buffer : empty 112 | } 113 | } 114 | 115 | return Buffer.concat(list) 116 | } 117 | 118 | function grow (pager, index) { 119 | while (pager.maxPages < index) { 120 | var old = pager.pages 121 | pager.pages = new Array(32768) 122 | pager.pages[0] = old 123 | pager.level++ 124 | pager.maxPages *= 32768 125 | } 126 | } 127 | 128 | function truncate (buf, len) { 129 | if (buf.length === len) return buf 130 | if (buf.length > len) return buf.slice(0, len) 131 | var cpy = alloc(len) 132 | buf.copy(cpy) 133 | return cpy 134 | } 135 | 136 | function alloc (size) { 137 | if (Buffer.alloc) return Buffer.alloc(size) 138 | var buf = new Buffer(size) 139 | buf.fill(0) 140 | return buf 141 | } 142 | 143 | function copy (buf) { 144 | var cpy = Buffer.allocUnsafe ? Buffer.allocUnsafe(buf.length) : new Buffer(buf.length) 145 | buf.copy(cpy) 146 | return cpy 147 | } 148 | 149 | function Page (i, buf) { 150 | this.offset = i * buf.length 151 | this.buffer = buf 152 | this.updated = false 153 | this.deduplicate = 0 154 | } 155 | 156 | function factor (n, out) { 157 | n = (n - (out[0] = (n & 32767))) / 32768 158 | n = (n - (out[1] = (n & 32767))) / 32768 159 | out[3] = ((n - (out[2] = (n & 32767))) / 32768) & 32767 160 | } 161 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "memory-pager", 3 | "version": "1.5.0", 4 | "description": "Access memory using small fixed sized buffers", 5 | "main": "index.js", 6 | "dependencies": {}, 7 | "devDependencies": { 8 | "standard": "^9.0.0", 9 | "tape": "^4.6.3" 10 | }, 11 | "scripts": { 12 | "test": "standard && tape test.js" 13 | }, 14 | "repository": { 15 | "type": "git", 16 | "url": "https://github.com/mafintosh/memory-pager.git" 17 | }, 18 | "author": "Mathias Buus (@mafintosh)", 19 | "license": "MIT", 20 | "bugs": { 21 | "url": "https://github.com/mafintosh/memory-pager/issues" 22 | }, 23 | "homepage": "https://github.com/mafintosh/memory-pager" 24 | } 25 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | var tape = require('tape') 2 | var pager = require('./') 3 | 4 | tape('get page', function (t) { 5 | var pages = pager(1024) 6 | 7 | var page = pages.get(0) 8 | 9 | t.same(page.offset, 0) 10 | t.same(page.buffer, Buffer.alloc(1024)) 11 | t.end() 12 | }) 13 | 14 | tape('get page twice', function (t) { 15 | var pages = pager(1024) 16 | t.same(pages.length, 0) 17 | 18 | var page = pages.get(0) 19 | 20 | t.same(page.offset, 0) 21 | t.same(page.buffer, Buffer.alloc(1024)) 22 | t.same(pages.length, 1) 23 | 24 | var other = pages.get(0) 25 | 26 | t.same(other, page) 27 | t.end() 28 | }) 29 | 30 | tape('get no mutable page', function (t) { 31 | var pages = pager(1024) 32 | 33 | t.ok(!pages.get(141, true)) 34 | t.ok(pages.get(141)) 35 | t.ok(pages.get(141, true)) 36 | 37 | t.end() 38 | }) 39 | 40 | tape('get far out page', function (t) { 41 | var pages = pager(1024) 42 | 43 | var page = pages.get(1000000) 44 | 45 | t.same(page.offset, 1000000 * 1024) 46 | t.same(page.buffer, Buffer.alloc(1024)) 47 | t.same(pages.length, 1000000 + 1) 48 | 49 | var other = pages.get(1) 50 | 51 | t.same(other.offset, 1024) 52 | t.same(other.buffer, Buffer.alloc(1024)) 53 | t.same(pages.length, 1000000 + 1) 54 | t.ok(other !== page) 55 | 56 | t.end() 57 | }) 58 | 59 | tape('updates', function (t) { 60 | var pages = pager(1024) 61 | 62 | t.same(pages.lastUpdate(), null) 63 | 64 | var page = pages.get(10) 65 | 66 | page.buffer[42] = 1 67 | pages.updated(page) 68 | 69 | t.same(pages.lastUpdate(), page) 70 | t.same(pages.lastUpdate(), null) 71 | 72 | page.buffer[42] = 2 73 | pages.updated(page) 74 | pages.updated(page) 75 | 76 | t.same(pages.lastUpdate(), page) 77 | t.same(pages.lastUpdate(), null) 78 | 79 | t.end() 80 | }) 81 | --------------------------------------------------------------------------------