├── .gitignore ├── LICENSE ├── README.md ├── bench.js ├── frame ├── basic.js ├── offset-codecs.js └── recoverable.js ├── index.js ├── inject.js ├── legacy.js ├── package-lock.json ├── package.json └── test ├── flumelog.js ├── restore.js ├── simple.js └── stream.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 'Dominic Tarr' 2 | 3 | Permission is hereby granted, free of charge, 4 | to any person obtaining a copy of this software and 5 | associated documentation files (the "Software"), to 6 | deal in the Software without restriction, including 7 | without limitation the rights to use, copy, modify, 8 | merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom 10 | the Software is furnished to do so, 11 | subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice 14 | shall be included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 18 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR 20 | ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 21 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 22 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # flumelog-offset 2 | 3 | An flumelog where the offset into the file is the key. 4 | Each value is appended to the log with a double ended framing, 5 | and the "sequence" is the position in the physical file where the value starts, 6 | this means if you can do a read in O(1) time! 7 | 8 | Also, this is built on top of [aligned-block-file](https://github.com/flumedb/aligned-block-file) 9 | so that caching works very well. 10 | 11 | ## Usage 12 | 13 | initialize with a file and a codec, and wrap with flumedb. 14 | 15 | ``` js 16 | var OffsetLog = require('flumelog-offset') 17 | var codec = require('flumecodec') 18 | var Flume = require('flumedb') 19 | 20 | var db = Flume(OffsetLog(filename, {codec: codec.json})) 21 | .use(...) //also add some flumeviews 22 | 23 | db.append({greets: 'hello!'}, function (cb) { 24 | 25 | }) 26 | 27 | ``` 28 | 29 | ## Options 30 | 31 | ``` 32 | var OffsetLog = require('flumelog-offset') 33 | var log = OffsetLog('/data/log', { 34 | blockSize: 1024, // default is 1024*16 35 | codec: {encode, decode} // defaults to no codec, expects buffers. for json use flumecodec/json 36 | flags: 'r', // default is 'r+' (from aligned-block-file) 37 | cache: {set, get} // default is require('hashlru')(1024) 38 | offsetCodec: { // default is require('./frame/offset-codecs')[32] 39 | byteWidth, // with the default offset-codec, the file can have 40 | encode, // a size of 4GB max. 41 | decodeAsync 42 | } 43 | }) 44 | ``` 45 | 46 | ## legacy 47 | 48 | if you used `flumelog-offset` before 3, and want to read your old 49 | data, use `require('flumelog-offset/legacy')` 50 | 51 | 52 | ## recovery 53 | 54 | If your system crashes while an append is in progress, it's unlikely 55 | but possible to have a partially written state. `flumelog-offset` 56 | will rewind to the last good state on the next start up. 57 | 58 | After running this for several months (in my personal secure-scuttlebutt 59 | instance) I eventually got an error, which lead to the changes 60 | in this version. 61 | 62 | ## format 63 | 64 | data is stored in a append only log, where the byte index 65 | of the start of a record is the primary key (`offset`). 66 | 67 | ``` 68 | offset-> 69 | 70 | 71 | 72 | ``` 73 | by writing the length of the data both before and after each record 74 | it becomes possible to scan forward and backward (like a doubly linked list) 75 | 76 | It's very handly to be able to scan backwards, as often you want 77 | to see the last N items, and so you don't need an index for this. 78 | 79 | ## future ideas 80 | 81 | * secured file (hashes etc) 82 | * encrypted file 83 | * make the end of the record be the primary key. 84 | this might make other code nicer... 85 | 86 | ## License 87 | 88 | MIT 89 | -------------------------------------------------------------------------------- /bench.js: -------------------------------------------------------------------------------- 1 | var FlumeLog = require('./') 2 | var codec = require('flumecodec') 3 | 4 | require('bench-flumelog')(function () { 5 | return FlumeLog('/tmp/bench-flumelog-offset' + Date.now(), { 6 | blockSize: 1024*64, 7 | codec: codec.json 8 | }) 9 | }, null, null, function (obj) { 10 | return obj 11 | // return Buffer.from(JSON.stringify(obj)) 12 | }) 13 | 14 | -------------------------------------------------------------------------------- /frame/basic.js: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | 4 | */ 5 | module.exports = function (blocks, codec) { 6 | 7 | function frame (data, _offset) { 8 | var length = data.reduce(function (total, value) { return total + value.length }, 0) 9 | var b = Buffer.alloc(length + data.length * 8) 10 | var offset = 0 11 | for(var i = 0; i < data.length; i++) { 12 | var buf = data[i] 13 | //mutate the items 14 | //var buf = item.value 15 | b.writeUInt32BE(buf.length, 0 + offset) //start 16 | b.writeUInt32BE(buf.length, 4+buf.length + offset) //end 17 | buf.copy(b, 4 + offset, 0, buf.length) 18 | //remember the offset of the _last_ item 19 | //this is for framings starting at the first byte. 20 | frame.offset = _offset + offset 21 | offset += buf.length + 8 22 | } 23 | 24 | return b 25 | } 26 | 27 | function getMeta (offset, cb) { 28 | //special case for first value 29 | if(offset === 0) 30 | blocks.readUInt32BE(0, function (err, length) { 31 | if(err) return cb(err) 32 | blocks.read(4, 4+length, function (err, value) { 33 | cb(err, value, -1, 4+length+4) 34 | }) 35 | }) 36 | else 37 | blocks.readUInt32BE(offset, function (err, len) { 38 | if(err) return cb(err) 39 | 40 | //read the length of the previous item. 41 | //unless this falls right on the overlap between 42 | //two blocks, this will already be in the cache, 43 | //so will be just a mem read. 44 | blocks.readUInt32BE(offset - 4, function (err, prev_len) { 45 | if(err) return cb(err) 46 | blocks.read(offset+4, offset+4+len, function (err, value) { 47 | cb(err, value, offset-(4+prev_len+4), offset+(4+len+4)) 48 | }) 49 | }) 50 | }) 51 | 52 | } 53 | 54 | //restore the previous positon, used to set the first offset. 55 | function restore (cb) { 56 | //basic doesn't have a good way to do this, 57 | //except check the latest item, and error if it's broke 58 | //though could recopy the entire log then mv it over... 59 | blocks.offset.once(function (offset) { 60 | if(offset === 0) return cb(null, -1) 61 | 62 | blocks.readUInt32BE(offset - 4, function (err, len) { 63 | var _offset = offset - (4+len+4) 64 | getMeta(_offset, function (err, value) { 65 | if(err) return cb(err) 66 | try { 67 | codec.decode(value) 68 | } catch (err) { 69 | return cb(err) 70 | } 71 | cb(null, _offset) 72 | }) 73 | }) 74 | }) 75 | 76 | } 77 | 78 | return { 79 | frame: frame, 80 | getMeta: getMeta, 81 | restore: restore 82 | } 83 | } 84 | 85 | -------------------------------------------------------------------------------- /frame/offset-codecs.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | var uint48be = require('uint48be') 3 | var int53 = require('int53') 4 | 5 | module.exports = { 6 | 32: { 7 | byteWidth: 4, 8 | encode: function(buf, value, offset) { 9 | buf.writeUInt32BE(value, offset) 10 | }, 11 | decode: function(buf, offset) { 12 | return buf.readUInt32BE(offset) 13 | }, 14 | decodeAsync: function(blocks, offset, cb) { 15 | blocks.readUInt32BE(offset, cb) 16 | } 17 | }, 18 | 48: { 19 | byteWidth: 6, 20 | encode: function(buf, value, offset) { 21 | uint48be.encode(value, buf, offset) 22 | }, 23 | decode: function(buf, offset) { 24 | return uint48be.decode(buf, offset) 25 | }, 26 | decodeAsync: function(blocks, offset, cb) { 27 | blocks.readUInt48BE(offset, cb) 28 | } 29 | }, 30 | 53: { 31 | byteWidth: 8, 32 | encode: function(buf, value, offset) { 33 | int53.writeUInt64BE(value, buf, offset) 34 | }, 35 | decode: function(buf, offset) { 36 | return int53.readUInt64BE(buf, offset) 37 | }, 38 | decodeAsync: function(blocks, offset, cb) { 39 | blocks.readUInt64BE(offset, cb) 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /frame/recoverable.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | var Looper = require('looper') 3 | var offsetCodecs = require('./offset-codecs') 4 | 5 | module.exports = function (blocks, blockSize, offsetCodec) { 6 | if (typeof offsetCodec === 'number') { 7 | offsetCodec = offsetCodecs[offsetCodec] 8 | if (!offsetCodec) throw Error('Invalid number of bits to encode file offsets. Must be one of ' + Object.keys(offsetCodecs).join(' ')) 9 | } 10 | offsetCodec = offsetCodec || offsetCodecs[32] 11 | var fsw = offsetCodec.byteWidth 12 | 13 | function frame(data, start) { 14 | var _start = start 15 | var length = data.reduce(function (total, value) { return total + value.length }, 0) 16 | var b = Buffer.alloc(length + data.length * (8+fsw)) 17 | var offset = 0 18 | for(var i = 0; i < data.length; i++) { 19 | var buf = data[i] 20 | b.writeUInt32BE(buf.length, 0 + offset) //start 21 | buf.copy(b, 4+offset, 0, buf.length) 22 | b.writeUInt32BE(buf.length, 4+buf.length + offset) //end 23 | offsetCodec.encode(b, start+=buf.length+(8+fsw), 8+buf.length + offset) //length of the file, if valid 24 | frame.offset = _start + offset 25 | offset += buf.length + (8 + fsw) 26 | } 27 | return b 28 | } 29 | 30 | function getMeta (offset, cb) { 31 | blocks.readUInt32BE(offset, function (err, len) { 32 | if(err) return cb(err) 33 | 34 | //read the length of the previous item. 35 | //unless this falls right on the overlap between 36 | //two blocks, this will already be in the cache, 37 | //so will be just a mem read. 38 | if(offset === 0) 39 | next(4, 4+len, -1, (fsw+len+8)) 40 | else 41 | blocks.readUInt32BE(offset - (4+fsw), function (err, prev_len) { 42 | if(err) return cb(err) 43 | next(offset+4, offset+4+len, offset-(prev_len+8+fsw), offset+(len+8+fsw)) 44 | }) 45 | 46 | function next (start, end, prev, next) { 47 | blocks.read(start, end, function (err, value) { 48 | cb(err, value, prev, next) 49 | }) 50 | } 51 | }) 52 | } 53 | 54 | function restore (cb) { 55 | blocks.offset.once(function (offset) { 56 | if(offset === 0) return cb(null, -1) //the file is just empty! 57 | 58 | var end = offset //the very end of the file! 59 | var again = Looper(function () { 60 | offsetCodec.decodeAsync(blocks, end-fsw, function (err, _end) { 61 | if(_end != end) { 62 | if((--end) >= 0) again() 63 | //completely corrupted file! 64 | else blocks.truncate(0, next) 65 | } 66 | else { 67 | if(end != offset) { 68 | blocks.truncate(end, next) 69 | } else 70 | next() 71 | } 72 | }) 73 | }) 74 | again() 75 | function next () { 76 | blocks.readUInt32BE(end-(4+fsw), function (err, length) { 77 | if(err) cb(err) 78 | else cb(null, end-(length+8+fsw)) //start of the last record 79 | }) 80 | } 81 | }) 82 | } 83 | 84 | /** 85 | * Overwrites an item at `offset` with null bytes. 86 | * 87 | * @param {number} offset - the offset of the item to overwrite 88 | * @param {function} cb - callback that returns any error as an argument 89 | */ 90 | const overwrite = (offset, cb) => { 91 | blocks.readUInt32BE(offset, function (err, len) { 92 | if (err) return cb(err) 93 | 94 | const bookend = Buffer.alloc(4) 95 | bookend.writeUInt32BE(len, 0) 96 | 97 | const nullBytes = Buffer.alloc(len) 98 | const full = Buffer.concat([bookend, nullBytes, bookend]) 99 | 100 | blocks.write(full, offset, cb) 101 | }) 102 | } 103 | 104 | return { 105 | frame, getMeta, restore, overwrite 106 | } 107 | } 108 | 109 | 110 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var Blocks = require('aligned-block-file') 2 | var createFrame = require('./frame/recoverable') 3 | var Cache = require('hashlru') 4 | var inject = require('./inject') 5 | function id (e) { return e } 6 | function isNumber(n) { return 'number' == typeof n && !isNaN(n) } 7 | 8 | module.exports = function (file, opts) { 9 | if (!opts) opts = {} 10 | //file, blocks, frame, codec 11 | if (typeof opts !== 'object') 12 | opts = legacy.apply(null, arguments) 13 | 14 | var blockSize = opts.blockSize || 1024*16 15 | var codec = opts.codec || {encode: id, decode: id, buffer: true} 16 | var cache = opts.cache || Cache(1024) 17 | var offsetCodec = opts.offsetCodec || 32 18 | 19 | var blocks = Blocks(file, blockSize, opts.flags, cache) 20 | var frame = createFrame(blocks, blockSize, offsetCodec) 21 | return inject(blocks, frame, codec, file) 22 | } 23 | 24 | var warned = false 25 | var msg = 'flumelog-offset: blockSize and codec params moved into an object. https://github.com/flumedb/flumelog-offset' 26 | function legacy (file, blockSize, codec) { 27 | if (!warned) warned = true, console.warn(msg) 28 | if (!isNumber(blockSize)) codec = blockSize, blockSize = undefined 29 | return {blockSize: blockSize, codec: codec} 30 | } 31 | 32 | 33 | -------------------------------------------------------------------------------- /inject.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | var Obv = require('obv') 3 | var Append = require('append-batch') 4 | var createStreamCreator = require('pull-cursor') 5 | var Cache = require('hashlru') 6 | var Looper = require('pull-looper') 7 | var pull = require('pull-stream') 8 | var filter = require('pull-stream/throughs/filter') 9 | 10 | module.exports = function (blocks, frame, codec, file, cache) { 11 | var since = Obv() 12 | cache = cache || Cache(256) 13 | 14 | var append = Append(function (batch, cb) { 15 | since.once(function () { // wait for file to load before appending... 16 | batch = batch.map(codec.encode).map(function (e) { 17 | return Buffer.isBuffer(e) ? e : Buffer.from(e) 18 | }) 19 | var framed = frame.frame(batch, blocks.offset.value) 20 | var _since = frame.frame.offset 21 | blocks.append(framed, function (err, offset) { 22 | if(err) return cb(err) 23 | //else, get offset of last item. 24 | since.set(_since) 25 | cb(null, since.value) 26 | }) 27 | }) 28 | }) 29 | 30 | var isDeleted = (b) => Buffer.isBuffer(b) && b.every(x => x === 0) 31 | var isNotDeleted = (b) => isDeleted(b) === false 32 | 33 | function getMeta (offset, useCache, cb) { 34 | if (useCache) { 35 | var data = cache.get(offset) 36 | if (data) { 37 | cb(null, data.value, data.prev, data.next) 38 | return 39 | } 40 | } 41 | 42 | frame.getMeta(offset, function (err, value, prev, next) { 43 | if(err) return cb(err) 44 | if (isDeleted(value)) return cb(null, value, prev, next) // skip decode 45 | 46 | var data = { 47 | value: codec.decode(codec.buffer ? value : value.toString()), 48 | prev: prev, 49 | next: next 50 | } 51 | 52 | if (useCache) 53 | cache.set(offset, data) 54 | cb(null, data.value, data.prev, data.next) 55 | }) 56 | } 57 | 58 | var createStream = createStreamCreator(since, getMeta) 59 | 60 | frame.restore(function (err, offset) { 61 | if(err) throw err 62 | since.set(offset) 63 | }) 64 | 65 | return { 66 | filename: file, 67 | since: since, 68 | stream: function (opts) { 69 | return pull( 70 | Looper(createStream(opts)), 71 | filter(item => { 72 | if (opts && opts.seqs === false) { 73 | return isNotDeleted(item) 74 | } else { 75 | return isNotDeleted(item.value) 76 | } 77 | }) 78 | ) 79 | }, 80 | 81 | //if value is an array of buffers, then treat that as a batch. 82 | append: append, 83 | 84 | get: function (offset, cb) { 85 | frame.getMeta(offset, function (err, value) { 86 | if (err) return cb(err) 87 | if (isDeleted(value)) { 88 | const err = new Error('item has been deleted') 89 | err.code = 'flumelog:deleted' 90 | return cb(err, -1) 91 | } 92 | 93 | cb(null, codec.decode(value)) 94 | }) 95 | }, 96 | /** 97 | * Overwrite items from the log with null bytes, which are filtered out by 98 | * `get()` and `stream()` methods, effectively deleting the database items. 99 | * 100 | * @param {(number|number[])} offsets - item offset(s) to be deleted 101 | * @param {function} cb - the callback that returns operation errors, if any 102 | */ 103 | del: (offsets, cb) => { 104 | if (Array.isArray(offsets) === false) { 105 | // The `seqs` argument may be a single value or an array. 106 | // To minimize complexity, this ensures `seqs` is always an array. 107 | offsets = [ offsets ] 108 | } 109 | 110 | Promise.all(offsets.map(offset => 111 | new Promise((resolve, reject) => { 112 | // Simple callback handler for promises. 113 | const promiseCb = (err) => { 114 | if (err) { 115 | reject(err) 116 | } else { 117 | resolve() 118 | } 119 | } 120 | 121 | cache.remove(offset) 122 | frame.overwrite(offset, promiseCb) 123 | }) 124 | )).catch((err) => cb(err)) 125 | .then(() => cb(null)) 126 | }, 127 | methods: { 128 | del: 'async' 129 | } 130 | } 131 | } 132 | 133 | 134 | 135 | 136 | 137 | -------------------------------------------------------------------------------- /legacy.js: -------------------------------------------------------------------------------- 1 | var Blocks = require('aligned-block-file') 2 | var createFrame = require('./frame/basic') 3 | var Cache = require('hashlru') 4 | var inject = require('./inject') 5 | function id (e) { return e } 6 | function isNumber(n) { return 'number' == typeof n && !isNaN(n) } 7 | 8 | module.exports = function (file, block_size, codec) { 9 | if(!isNumber(block_size)) 10 | codec = block_size, block_size = 1024*16 11 | codec = codec || {encode: id, decode: id} 12 | 13 | var blocks = Blocks(file, block_size, 'a+', Cache(1024)) 14 | return inject( 15 | blocks, 16 | createFrame(blocks, codec), 17 | codec, 18 | file 19 | ) 20 | } 21 | 22 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "flumelog-offset", 3 | "version": "3.4.4", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "aligned-block-file": { 8 | "version": "1.2.0", 9 | "resolved": "https://registry.npmjs.org/aligned-block-file/-/aligned-block-file-1.2.0.tgz", 10 | "integrity": "sha512-kBF1xv3mlGBuMxJ/5IrbQD43q7Pi3yyM5IedXxuTbbc6QV3vEnZK18fH9MadoA5LvIKkgCVWRPEMlHemfz5tMg==", 11 | "requires": { 12 | "hashlru": "^2.1.0", 13 | "int53": "^1.0.0", 14 | "mkdirp": "^0.5.1", 15 | "obv": "^0.0.1", 16 | "uint48be": "^2.0.1" 17 | } 18 | }, 19 | "append-batch": { 20 | "version": "0.0.2", 21 | "resolved": "https://registry.npmjs.org/append-batch/-/append-batch-0.0.2.tgz", 22 | "integrity": "sha1-1zm0UDiIJF1Hkz1HVisRSf+d+Lc=" 23 | }, 24 | "balanced-match": { 25 | "version": "1.0.0", 26 | "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", 27 | "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", 28 | "dev": true 29 | }, 30 | "bench-flumelog": { 31 | "version": "1.2.0", 32 | "resolved": "https://registry.npmjs.org/bench-flumelog/-/bench-flumelog-1.2.0.tgz", 33 | "integrity": "sha512-hzjcwPIUVlbMo2vawnKYrilKTu85IwBk1SBYGjwJkyyoggnGLH7aEUZVuZwhtUhzyImnGRykLqeaoqitBvQJug==", 34 | "dev": true, 35 | "requires": { 36 | "pull-paramap": "^1.2.1", 37 | "pull-stream": "^3.5.0" 38 | } 39 | }, 40 | "brace-expansion": { 41 | "version": "1.1.11", 42 | "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", 43 | "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", 44 | "dev": true, 45 | "requires": { 46 | "balanced-match": "^1.0.0", 47 | "concat-map": "0.0.1" 48 | } 49 | }, 50 | "concat-map": { 51 | "version": "0.0.1", 52 | "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", 53 | "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", 54 | "dev": true 55 | }, 56 | "deep-equal": { 57 | "version": "1.0.1", 58 | "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz", 59 | "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=", 60 | "dev": true 61 | }, 62 | "define-properties": { 63 | "version": "1.1.3", 64 | "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", 65 | "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", 66 | "dev": true, 67 | "requires": { 68 | "object-keys": "^1.0.12" 69 | } 70 | }, 71 | "defined": { 72 | "version": "1.0.0", 73 | "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.0.tgz", 74 | "integrity": "sha1-yY2bzvdWdBiOEQlpFRGZ45sfppM=", 75 | "dev": true 76 | }, 77 | "es-abstract": { 78 | "version": "1.13.0", 79 | "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.13.0.tgz", 80 | "integrity": "sha512-vDZfg/ykNxQVwup/8E1BZhVzFfBxs9NqMzGcvIJrqg5k2/5Za2bWo40dK2J1pgLngZ7c+Shh8lwYtLGyrwPutg==", 81 | "dev": true, 82 | "requires": { 83 | "es-to-primitive": "^1.2.0", 84 | "function-bind": "^1.1.1", 85 | "has": "^1.0.3", 86 | "is-callable": "^1.1.4", 87 | "is-regex": "^1.0.4", 88 | "object-keys": "^1.0.12" 89 | } 90 | }, 91 | "es-to-primitive": { 92 | "version": "1.2.0", 93 | "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.0.tgz", 94 | "integrity": "sha512-qZryBOJjV//LaxLTV6UC//WewneB3LcXOL9NP++ozKVXsIIIpm/2c13UDiD9Jp2eThsecw9m3jPqDwTyobcdbg==", 95 | "dev": true, 96 | "requires": { 97 | "is-callable": "^1.1.4", 98 | "is-date-object": "^1.0.1", 99 | "is-symbol": "^1.0.2" 100 | } 101 | }, 102 | "flumecodec": { 103 | "version": "0.0.1", 104 | "resolved": "https://registry.npmjs.org/flumecodec/-/flumecodec-0.0.1.tgz", 105 | "integrity": "sha1-rgSacUOGu4PjQmV6gpJLcDZKkNY=", 106 | "dev": true, 107 | "requires": { 108 | "level-codec": "^6.2.0" 109 | } 110 | }, 111 | "for-each": { 112 | "version": "0.3.3", 113 | "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", 114 | "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", 115 | "dev": true, 116 | "requires": { 117 | "is-callable": "^1.1.3" 118 | } 119 | }, 120 | "fs.realpath": { 121 | "version": "1.0.0", 122 | "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", 123 | "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", 124 | "dev": true 125 | }, 126 | "function-bind": { 127 | "version": "1.1.1", 128 | "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", 129 | "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", 130 | "dev": true 131 | }, 132 | "glob": { 133 | "version": "7.1.3", 134 | "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", 135 | "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", 136 | "dev": true, 137 | "requires": { 138 | "fs.realpath": "^1.0.0", 139 | "inflight": "^1.0.4", 140 | "inherits": "2", 141 | "minimatch": "^3.0.4", 142 | "once": "^1.3.0", 143 | "path-is-absolute": "^1.0.0" 144 | } 145 | }, 146 | "has": { 147 | "version": "1.0.3", 148 | "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", 149 | "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", 150 | "dev": true, 151 | "requires": { 152 | "function-bind": "^1.1.1" 153 | } 154 | }, 155 | "has-symbols": { 156 | "version": "1.0.0", 157 | "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz", 158 | "integrity": "sha1-uhqPGvKg/DllD1yFA2dwQSIGO0Q=", 159 | "dev": true 160 | }, 161 | "hashlru": { 162 | "version": "2.3.0", 163 | "resolved": "https://registry.npmjs.org/hashlru/-/hashlru-2.3.0.tgz", 164 | "integrity": "sha512-0cMsjjIC8I+D3M44pOQdsy0OHXGLVz6Z0beRuufhKa0KfaD2wGwAev6jILzXsd3/vpnNQJmWyZtIILqM1N+n5A==" 165 | }, 166 | "inflight": { 167 | "version": "1.0.6", 168 | "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", 169 | "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", 170 | "dev": true, 171 | "requires": { 172 | "once": "^1.3.0", 173 | "wrappy": "1" 174 | } 175 | }, 176 | "inherits": { 177 | "version": "2.0.3", 178 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", 179 | "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", 180 | "dev": true 181 | }, 182 | "int53": { 183 | "version": "1.0.0", 184 | "resolved": "https://registry.npmjs.org/int53/-/int53-1.0.0.tgz", 185 | "integrity": "sha512-u8BMiMa05OPBgd32CKTead0CVTsFVgwFk23nNXo1teKPF6Sxcu0lXxEzP//zTcaKzXbGgPDXGmj/woyv+I4C5w==" 186 | }, 187 | "is-callable": { 188 | "version": "1.1.4", 189 | "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz", 190 | "integrity": "sha512-r5p9sxJjYnArLjObpjA4xu5EKI3CuKHkJXMhT7kwbpUyIFD1n5PMAsoPvWnvtZiNz7LjkYDRZhd7FlI0eMijEA==", 191 | "dev": true 192 | }, 193 | "is-date-object": { 194 | "version": "1.0.1", 195 | "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", 196 | "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=", 197 | "dev": true 198 | }, 199 | "is-regex": { 200 | "version": "1.0.4", 201 | "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", 202 | "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=", 203 | "dev": true, 204 | "requires": { 205 | "has": "^1.0.1" 206 | } 207 | }, 208 | "is-symbol": { 209 | "version": "1.0.2", 210 | "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.2.tgz", 211 | "integrity": "sha512-HS8bZ9ox60yCJLH9snBpIwv9pYUAkcuLhSA1oero1UB5y9aiQpRA8y2ex945AOtCZL1lJDeIk3G5LthswI46Lw==", 212 | "dev": true, 213 | "requires": { 214 | "has-symbols": "^1.0.0" 215 | } 216 | }, 217 | "level-codec": { 218 | "version": "6.2.0", 219 | "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-6.2.0.tgz", 220 | "integrity": "sha1-pLUkS7akwvcj1oodZOmAxTYn2dQ=", 221 | "dev": true 222 | }, 223 | "looper": { 224 | "version": "4.0.0", 225 | "resolved": "https://registry.npmjs.org/looper/-/looper-4.0.0.tgz", 226 | "integrity": "sha1-dwat7VmpntygbmtUu4bI7BnJUVU=" 227 | }, 228 | "ltgt": { 229 | "version": "2.2.0", 230 | "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.0.tgz", 231 | "integrity": "sha1-tlul/LNJopkkyOMz98alVi8uSEI=" 232 | }, 233 | "minimatch": { 234 | "version": "3.0.4", 235 | "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", 236 | "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", 237 | "dev": true, 238 | "requires": { 239 | "brace-expansion": "^1.1.7" 240 | } 241 | }, 242 | "minimist": { 243 | "version": "0.0.8", 244 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", 245 | "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" 246 | }, 247 | "mkdirp": { 248 | "version": "0.5.1", 249 | "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", 250 | "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", 251 | "requires": { 252 | "minimist": "0.0.8" 253 | } 254 | }, 255 | "object-inspect": { 256 | "version": "1.6.0", 257 | "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.6.0.tgz", 258 | "integrity": "sha512-GJzfBZ6DgDAmnuaM3104jR4s1Myxr3Y3zfIyN4z3UdqN69oSRacNK8UhnobDdC+7J2AHCjGwxQubNJfE70SXXQ==", 259 | "dev": true 260 | }, 261 | "object-keys": { 262 | "version": "1.1.0", 263 | "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.0.tgz", 264 | "integrity": "sha512-6OO5X1+2tYkNyNEx6TsCxEqFfRWaqx6EtMiSbGrw8Ob8v9Ne+Hl8rBAgLBZn5wjEz3s/s6U1WXFUFOcxxAwUpg==", 265 | "dev": true 266 | }, 267 | "obv": { 268 | "version": "0.0.1", 269 | "resolved": "https://registry.npmjs.org/obv/-/obv-0.0.1.tgz", 270 | "integrity": "sha1-yyNhBjQVNvDaxIFeBnCCIcrX+14=" 271 | }, 272 | "once": { 273 | "version": "1.4.0", 274 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", 275 | "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", 276 | "dev": true, 277 | "requires": { 278 | "wrappy": "1" 279 | } 280 | }, 281 | "path-is-absolute": { 282 | "version": "1.0.1", 283 | "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", 284 | "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", 285 | "dev": true 286 | }, 287 | "path-parse": { 288 | "version": "1.0.6", 289 | "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", 290 | "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", 291 | "dev": true 292 | }, 293 | "pull-cursor": { 294 | "version": "3.0.0", 295 | "resolved": "https://registry.npmjs.org/pull-cursor/-/pull-cursor-3.0.0.tgz", 296 | "integrity": "sha512-95lZVSF2eSEdOmUtlOBaD9p5YOvlYeCr5FBv2ySqcj/4rpaXI6d8OH+zPHHjKAf58R8QXJRZuyfHkcCX8TZbAg==", 297 | "requires": { 298 | "looper": "^4.0.0", 299 | "ltgt": "^2.2.0", 300 | "pull-stream": "^3.6.0" 301 | } 302 | }, 303 | "pull-looper": { 304 | "version": "1.0.0", 305 | "resolved": "https://registry.npmjs.org/pull-looper/-/pull-looper-1.0.0.tgz", 306 | "integrity": "sha512-djlD60A6NGe5goLdP5pgbqzMEiWmk1bInuAzBp0QOH4vDrVwh05YDz6UP8+pOXveKEk8wHVP+rB2jBrK31QMPA==", 307 | "requires": { 308 | "looper": "^4.0.0" 309 | } 310 | }, 311 | "pull-paramap": { 312 | "version": "1.2.2", 313 | "resolved": "https://registry.npmjs.org/pull-paramap/-/pull-paramap-1.2.2.tgz", 314 | "integrity": "sha1-UaQZPOnI1yFdla2tReK824STsjo=", 315 | "dev": true, 316 | "requires": { 317 | "looper": "^4.0.0" 318 | } 319 | }, 320 | "pull-stream": { 321 | "version": "3.6.9", 322 | "resolved": "https://registry.npmjs.org/pull-stream/-/pull-stream-3.6.9.tgz", 323 | "integrity": "sha512-hJn4POeBrkttshdNl0AoSCVjMVSuBwuHocMerUdoZ2+oIUzrWHFTwJMlbHND7OiKLVgvz6TFj8ZUVywUMXccbw==" 324 | }, 325 | "resolve": { 326 | "version": "1.10.0", 327 | "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.0.tgz", 328 | "integrity": "sha512-3sUr9aq5OfSg2S9pNtPA9hL1FVEAjvfOC4leW0SNf/mpnaakz2a9femSd6LqAww2RaFctwyf1lCqnTHuF1rxDg==", 329 | "dev": true, 330 | "requires": { 331 | "path-parse": "^1.0.6" 332 | } 333 | }, 334 | "resumer": { 335 | "version": "0.0.0", 336 | "resolved": "https://registry.npmjs.org/resumer/-/resumer-0.0.0.tgz", 337 | "integrity": "sha1-8ej0YeQGS6Oegq883CqMiT0HZ1k=", 338 | "dev": true, 339 | "requires": { 340 | "through": "~2.3.4" 341 | } 342 | }, 343 | "string.prototype.trim": { 344 | "version": "1.1.2", 345 | "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.1.2.tgz", 346 | "integrity": "sha1-0E3iyJ4Tf019IG8Ia17S+ua+jOo=", 347 | "dev": true, 348 | "requires": { 349 | "define-properties": "^1.1.2", 350 | "es-abstract": "^1.5.0", 351 | "function-bind": "^1.0.2" 352 | } 353 | }, 354 | "tape": { 355 | "version": "4.10.1", 356 | "resolved": "https://registry.npmjs.org/tape/-/tape-4.10.1.tgz", 357 | "integrity": "sha512-G0DywYV1jQeY3axeYnXUOt6ktnxS9OPJh97FGR3nrua8lhWi1zPflLxcAHavZ7Jf3qUfY7cxcVIVFa4mY2IY1w==", 358 | "dev": true, 359 | "requires": { 360 | "deep-equal": "~1.0.1", 361 | "defined": "~1.0.0", 362 | "for-each": "~0.3.3", 363 | "function-bind": "~1.1.1", 364 | "glob": "~7.1.3", 365 | "has": "~1.0.3", 366 | "inherits": "~2.0.3", 367 | "minimist": "~1.2.0", 368 | "object-inspect": "~1.6.0", 369 | "resolve": "~1.10.0", 370 | "resumer": "~0.0.0", 371 | "string.prototype.trim": "~1.1.2", 372 | "through": "~2.3.8" 373 | }, 374 | "dependencies": { 375 | "minimist": { 376 | "version": "1.2.0", 377 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", 378 | "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", 379 | "dev": true 380 | } 381 | } 382 | }, 383 | "test-flumelog": { 384 | "version": "1.0.0", 385 | "resolved": "https://registry.npmjs.org/test-flumelog/-/test-flumelog-1.0.0.tgz", 386 | "integrity": "sha1-fiUYfvsN4HBvY3qmVvfSFDuHi9A=", 387 | "dev": true, 388 | "requires": { 389 | "pull-spec": "0.0.1", 390 | "pull-stream": "^3.5.0", 391 | "tape": "^4.6.2" 392 | }, 393 | "dependencies": { 394 | "pull-spec": { 395 | "version": "0.0.1", 396 | "resolved": "https://registry.npmjs.org/pull-spec/-/pull-spec-0.0.1.tgz", 397 | "integrity": "sha1-ctW7OEfGUm2FvAmSwun0EcIzUak=", 398 | "dev": true 399 | } 400 | } 401 | }, 402 | "through": { 403 | "version": "2.3.8", 404 | "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", 405 | "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", 406 | "dev": true 407 | }, 408 | "uint48be": { 409 | "version": "2.0.1", 410 | "resolved": "https://registry.npmjs.org/uint48be/-/uint48be-2.0.1.tgz", 411 | "integrity": "sha512-LQvWofTo3RCz+XaQR3VNch+dDFwpIvWr/98imhQne++vFhpQP16YAC/a8w9N00Heqqra00ACjHT18cgvn5H+bg==" 412 | }, 413 | "wrappy": { 414 | "version": "1.0.2", 415 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", 416 | "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", 417 | "dev": true 418 | } 419 | } 420 | } 421 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "flumelog-offset", 3 | "description": "a flumelog based on offset into a file", 4 | "version": "3.4.4", 5 | "homepage": "https://github.com/flumedb/flumelog-offset", 6 | "repository": { 7 | "type": "git", 8 | "url": "git://github.com/flumedb/flumelog-offset.git" 9 | }, 10 | "dependencies": { 11 | "aligned-block-file": "^1.2.0", 12 | "append-batch": "0.0.2", 13 | "hashlru": "^2.3.0", 14 | "int53": "^1.0.0", 15 | "looper": "^4.0.0", 16 | "obv": "0.0.1", 17 | "pull-cursor": "^3.0.0", 18 | "pull-looper": "^1.0.0", 19 | "pull-stream": "^3.6.13", 20 | "uint48be": "^2.0.1" 21 | }, 22 | "devDependencies": { 23 | "bench-flumelog": "^1.2.0", 24 | "flumecodec": "0.0.1", 25 | "tape": "^4.10.1", 26 | "test-flumelog": "1.0.0" 27 | }, 28 | "scripts": { 29 | "test": "set -e; for t in test/*.js; do node $t; done" 30 | }, 31 | "author": "'Dominic Tarr' (dominictarr.com)", 32 | "license": "MIT" 33 | } 34 | -------------------------------------------------------------------------------- /test/flumelog.js: -------------------------------------------------------------------------------- 1 | var pull = require('pull-stream') 2 | var create = require('../') 3 | var testLog = require('test-flumelog') 4 | 5 | function test(name, opts, cb) { 6 | testLog(function () { 7 | return create('/tmp/test_flumelog-offset_'+Date.now(), Object.assign({ 8 | blockSize: 1024, 9 | codec: { 10 | encode: function (v) { 11 | return Buffer.from(JSON.stringify(v)) 12 | }, 13 | decode: function (v) { 14 | return JSON.parse(v) 15 | }, 16 | buffer: false 17 | } 18 | }, opts)) 19 | }, function () { 20 | console.log(name + ' done') 21 | cb() 22 | }) 23 | } 24 | 25 | pull( 26 | pull.values([32, 48, 53]), 27 | pull.asyncMap( function(bits, cb) { 28 | test(bits + 'bit', {offsetCodec: bits}, cb) 29 | }), 30 | pull.drain() 31 | ) 32 | -------------------------------------------------------------------------------- /test/restore.js: -------------------------------------------------------------------------------- 1 | 2 | var fs = require('fs') 3 | var tape = require('tape') 4 | var crypto = require('crypto') 5 | var OffsetLog = require('../') 6 | var pull = require('pull-stream') 7 | var offsetCodecs = require('../frame/offset-codecs') 8 | 9 | tape('Create, break and restore', function(t) { 10 | test(t, '32bit', {offsetCodec: offsetCodecs[32]}) 11 | test(t, '48bit', {offsetCodec: offsetCodecs[48]}) 12 | test(t, '53bit', {offsetCodec: offsetCodecs[53]}) 13 | }) 14 | 15 | function test(t, name, opts) { 16 | var file = '/tmp/flumelog_restore_'+name+Date.now() 17 | var n = 13, ary = [], since 18 | while(n--) 19 | ary.push(crypto.randomBytes(17)) 20 | 21 | t.test(name +' setup', function (t) { 22 | var log = OffsetLog(file, Object.assign({blockSize: 23}, opts)) 23 | 24 | log.since.once(function (value) { 25 | t.equal(value, -1) 26 | log.append(ary, function (err, value) { 27 | // t.equal(log.since.value, 1024*9 + 9*12 + 4) 28 | console.log('after append: since=' + log.since.value) 29 | log.get(log.since.value, function (err, value) { 30 | if(err) throw err 31 | t.deepEqual(value, ary[ary.length-1]) 32 | 33 | pull( 34 | log.stream(), 35 | pull.collect(function (err, _ary) { 36 | if(err) throw err 37 | t.deepEqual(ary, _ary.map(function (e) { return e.value })) 38 | since = log.since.value 39 | t.end() 40 | }) 41 | ) 42 | 43 | }) 44 | }) 45 | 46 | }) 47 | }) 48 | 49 | t.test(name + ' restore, valid', function (t) { 50 | 51 | var log = OffsetLog(file, Object.assign({blockSize: 23}, opts)) 52 | 53 | log.since(function (v) { 54 | t.equal(v, since) 55 | t.end() 56 | }) 57 | }) 58 | 59 | t.test(name + ' truncate', function (t) { 60 | fs.stat(file, function (err, stat) { 61 | if(err) throw err 62 | fs.readFile(file, function (err, buf) { 63 | if(err) throw err 64 | var slice = stat.size - ~~(ary[ary.length-1].length/2) 65 | console.log('slice at:', slice) 66 | fs.truncate(file, slice, function (err) { 67 | if(err) throw err 68 | t.end() 69 | }) 70 | }) 71 | 72 | }) 73 | }) 74 | 75 | 76 | var end = null 77 | t.test(name + ' restore', function (t) { 78 | var log = OffsetLog(file, Object.assign({blockSize: 23}, opts)) 79 | log.since.once(function (v) { 80 | t.ok(v < since) 81 | t.ok(v > 0) 82 | end = v 83 | pull( 84 | log.stream(), 85 | pull.collect(function (err, _ary) { 86 | if(err) throw err 87 | t.deepEqual(ary.slice(0, ary.length-1), _ary.map(function (e) { return e.value })) 88 | since = log.since.value 89 | t.end() 90 | }) 91 | ) 92 | 93 | }) 94 | }) 95 | 96 | 97 | t.test(name = ' restore, again', function (t) { 98 | var log = OffsetLog(file, Object.assign({blockSize: 23}, opts)) 99 | log.since.once(function (v) { 100 | t.equal(v, end) 101 | t.end() 102 | }) 103 | }) 104 | 105 | } 106 | 107 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /test/simple.js: -------------------------------------------------------------------------------- 1 | 2 | var tape = require('tape') 3 | var pull = require('pull-stream') 4 | var Offset = require('../') 5 | 6 | 7 | var file = '/tmp/offset-test_'+Date.now()+'.log' 8 | var db = Offset(file, {blockSize: 16}) 9 | var live = [] 10 | 11 | pull( 12 | db.stream({live: true, sync: false, seqs: false}), 13 | pull.drain(function (data) { 14 | live.push(data) 15 | }) 16 | ) 17 | 18 | tape('simple', function (t) { 19 | t.equal(db.since.value, undefined) 20 | var offsets = [] 21 | db.since(function (_v) { 22 | offsets.push(_v) 23 | }) 24 | db.since.once(function (_v) { 25 | t.equal(_v, -1) 26 | 27 | db.append(Buffer.from('hello world'), function (err, offset1) { 28 | if(err) throw err 29 | t.equal(offset1, db.since.value) 30 | //NOTE: 'hello world'.length + 8 (start frame + end frame) 31 | t.equal(db.since.value, 0) 32 | db.append(Buffer.from('hello offset db'), function (err, offset2) { 33 | if(err) throw err 34 | t.ok(offset2 > offset1) 35 | t.equal(offset2, db.since.value) 36 | // t.deepEqual(offsets, [-1, 0, 19], 'appended two records') 37 | db.get(offset1, function (err, b) { 38 | if(err) throw err 39 | t.equal(b.toString(), 'hello world', 'read 1st value') 40 | 41 | db.get(offset2, function (err, b2) { 42 | if(err) throw err 43 | t.equal(b2.toString(), 'hello offset db') 44 | db.del(offset1, function (err) { 45 | t.error(err) 46 | db.get(offset1, function (err) { 47 | t.ok(err) 48 | t.equal(err.message, 'item has been deleted') 49 | t.end() 50 | }) 51 | }) 52 | }) 53 | }) 54 | }) 55 | }) 56 | }) 57 | }) 58 | 59 | /* 60 | 8, 4, 32, 32 = 76 61 | _header = {offset, length, prev_mac, hash(data)} 62 | 32, 76, = 108 63 | header = mac(_header)|_header 64 | data (length) 65 | 4 66 | footer = length 67 | 68 | OR, encrypted database? 69 | 70 | header_mac (16) 71 | [offset(8), length(4), data_mac(16)] 72 | 73 | footer_mac(16) 74 | [length (4)] 75 | 76 | */ 77 | tape('stream', function (t) { 78 | 79 | pull( 80 | db.stream({min: 0, seqs: false}), 81 | pull.collect(function (err, ary) { 82 | if(err) throw err 83 | t.deepEqual(ary.map(String), ['hello offset db']) 84 | t.end() 85 | }) 86 | ) 87 | 88 | }) 89 | 90 | tape('stream with options', function (t) { 91 | pull( 92 | db.stream({min: 0, seqs: true}), 93 | pull.collect(function (err, ary) { 94 | if(err) throw err 95 | 96 | t.deepEqual(ary.map(item => String(item.value)), ['hello offset db']) 97 | t.end() 98 | }) 99 | ) 100 | }) 101 | 102 | tape('live', function (t) { 103 | t.deepEqual(live.map(String), ['hello world', 'hello offset db']) 104 | t.end() 105 | }) 106 | 107 | tape('reverse', function (t) { 108 | pull( 109 | db.stream({reverse: true, seqs: false}), 110 | pull.collect(function (err, ary) { 111 | console.log(ary, db.since.value) 112 | t.deepEqual(ary.map(String), ['hello offset db']) 113 | t.end() 114 | }) 115 | ) 116 | }) 117 | 118 | tape('append batch', function (t) { 119 | var file = '/tmp/offset-test_2_'+Date.now()+'.log' 120 | var db = Offset(file, {blockSize: 16}) 121 | 122 | db.append([ 123 | Buffer.from('hello world'), 124 | Buffer.from('hello offset db'), 125 | ], function (err, offsets) { 126 | if(err) throw err 127 | t.ok(offsets) 128 | // t.deepEqual(offsets, 19) 129 | t.end() 130 | }) 131 | 132 | }) 133 | 134 | tape('stream in empty database', function (t) { 135 | var file = '/tmp/offset-test_3_'+Date.now()+'.log' 136 | var db = Offset(file, {blockSize: 16}) 137 | 138 | db.since.once(function (_offset) { 139 | t.equal(_offset, -1, 'offset is zero') 140 | }) 141 | 142 | pull( 143 | db.stream(), 144 | pull.collect(function (err, ary) { 145 | if(err) throw err 146 | t.deepEqual(ary, []) 147 | t.end() 148 | }) 149 | ) 150 | 151 | }) 152 | 153 | 154 | tape('stream in before append cb', function (t) { 155 | var file = '/tmp/offset-test_4_'+Date.now()+'.log' 156 | var db = Offset(file, {blockSize: 16}) 157 | 158 | db.since.once(function (_offset) { 159 | t.equal(_offset, -1, 'offset is zero') 160 | }) 161 | 162 | db.append(Buffer.from('hello world'), function (err, offset) { 163 | 164 | }) 165 | 166 | pull( 167 | db.stream(), 168 | pull.collect(function (err, ary) { 169 | if(err) throw err 170 | t.deepEqual(ary, []) 171 | t.end() 172 | }) 173 | ) 174 | 175 | }) 176 | 177 | -------------------------------------------------------------------------------- /test/stream.js: -------------------------------------------------------------------------------- 1 | var Log = require('../') 2 | var pull = require('pull-stream') 3 | var tape = require('tape') 4 | 5 | var log = Log('/tmp/test_offset-log_'+Date.now(), {blockSize: 1024}) 6 | 7 | function encode (obj) { 8 | return Buffer.from(JSON.stringify(obj)) 9 | } 10 | 11 | function decode (b) { 12 | return JSON.parse(b.toString()) 13 | } 14 | 15 | tape('stream on an empty database', function (t) { 16 | pull( 17 | log.stream(), 18 | pull.collect(function (err, ary) { 19 | t.notOk(err) 20 | t.deepEqual(ary, []) 21 | t.end() 22 | }) 23 | ) 24 | }) 25 | 26 | tape('append objects, and stream them out the same', function (t) { 27 | 28 | var n = 4 29 | var a = [ 30 | {foo: true, bar: false, r: Math.random()}, 31 | {foo: true, bar: true, r: Math.random()}, 32 | {foo: false, bar: true, r: Math.random()}, 33 | {foo: false, bar: false, r: Math.random()} 34 | ] 35 | 36 | var ary = [] 37 | 38 | log.append(encode(a[0]), next) 39 | log.append(encode(a[1]), next) 40 | log.append(encode(a[2]), next) 41 | log.append(encode(a[3]), next) 42 | 43 | function next () { 44 | if(--n) return 45 | pull( 46 | log.stream({keys:true, values: true}), 47 | 48 | pull.map(function (data) { 49 | if(data.sync) return data 50 | return {key: data.key, value: decode(data.value)} 51 | }), 52 | pull.through(console.log), 53 | pull.collect(function (err, a) { 54 | if(err) throw err 55 | t.deepEqual(ary, a) 56 | t.end() 57 | }) 58 | ) 59 | } 60 | 61 | pull( 62 | log.stream({live: true, keys: true, values: true, sync: false}), 63 | pull.map(function (data) { 64 | if(data.sync) return data 65 | return {key: data.key, value: decode(data.value)} 66 | }), 67 | pull.drain(function (data) { 68 | ary.push(data) 69 | }) 70 | ) 71 | }) 72 | --------------------------------------------------------------------------------