├── .gitignore ├── .npmignore ├── README.md ├── file.js ├── package.json └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .npmignore 2 | .gitignore 3 | data.json* 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gun-file 2 | 3 | A streaming file extension for Gun. 4 | 5 | ## Controlling Options 6 | 7 | ```js 8 | var gun = new Gun( { 9 | 'file-name' : 'yourData.json', // default is 'data.json6' 10 | 'file-mode' : 0666, // default is 0666 11 | 'file-pretty' : true, // default, if false, will write ugly/compressed json 12 | 'file-delay' : 100, // default. control flush interval/delay default. 13 | } ); 14 | ``` 15 | 16 | ## Important 17 | 18 | To avoid conflict with Gun's builtin file driver, you will have to use ```require('gun/gun')``` plus any other modules you want to use. 19 | The option ```file:null``` should work in gun to disable the builtin driver, but at this point does not. 20 | 21 | ```js 22 | var Gun = require( "gun/gun" ); 23 | require( 'gun-file' ); 24 | ``` 25 | 26 | 27 | 28 | ### Changelog 29 | - 1.0.123 - update to gun latest API ( deleted internal value) 30 | - 1.0.121 - generate 'in' on ctx instead of gun. (Issue #2) 31 | - 1.0.120 - test file path dyanamically to use common gun instead of potential private path... 32 | - 1.0.119 - update to gun 0.9 33 | - 1.0.118 - 'in' events require posting the gun instance they are on in the message now. Change default filename to data.json6. 34 | - 1.0.117 - remove alpha warning message. 35 | - 1.0.116 - add shared cache between instances using the same filename 36 | - 1.0.115 - fix a couple more short timeout cross read/writes. 37 | - 1.0.114 - handle short timeouts better; flushes during flushes would lose the event to flush. 38 | - 1.0.113 - Handle multiple connections with same file better; prevent writing while another is reading. 39 | - 1.0.112 - add acks on puts; add method to attach gun-file to a gun datbase should it fail to load correct gun base. 40 | - 1.0.111 - fix getting file-pretty option. 41 | - 1.0.11 - update gun revision in package.json; added .npmignore 42 | - 1.0.1 - update to JSON6 library rename of 'add' to 'write' 43 | - 1.0.0 - Initial release 44 | 45 | 46 | 47 | ### Benchmark Results 48 | 49 | ``` 50 | __ Small Nodes: 10 Properties Each __ 51 | Write 10000 nodes: : 2164ms; 2.164s; 0.216 ms/node; errors: 0. 52 | Read 10000 nodes: : 3005ms; 3.005s; 0.300 ms/node; errors: 0. 53 | Update 10000 nodes: : 2404ms; 2.404s; 0.240 ms/node; errors: 0. 54 | Update single field on 10000 nodes: : 2593ms; 2.593s; 0.259 ms/node; errors: 0. 55 | __ Medium Nodes: 1000 Properties Each __ 56 | Write 100 nodes: : 2302ms; 2.302s; 22.792 ms/node; errors: 0. 57 | Read 100 nodes: : 2411ms; 2.411s; 23.871 ms/node; errors: 0. 58 | Update 100 nodes: : 1636ms; 1.636s; 16.198 ms/node; errors: 0. 59 | Update single field on 100 nodes: : 1667ms; 1.667s; 16.505 ms/node; errors: 0. 60 | __ Large Nodes: 10000 Properties Each __ 61 | Write 10 nodes: : 2838ms; 2.838s; 258.000 ms/node; errors: 0. 62 | Read 10 nodes: : 3444ms; 3.444s; 313.091 ms/node; errors: 0. 63 | Update 10 nodes: : 2097ms; 2.097s; 190.636 ms/node; errors: 0. 64 | Update single field on 10 nodes: : 2177ms; 2.177s; 197.909 ms/node; errors: 0. 65 | ``` 66 | -------------------------------------------------------------------------------- /file.js: -------------------------------------------------------------------------------- 1 | // This was written by the wonderful d3x0r 2 | //console.log( "module:",module, module.filename.includes( "node_modules/gun-file" )); 3 | 4 | const Gun = require(module.filename.includes( "node_modules/gun-file" )?'../gun/gun':'gun/gun'); 5 | const json6 = require( 'json-6' ); 6 | const fs = require('fs'); 7 | const _debug = false; 8 | const _json_debug = false; 9 | const _debug_write_time = false; 10 | 11 | const rel_ = Gun.val.rel._; // '#' 12 | const val_ = Gun.obj.has._; // '.' 13 | const node_ = Gun.node._; // '_' 14 | const state_ = Gun.state._;// '>'; 15 | const soul_ = Gun.node.soul._; // '#' 16 | const ACK_ = '@'; 17 | const SEQ_ = '#'; 18 | 19 | const fileStates = {}; 20 | 21 | exports.attach = function( Gun ) { 22 | 23 | Gun.on('opt', function(ctx){ 24 | this.to.next(ctx); 25 | var opt = ctx.opt; 26 | if(ctx.once){ return } 27 | var fileName = String(opt['file-name'] || 'data.json6'); 28 | var fileMode = opt['file-mode'] || 0666; 29 | var filePretty = ('file-pretty' in opt)?opt['file-pretty']:true; 30 | var fileDelay = opt['file-delay'] || 100; 31 | var gun = ctx.gun; 32 | var graph = ctx.graph, acks = {}, count = 0, to; 33 | 34 | //var reading = false; 35 | var pending = []; 36 | var pendingPut = []; 37 | var pend = null; 38 | var pendPut = null; 39 | var wantFlush = false; 40 | var loaded = false; 41 | var fileState = fileStates[fileName] || ( fileStates[fileName] = { reading:false, writing:false, flushPending : false, disk : {} } ); 42 | 43 | preloadDisk(opt, fileState.disk); 44 | 45 | //Gun.log.once( 46 | // 'file-warning', 47 | // 'WARNING! This `gun-file` pre-alpha module for gun for testing only!' 48 | //); 49 | 50 | var skip_put; 51 | function doPut( at ) { 52 | //_debug && console.log( "So this should be updating db?", skip_put, pend ); 53 | Gun.graph.is(at.put, null, map); 54 | if( !loaded ) { 55 | pendingPut.push( at ); 56 | return; 57 | } 58 | if( skip_put && skip_put == at['@'] ) { 59 | //_debug && console.log( "skipping put in-get", skip_put, at['@'] ); 60 | return; 61 | } 62 | if(!at['@']){ acks[at[SEQ_]] = true; } // only ack non-acks. 63 | else if( pend && at['@'] == pend.at[SEQ_] ) { 64 | _debug && console.log( "Prevent self flush", pend ); 65 | return; 66 | } 67 | //console.log( "WILL FLUSH WITH at:", at ); 68 | //console.log( "pend:", pend ); 69 | //console.log( "skip:", skip_put ); 70 | count += 1; 71 | if(count >= (opt.batch || 10000)){ 72 | clearTimeout( to ); 73 | to = null; 74 | return flush(); 75 | } 76 | if( !to ) 77 | _debug && console.log( "put happened?", to !== null ); 78 | if(to) clearTimeout( to ); 79 | to = setTimeout(flush, fileDelay ); 80 | fileState.flushPending = true; 81 | ctx.on('in', {[ACK_]: at[rel_], gun:gun, ok: 1}); 82 | } 83 | 84 | ctx.on('put', function(at) { 85 | this.to.next(at); 86 | doPut(at) 87 | }); 88 | 89 | ctx.on('get', function(at){ 90 | this.to.next(at); 91 | var gun = at.gun, lex = at.get, soul, data, opt, u; 92 | if(!lex || !(soul = lex[soul_])){ return } 93 | var field = lex[val_]; 94 | if( fileState.reading || fileState.flushPending || !loaded ) { 95 | _debug && console.log( "Still reading... pushing request." ); 96 | pending.push( {gun:gun, ctx:ctx, soul:soul, at:at, field:field} ); 97 | } else { 98 | 99 | data = fileState.disk[soul] || u; 100 | if(data && field){ 101 | _debug && console.log( "get field?", field, data ); 102 | data = Gun.state.to(data, field); 103 | } 104 | //else console.log( "not data or not field?", field ); 105 | if( data ) { 106 | skip_put = at[SEQ_]; 107 | ctx.on('in', {[ACK_]: at[SEQ_], gun:gun, put: Gun.graph.node(data)}); 108 | //_debug && console.log( "getting data:", data ); 109 | skip_put = null; 110 | } 111 | else { 112 | _debug && console.log( "didn't get dat for", soul ); 113 | ctx.on('in', {[ACK_]: at[SEQ_], gun:gun, err: "no data"}); 114 | } 115 | } 116 | //},11); 117 | }); 118 | 119 | var map = function(val, key, node, soul){ 120 | //_debug && console.log( "mapping graph?", soul ); 121 | fileState.disk[soul] = Gun.state.to(node, key, fileState.disk[soul]); 122 | } 123 | 124 | 125 | function preloadDisk( opt, disk ) { 126 | if( fileState.loaded ) 127 | return; // already have the disk image in memory 128 | if( fileState.flushPending || fileState.writing ) { 129 | _debug && console.log( "wait for pending flush on another connection?" ); 130 | setTimeout( ()=>preloadDisk( opt, disk ), fileDelay/3 ); 131 | return; 132 | } 133 | fileState.reading = true; 134 | _debug && console.log( new Date(), "preload happened." ); 135 | const stream = fs.createReadStream( fileName, { flags:"r", encoding:"utf8"} ); 136 | const parser = json6.begin( function(val) { 137 | //_json_debug && console.log( "Recover:", val[0] ); 138 | disk[val[0]] = val[1]; 139 | } ); 140 | stream.on('open', function() { 141 | _debug && console.log( new Date(), "Read stream opened.." ); 142 | } ); 143 | stream.on('error', function( err ){ 144 | if( err.code !== 'ENOENT' ) 145 | console.log( "READ STREAM ERROR:", err ); 146 | loaded = true; 147 | fileState.reading = false; 148 | handlePending(); 149 | } ); 150 | stream.on('data', function(chunk) { 151 | //_json_debug && console.log( "got stream data",chunk ); 152 | parser.write( chunk ); 153 | }); 154 | stream.on( "close", function(){ 155 | _debug && console.log( new Date(), "reading done..." ); 156 | loaded = true; 157 | fileState.reading = false; 158 | //console.log( "File done" ); 159 | _debug && console.log( "Handle ", pending.length, "pending reads" ); 160 | handlePending(); 161 | } ); 162 | function handlePending() { 163 | while( pendPut = pendingPut.shift() ) { 164 | doPut( pendPut ); 165 | } 166 | while( pend = pending.shift() ) { 167 | var data = disk[pend.soul] || pend.u; 168 | if(data && pend.field){ 169 | data = Gun.state.to(data, pend.field); 170 | } 171 | _debug && console.log( "Sending pending response...", pend.at[SEQ_] ); 172 | pend.ctx.on('in', {[ACK_]: pend.at[SEQ_], put: Gun.graph.node(data)}); 173 | _debug && console.log( "Sent pending response...", pend.at[SEQ_] ); 174 | } 175 | if( wantFlush ) { 176 | _debug && console.log( "WANTED FLUSH during READ?", to ); 177 | if(to) clearTimeout( to ); 178 | to = setTimeout(flush, fileDelay ); 179 | } 180 | } 181 | } 182 | 183 | var startWrite; 184 | var flush = function(){ 185 | //_debug && console.log( Date.now(), "DOING FLUSH", reading, writing, count ); 186 | if( to ) 187 | clearTimeout(to); 188 | to = null; 189 | count = 0; 190 | if(fileState.reading) { 191 | _debug && console.log( "Still reading, don't write", fileState.flushPending ); 192 | to = setTimeout( flush, fileDelay/3 ); 193 | fileState.flushPending = true; 194 | wantFlush = true; 195 | return; 196 | } 197 | if(fileState.writing){ 198 | to = setTimeout( flush, fileDelay ); 199 | _debug && console.log( "Still flushing, don't flush", fileState.flushPending ); 200 | fileState.flushPending = true; 201 | return; 202 | } 203 | _debug && console.log( Date.now(), "DOING FLUSH", fileState.reading, fileState.writing, count ); 204 | 205 | var ack = acks; 206 | acks = {}; 207 | fileState.writing = true; 208 | fileState.flushPending = false; 209 | var pretty = filePretty?3:null; 210 | var stream = fs.createWriteStream( fileName, {encoding:'utf8', mode:fileMode, flags:"w+"} ); 211 | var waitDrain = false; 212 | stream.on('open', function () { 213 | var keys = Object.keys( fileState.disk ); 214 | var n = 0; 215 | if( _debug_write_time ) startWrite = Date.now(); 216 | _debug && console.log( new Date(), "stream write opened..." ); 217 | function writeOne() { 218 | if( n >= keys.length ) { 219 | fileState.writing = false; 220 | _debug_write_time && console.log( "Write to file:", Date.now() - startWrite ); 221 | _debug && console.log( "done; closing stream." ); 222 | stream.end(); 223 | //var tmp = count; 224 | count = 0; 225 | Gun.obj.map(ack, function(yes, id){ 226 | ctx.on('in', { 227 | [ACK_]: id, 228 | gun: gun, 229 | err: false, 230 | ok: 1 231 | }); 232 | }); 233 | //if(1 < tmp){ flush() } 234 | return; 235 | } 236 | var key = keys[n++]; 237 | var out = JSON.stringify( [key, fileState.disk[key]], null, pretty ) + (pretty?"\n":""); 238 | if( !stream.write( out, 'utf8', function() { 239 | if( !waitDrain ){ 240 | if( fileState.writing ) writeOne(); // otherwise already completed writing everything. 241 | } 242 | else { 243 | //_debug && console.log( "Skipped doing next?" ); 244 | } 245 | } ) ) { 246 | // writing for on_something before next write. 247 | waitDrain = true; 248 | //_debug && console.log( "wait DRAIN" ); 249 | stream.once('drain', function(){ 250 | //_debug && console.log( "Continue after drain happened." ); 251 | if( waitDrain ) { 252 | waitDrain = false; 253 | writeOne(); 254 | } else { 255 | // just the last write finishing don't retrigger. 256 | } 257 | }); 258 | } 259 | } 260 | writeOne(); 261 | } ); 262 | 263 | } 264 | }); 265 | } 266 | 267 | exports.attach( Gun ); -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gun-file", 3 | "version": "1.0.123", 4 | "description": "Streaming JSON file driver for Gun.", 5 | "main": "file.js", 6 | "scripts": { 7 | "test": "node test.js" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "https://github.com/d3x0r/gun-file" 12 | }, 13 | "keywords": [ 14 | "gun", 15 | "gundb", 16 | "file", 17 | "database", 18 | "json", 19 | "json6" 20 | ], 21 | "author": "d3x0r", 22 | "license": "ISC", 23 | "dependencies": { 24 | "gun": "^0.2019.416", 25 | "json-6": "^0" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | 2 | 3 | var Gun = require( "gun/gun" ); 4 | var gunNot = require('gun/lib/not') 5 | var gunFile = require( "." ); 6 | 7 | var gun = new Gun( { 'file-name':'data.other.json' } ); 8 | 9 | var root = gun.get( "db" ); 10 | 11 | root.not( ()=>{ 12 | console.log( "not happened." ); 13 | root.put( { hello:"world" } ); 14 | root.put( { other:"test" } ); 15 | root.set( { field: "randomkey" } ); 16 | root.set( { field: "randomkey" } ); 17 | root.set( { field: "randomkey" } ); 18 | } ); 19 | 20 | var count = 0; 21 | var start = Date.now(); 22 | var first = false; 23 | function showItems() { 24 | console.log( "Got", count, "items" ); 25 | } 26 | var timeout; 27 | root.map( (field,val)=>{ 28 | if( !first ) { 29 | console.log( "first map in ", Date.now() - start ); 30 | first = true; 31 | } 32 | count++; 33 | if( timeout ) 34 | clearTimeout( timeout ); 35 | timeout = setTimeout( showItems, 100 ); 36 | //console.log( "Got:", val, field ) 37 | if( val == "hello" ) { 38 | for( var n = 0; n < 10000; n++ ) 39 | root.set( { field: "randomkey" } ); 40 | } 41 | } ) 42 | 43 | 44 | --------------------------------------------------------------------------------