├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── index.js ├── package.json └── test └── index.test.js /.gitignore: -------------------------------------------------------------------------------- 1 | # IDEA files 2 | .idea 3 | *.iml 4 | 5 | # Logs 6 | logs 7 | *.log 8 | npm-debug.log* 9 | 10 | # Runtime data 11 | pids 12 | *.pid 13 | *.seed 14 | 15 | # Directory for instrumented libs generated by jscoverage/JSCover 16 | lib-cov 17 | 18 | # Coverage directory used by tools like istanbul 19 | coverage 20 | 21 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 22 | .grunt 23 | 24 | # node-waf configuration 25 | .lock-wscript 26 | 27 | # Compiled binary addons (http://nodejs.org/api/addons.html) 28 | build/Release 29 | 30 | # Dependency directory 31 | node_modules 32 | 33 | # Optional npm cache directory 34 | .npm 35 | 36 | # Optional REPL history 37 | .node_repl_history 38 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "5" 4 | - "5.1" 5 | - "4" 6 | - "4.2" 7 | - "4.1" 8 | - "4.0" 9 | - "iojs" -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Ilya Sheershoff 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![NPM](https://nodei.co/npm/cache-manager-fs-binary.png)](https://nodei.co/npm/cache-manager-fs-binary/) 2 | [![Build Status](https://api.travis-ci.org/sheershoff/node-cache-manager-fs-binary.png)](https://travis-ci.org/sheershoff/node-cache-manager-fs-binary) 3 | 4 | # node-cache-manager-fs-binary 5 | 6 | Node Cache Manager store for Filesystem with binary data 7 | ======================================================== 8 | 9 | The Filesystem store for the [node-cache-manager](https://github.com/BryanDonovan/node-cache-manager) module, storing binary data as separate files, returning them as readable streams or buffers. 10 | This should be convenient for caching binary data and sending them as streams to a consumer, e.g. `res.send()`. 11 | The library caches on disk arbitrary data, but values of an object under the special key `binary` is stored as separate files. 12 | 13 | Node.js versions 14 | ---------------- 15 | 16 | Works with versions 4, 5 and iojs. 17 | 18 | Installation 19 | ------------ 20 | 21 | ```sh 22 | npm install cache-manager-fs-binary --save 23 | ``` 24 | 25 | Usage examples 26 | -------------- 27 | 28 | Here are examples that demonstrate how to implement the Filesystem cache store. 29 | 30 | 31 | ## Features 32 | 33 | * limit maximum size on disk 34 | * refill cache on startup (in case of application restart) 35 | * returns binary data as buffers or readable streams (keys of the `binary` key) 36 | * can store buffers inside the single cache file (not keys of the `binary` key) 37 | 38 | ## Single store 39 | 40 | ```javascript 41 | // node cachemanager 42 | var cacheManager = require('cache-manager'); 43 | // storage for the cachemanager 44 | var fsStore = require('cache-manager-fs-binary'); 45 | // initialize caching on disk 46 | var diskCache = cacheManager.caching({ 47 | store: fsStore, 48 | options: { 49 | reviveBuffers: true, 50 | binaryAsStream: true, 51 | ttl: 60 * 60 /* seconds */, 52 | maxsize: 1000 * 1000 * 1000 /* max size in bytes on disk */, 53 | path: 'diskcache', 54 | preventfill: true 55 | } 56 | }); 57 | 58 | // ... 59 | var cacheKey = 'userImageWatermarked:' + user.id + ':' + image.id; 60 | var ttl = 60 * 60 * 24 * 7; // in seconds 61 | 62 | // wrapper function, see more examples at node-cache-manager 63 | diskCache.wrap(cacheKey, 64 | // called if the cache misses in order to generate the value to cache 65 | function (cacheCallback) { 66 | var image; // buffer that will be saved to separate file 67 | var moreData; // string that will be saved to a separate file 68 | var userLastVisit; // Date 69 | var signature; // small binary data to store inside as buffer 70 | 71 | // ... generating the image 72 | 73 | // now returning value to cache and process further 74 | cacheCallback(err, 75 | // Some JSONable object. Note that null and undefined values not stored. 76 | // One can redefine isCacheableValue method to tweak the behavior. 77 | { 78 | binary: { 79 | // These will be saved to separate files and returned as buffers or 80 | // readable streams depending on the cache settings. 81 | // **NB!** The initial values will be changed to buffers or readable streams 82 | // for the sake of simplicity, usability and lowering the memory footprint. 83 | // Check that the keys are suitable as parts of filenames. 84 | image: image, 85 | someOtherBinaryData: moreData 86 | }, 87 | // Other data are saved into the main cache file 88 | someArbitraryValues: { 89 | eg: userLastVisit 90 | }, 91 | someSmallBinaryValues: { 92 | // While buffer data could be saved to the main file, it is strongly 93 | // discouraged to do so for large buffers, since they are stored in JSON 94 | // as Array of bytes. Use wisely, do the benchmarks, mind inodes, disk 95 | // space and performance balance. 96 | pgpSignatureBufferForm: signature 97 | } 98 | }); 99 | }, 100 | // Options, see node-cache-manager for more examples 101 | {ttl: ttl}, 102 | // Do your work on the cached or freshly generated and cached value. 103 | // Note, that result.binary.image will come in readable stream form 104 | // in the result, if binaryAsStream is true 105 | function (err, result) { 106 | 107 | res.writeHead(200, {'Content-Type': 'image/jpeg'}); 108 | var image = result.binary.image; 109 | 110 | image.pipe(res); 111 | 112 | var usedStreams = ['image']; 113 | // you have to do the work to close the unused files 114 | // to prevent file descriptors leak 115 | for (var key in result.binary) { 116 | if (!result.binary.hasOwnProperty(key))continue; 117 | if (usedStreams.indexOf(key) < 0 118 | && result.binary[key] instanceof Stream.Readable) { 119 | if(typeof result.binary[key].close === 'function') { 120 | result.binary[key].close(); // close the stream (fs has it) 121 | }else{ 122 | result.binary[key].resume(); // resume to the end and close 123 | } 124 | } 125 | } 126 | } 127 | ) 128 | ``` 129 | 130 | ### Options 131 | 132 | options for store initialization 133 | 134 | ```javascript 135 | 136 | // default values 137 | 138 | // time to live in seconds 139 | options.ttl = 60; 140 | // path for cached files 141 | options.path = 'cache/'; 142 | // prevent filling of the cache with the files from the cache-directory 143 | options.preventfill = false; 144 | // callback fired after the initial cache filling is completed 145 | options.fillcallback = null; 146 | // if true the main cache files will be zipped (not the binary ones) 147 | options.zip = false; 148 | // if true buffers not from binary key are returned from cache as buffers, 149 | // not objects 150 | options.reviveBuffers = false; 151 | // if true, data in the binary key are returned as StreamReadable and 152 | // (**NB!**) the source object will also be changed. 153 | // You have to do the work for closing the files if you do not read them, 154 | // see example. 155 | options.binaryAsStream = false; 156 | 157 | ``` 158 | 159 | ## Tests 160 | 161 | To run tests: 162 | 163 | ```sh 164 | npm test 165 | ``` 166 | 167 | ## Code Coverage 168 | 169 | To run Coverage: 170 | 171 | ```sh 172 | npm run coverage 173 | ``` 174 | 175 | ## License 176 | 177 | cache-manager-fs-binary is licensed under the MIT license. 178 | 179 | ## Credits 180 | 181 | Based on https://github.com/hotelde/node-cache-manager-fs 182 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * Module dependencies 5 | */ 6 | var noop = function () { 7 | }; 8 | var fs = require("fs"); 9 | var fsp = require("fs-promise"); 10 | var path = require('path'); 11 | var async = require('async'); 12 | var extend = require('extend'); 13 | var uuid = require('uuid'); 14 | var zlib = require('zlib'); 15 | var glob = require('glob'); 16 | var streamifier = require('streamifier'); 17 | 18 | /** 19 | * Export 'DiskStore' 20 | */ 21 | 22 | module.exports = { 23 | create: function (args) { 24 | return new DiskStore(args && args.options ? args.options : args); 25 | } 26 | }; 27 | 28 | /** 29 | * Helper function that revives buffers from object representation on JSON.parse 30 | */ 31 | function bufferReviver(k, v) { 32 | if ( 33 | v !== null && 34 | typeof v === 'object' && 35 | 'type' in v && 36 | v.type === 'Buffer' && 37 | 'data' in v && 38 | Array.isArray(v.data)) { 39 | return new Buffer(v.data); 40 | } 41 | return v; 42 | } 43 | 44 | /** 45 | * helper object with meta-informations about the cached data 46 | */ 47 | function MetaData() { 48 | 49 | // the key for the storing 50 | this.key = null; 51 | // data to store 52 | this.value = null; 53 | // temporary filename for the cached file because filenames cannot represend urls completely 54 | this.filename = null; 55 | // expirydate of the entry 56 | this.expires = null; 57 | // size of the current entry 58 | this.size = null; 59 | } 60 | 61 | /** 62 | * construction of the disk storage 63 | */ 64 | function DiskStore(options) { 65 | options = options || {}; 66 | 67 | this.options = extend({ 68 | path: 'cache/', 69 | ttl: 60, 70 | maxsize: 0, 71 | zip: false 72 | }, options); 73 | 74 | 75 | // check storage directory for existence (or create it) 76 | if (!fs.existsSync(this.options.path)) { 77 | fs.mkdirSync(this.options.path); 78 | } 79 | 80 | this.name = 'diskstore'; 81 | 82 | // current size of the cache 83 | this.currentsize = 0; 84 | 85 | // internal array for informations about the cached files - resists in memory 86 | this.collection = {}; 87 | 88 | // fill the cache on startup with already existing files 89 | if (!options.preventfill) { 90 | 91 | this.intializefill(options.fillcallback); 92 | } 93 | } 94 | 95 | 96 | /** 97 | * indicate, whether a key is cacheable 98 | */ 99 | DiskStore.prototype.isCacheableValue = function (value) { 100 | 101 | return value !== null && value !== undefined; 102 | }; 103 | 104 | /** 105 | * delete an entry from the cache 106 | */ 107 | DiskStore.prototype.del = function (key, options, cb) { 108 | 109 | if (typeof options === 'function') { 110 | cb = options; 111 | options = null; 112 | } 113 | cb = typeof cb === 'function' ? cb : noop; 114 | 115 | // get the metainformations for the key 116 | var metaData = this.collection[key]; 117 | if (!metaData) { 118 | return cb(null); 119 | } 120 | 121 | // check if the filename is set 122 | if (!metaData.filename) { 123 | return cb(null); 124 | } 125 | // check for existance of the file 126 | fsp.readFile(metaData.filename, { encoding : 'ascii' }). 127 | then(function(metaExtraContent) { 128 | // delete the files 129 | if (! metaExtraContent) { 130 | reject(); 131 | return; 132 | } 133 | 134 | try { 135 | var metaExtra = JSON.parse(metaExtraContent); 136 | } 137 | catch(e) { 138 | reject(); 139 | return; 140 | } 141 | 142 | if (metaExtra.value && metaExtra.value.binary && typeof metaExtra.value.binary === 'object' && metaExtra.value.binary != null) { 143 | // unlink binaries 144 | async.forEachOf(metaExtra.value.binary, function (v, k, cb) { 145 | fs.unlink(metaExtra.value.binary[k], cb); 146 | }, function (err) { 147 | }); 148 | return fsp.unlink(metaData.filename); 149 | } else { 150 | return fsp.unlink(metaData.filename); 151 | } 152 | }, function () { 153 | // not found 154 | cb(null); 155 | }).then(function () { 156 | // update internal properties 157 | this.currentsize -= metaData.size; 158 | this.collection[key] = null; 159 | delete this.collection[key]; 160 | cb(null); 161 | }.bind(this)).catch(function (err) { 162 | cb(null); 163 | }); 164 | }; 165 | 166 | 167 | /** 168 | * zip an input string if options want that 169 | */ 170 | DiskStore.prototype.zipIfNeeded = function (data, cb) { 171 | if (this.options.zip) { 172 | zlib.deflate(data, function (err, buffer) { 173 | if (!err) { 174 | cb(null, buffer); 175 | } 176 | else { 177 | cb(err, null); 178 | } 179 | }); 180 | } 181 | else { 182 | cb(null, data); 183 | } 184 | } 185 | 186 | /** 187 | * zip an input string if options want that 188 | */ 189 | DiskStore.prototype.unzipIfNeeded = function (data, cb) { 190 | if (this.options.zip) { 191 | zlib.unzip(data, function (err, buffer) { 192 | if (!err) { 193 | cb(null, buffer); 194 | } 195 | else { 196 | cb(err, null); 197 | } 198 | }); 199 | } 200 | else { 201 | cb(null, data); 202 | } 203 | } 204 | 205 | /** 206 | * set a key into the cache 207 | */ 208 | DiskStore.prototype.set = function (key, val, options, cb) { 209 | 210 | cb = typeof cb === 'function' ? cb : noop; 211 | 212 | if (typeof options === 'function') { 213 | cb = options; 214 | options = null; 215 | } 216 | 217 | // get ttl 218 | var ttl = (options && (options.ttl || options.ttl === 0)) ? options.ttl : this.options.ttl; 219 | 220 | // move binary data to binary from value 221 | var binary; 222 | if (typeof val.binary === 'object' && val.binary != null) { 223 | binary = val.binary; 224 | delete val['binary']; 225 | val.binary = {}; 226 | } 227 | 228 | var metaData = extend({}, new MetaData(), { 229 | key: key, 230 | value: val, 231 | expires: Date.now() + ((ttl || 60) * 1000), 232 | filename: this.options.path + '/cache_' + uuid.v4() + '.dat' 233 | }); 234 | 235 | var binarySize = 0; 236 | 237 | if (binary) { 238 | for (var binkey in binary) { 239 | if (!binary.hasOwnProperty(binkey)) continue; 240 | // put storage filenames into stored value.binary object 241 | metaData.value.binary[binkey] = metaData.filename.replace(/\.dat$/, '_' + binkey + '.bin'); 242 | // calculate the size of the binary data 243 | binarySize += binary[binkey].length || 0; 244 | } 245 | } 246 | 247 | metaData.size = JSON.stringify(metaData).length + binarySize; 248 | 249 | var stream = JSON.stringify(metaData); 250 | 251 | if (this.options.maxsize && metaData.size > this.options.maxsize) { 252 | return cb(new Error('Item size too big.')); 253 | } 254 | 255 | 256 | // remove the key from the cache (if it already existed, this updates also the current size of the store) 257 | this.del(key, function (err) { 258 | 259 | if (err) { 260 | return cb(err); 261 | } 262 | 263 | // check used space and remove entries if we use too much space 264 | this.freeupspace(function () { 265 | 266 | try { 267 | 268 | var self = this; 269 | // write binary data and cache file 270 | async.series( 271 | [ 272 | function (cb) { 273 | // write binary 274 | if (binary) { 275 | async.forEachOf(binary, function (v, k, cb) { 276 | fs.writeFile(metaData.value.binary[k], v, cb); 277 | }, function (err) { 278 | cb(err); 279 | }); 280 | } else { 281 | cb(); 282 | } 283 | }, 284 | function (cb) { 285 | self.zipIfNeeded(stream, function (err, processedStream) { 286 | 287 | // write data into the cache-file 288 | fs.writeFile(metaData.filename, processedStream, function (err) { 289 | 290 | if (err) { 291 | return cb(err); 292 | } 293 | 294 | // remove data value from memory 295 | metaData.value = null; 296 | delete metaData.value; 297 | 298 | self.currentsize += metaData.size; 299 | 300 | // place element with metainfos in internal collection 301 | self.collection[metaData.key] = metaData; 302 | 303 | // restore val binary key 304 | if (binary) { 305 | if(this.options.binaryAsStream) { 306 | for(key in binary){ 307 | if(binary.hasOwnProperty(key)){ 308 | val.binary[key] = streamifier.createReadStream(binary[key], {encoding: null}); 309 | } 310 | } 311 | }else{ 312 | val.binary = binary; 313 | } 314 | } 315 | 316 | return cb(null, val); 317 | 318 | }.bind(self)); 319 | }.bind(self)) 320 | } 321 | ], 322 | function (err, result) { 323 | cb(err, result); 324 | } 325 | ); 326 | 327 | } catch (err) { 328 | 329 | return cb(err); 330 | } 331 | 332 | }.bind(this)); 333 | 334 | }.bind(this)); 335 | 336 | }; 337 | 338 | /** 339 | * helper method to free up space in the cache (regarding the given spacelimit) 340 | */ 341 | DiskStore.prototype.freeupspace = function (cb) { 342 | 343 | cb = typeof cb === 'function' ? cb : noop; 344 | 345 | if (!this.options.maxsize) { 346 | return cb(null); 347 | } 348 | 349 | // do we use to much space? then cleanup first the expired elements 350 | if (this.currentsize > this.options.maxsize) { 351 | this.cleanExpired(); 352 | } 353 | 354 | // when the spaceusage is to high, remove the oldest entries until we gain enough diskspace 355 | if (this.currentsize <= this.options.maxsize) { 356 | return cb(null); 357 | } 358 | 359 | // for this we need a sorted list basend on the expire date of the entries (descending) 360 | var tuples = [], key; 361 | for (key in this.collection) { 362 | tuples.push([key, this.collection[key].expires]); 363 | } 364 | 365 | tuples.sort(function sort(a, b) { 366 | 367 | a = a[1]; 368 | b = b[1]; 369 | return a < b ? 1 : (a > b ? -1 : 0); 370 | }); 371 | 372 | return this.freeupspacehelper(tuples, cb); 373 | }; 374 | 375 | /** 376 | * freeup helper for asnyc space freeup 377 | */ 378 | DiskStore.prototype.freeupspacehelper = function (tuples, cb) { 379 | 380 | // check, if we have any entry to process 381 | if (tuples.length === 0) { 382 | return cb(null); 383 | } 384 | 385 | // get an entry from the list 386 | var tuple = tuples.pop(); 387 | var key = tuple[0]; 388 | 389 | // delete an entry from the store 390 | this.del(key, function deleted(err) { 391 | 392 | // return when an error occures 393 | if (err) { 394 | return cb(err); 395 | } 396 | 397 | // stop processing when enouth space has been cleaned up 398 | if (this.currentsize <= this.options.maxsize) { 399 | return cb(err); 400 | } 401 | 402 | // ok - we need to free up more space 403 | return this.freeupspacehelper(tuples, cb); 404 | }.bind(this)); 405 | }; 406 | 407 | /** 408 | * get entry from the cache 409 | */ 410 | DiskStore.prototype.get = function (key, options, cb) { 411 | 412 | if (typeof options === 'function') { 413 | cb = options; 414 | } 415 | 416 | cb = typeof cb === 'function' ? cb : noop; 417 | 418 | // get the metadata from the collection 419 | var data = this.collection[key]; 420 | 421 | if (!data) { 422 | 423 | // not found 424 | return cb(null, null); 425 | } 426 | 427 | // found but expired 428 | if (data.expires < new Date()) { 429 | 430 | // delete the elemente from the store 431 | this.del(key, function (err) { 432 | return cb(err, null); 433 | }); 434 | } else { 435 | 436 | // try to read the file 437 | try { 438 | 439 | fs.readFile(data.filename, function (err, fileContent) { 440 | if (err) { 441 | return cb(err); 442 | } 443 | var reviveBuffers = this.options.reviveBuffers; 444 | var binaryAsStream = this.options.binaryAsStream; 445 | var zipOption = this.options.zip; 446 | async.waterfall( 447 | [function (seriescb) { 448 | if (zipOption) { 449 | zlib.unzip(fileContent, function (err, buffer) { 450 | var diskdata; 451 | if (reviveBuffers) { 452 | diskdata = JSON.parse(buffer, bufferReviver); 453 | } else { 454 | diskdata = JSON.parse(buffer); 455 | } 456 | seriescb(null, diskdata); 457 | }); 458 | } 459 | else { 460 | var diskdata; 461 | if (reviveBuffers) { 462 | diskdata = JSON.parse(fileContent, bufferReviver); 463 | } else { 464 | diskdata = JSON.parse(fileContent); 465 | } 466 | seriescb(null, diskdata); 467 | } 468 | }, 469 | function (diskdata, seriescb) { 470 | if (diskdata && diskdata.value && diskdata.value.binary && diskdata.value.binary != null && typeof diskdata.value.binary == 'object') { 471 | async.forEachOf(diskdata.value.binary, function (v, k, cb) { 472 | diskdata.value.binary[k] = fs.createReadStream(v, { 473 | autoClose: true, 474 | encoding: null 475 | }); 476 | if (binaryAsStream) { 477 | cb(); 478 | } else { 479 | var bufs = []; 480 | diskdata.value.binary[k].on('data', function (d) { 481 | bufs.push(Buffer(d)); 482 | }); 483 | diskdata.value.binary[k].on('error', function (err) { 484 | cb(err); 485 | }); 486 | diskdata.value.binary[k].on('end', function () { 487 | diskdata.value.binary[k] = Buffer.concat(bufs); 488 | cb(); 489 | }); 490 | } 491 | }, function (err) { 492 | if (err) 493 | return seriescb(err); 494 | seriescb(null, diskdata.value); 495 | }); 496 | } else { 497 | seriescb(null, diskdata.value); 498 | } 499 | }], function(err, result){ 500 | cb(err, result); 501 | }) 502 | }.bind(this)); 503 | 504 | } catch (err) { 505 | 506 | cb(err); 507 | } 508 | } 509 | }; 510 | 511 | /** 512 | * get keys stored in cache 513 | * @param {Function} cb 514 | */ 515 | DiskStore.prototype.keys = function (cb) { 516 | 517 | cb = typeof cb === 'function' ? cb : noop; 518 | 519 | var keys = Object.keys(this.collection); 520 | 521 | cb(null, keys); 522 | }; 523 | 524 | /** 525 | * cleanup cache on disk -> delete all used files from the cache 526 | */ 527 | DiskStore.prototype.reset = function (key, cb) { 528 | 529 | cb = typeof cb === 'function' ? cb : noop; 530 | 531 | if (typeof key === 'function') { 532 | cb = key; 533 | key = null; 534 | } 535 | 536 | if (Object.keys(this.collection).length === 0) { 537 | return cb(null); 538 | } 539 | 540 | try { 541 | 542 | // delete special key 543 | if (key !== null) { 544 | 545 | this.del(key); 546 | return cb(null); 547 | } 548 | 549 | async.eachSeries(this.collection, 550 | function (elementKey, callback) { 551 | this.del(elementKey.key, callback); 552 | }.bind(this), 553 | function (err) { 554 | cb(null); 555 | }.bind(this) 556 | ); 557 | 558 | } catch (err) { 559 | 560 | return cb(err); 561 | } 562 | 563 | }; 564 | 565 | /** 566 | * helper method to clean all expired files 567 | */ 568 | DiskStore.prototype.cleanExpired = function () { 569 | 570 | var key, entry; 571 | 572 | for (key in this.collection) { 573 | 574 | entry = this.collection[key]; 575 | 576 | if (entry.expires < new Date()) { 577 | 578 | this.del(entry.key); 579 | } 580 | } 581 | } 582 | 583 | /** 584 | * clean the complete cache and all(!) files in the cache directory 585 | */ 586 | DiskStore.prototype.cleancache = function (cb) { 587 | 588 | cb = typeof cb === 'function' ? cb : noop; 589 | 590 | // clean all current used files 591 | this.reset(); 592 | 593 | // check, if other files still resist in the cache and clean them, too 594 | var files = fs.readdirSync(this.options.path); 595 | 596 | files 597 | .map(function (file) { 598 | 599 | return path.join(this.options.path, file); 600 | }.bind(this)) 601 | .filter(function (file) { 602 | 603 | return fs.statSync(file).isFile(); 604 | }.bind(this)) 605 | .forEach(function (file) { 606 | 607 | fs.unlinkSync(file); 608 | }.bind(this)); 609 | 610 | cb(null); 611 | 612 | }; 613 | 614 | /** 615 | * fill the cache from the cache directory (usefull e.g. on server/service restart) 616 | */ 617 | DiskStore.prototype.intializefill = function (cb) { 618 | 619 | cb = typeof cb === 'function' ? cb : noop; 620 | 621 | // get the current working directory 622 | fs.readdir(this.options.path, function (err, files) { 623 | 624 | // get potential files from disk 625 | files = files.map(function (filename) { 626 | 627 | return path.join(this.options.path, filename); 628 | }.bind(this)).filter(function (filename) { 629 | 630 | return fs.statSync(filename).isFile(); 631 | }); 632 | 633 | // use async to process the files and send a callback after completion 634 | async.eachSeries(files, function (filename, callback) { 635 | 636 | if (!/\.dat$/.test(filename)) { // only .dat files, no .bin files read 637 | return callback(); 638 | } 639 | 640 | fs.readFile(filename, function (err, data) { 641 | 642 | // stop file processing when there was an reading error 643 | if (err) { 644 | return callback(); 645 | } 646 | this.unzipIfNeeded(data, function(err,data) { 647 | try { 648 | if(err){ // if unzippable - throw to remove 649 | throw Error('unzippable: ' + err); 650 | } 651 | // get the json out of the data 652 | var diskdata = JSON.parse(data); 653 | 654 | } catch (err) { 655 | 656 | // when the deserialize doesn't work, probably the file is uncomplete - so we delete it and ignore the error 657 | try { 658 | fs.unlinksync(filename); 659 | // unlink binary 660 | glob(filename.replace(/\.dat$/, '*.bin'), function (err, result) { 661 | if (!err) { 662 | async.each(result, fs.unlink); 663 | } 664 | }); 665 | } catch (ignore) { 666 | 667 | } 668 | 669 | return callback(); 670 | } 671 | 672 | // update the size in the metadata - this value isn't correctly stored in the file 673 | // diskdata.size = data.length; 674 | 675 | // update collection size 676 | this.currentsize += diskdata.size; 677 | 678 | // remove the entrys content - we don't want the content in the memory (only the meta informations) 679 | diskdata.value = null; 680 | delete diskdata.value; 681 | 682 | // and put the entry in the store 683 | this.collection[diskdata.key] = diskdata; 684 | 685 | // check for expiry - in this case we instantly delete the entry 686 | if (diskdata.expires < new Date()) { 687 | 688 | this.del(diskdata.key, function () { 689 | 690 | return callback(); 691 | }); 692 | } else { 693 | 694 | return callback(); 695 | } 696 | }.bind(this)); 697 | }.bind(this)); 698 | 699 | }.bind(this), function (err) { 700 | 701 | cb(err || null); 702 | 703 | }); 704 | 705 | }.bind(this)); 706 | 707 | }; 708 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cache-manager-fs-binary", 3 | "version": "1.0.4", 4 | "description": "file system store for node cache manager with binary data as files", 5 | "keywords": [ 6 | "cache-manager", 7 | "storage", 8 | "filesystem" 9 | ], 10 | "repository": { 11 | "type": "git", 12 | "url": "git+https://github.com/sheershoff/node-cache-manager-fs-binary.git" 13 | }, 14 | "bugs": { 15 | "url": "https://github.com/sheershoff/node-cache-manager-fs-binary/issues" 16 | }, 17 | "main": "index.js", 18 | "scripts": { 19 | "test": "mocha", 20 | "coverage": "npm run create-coverage && npm run show-coverage", 21 | "create-coverage": "node ./node_modules/istanbul/lib/cli.js cover ./node_modules/mocha/bin/_mocha -- -R spec -t 5000", 22 | "show-coverage": "start coverage\\lcov-report\\index.html" 23 | }, 24 | "author": "Ilya Sheershoff", 25 | "license": "MIT", 26 | "dependencies": { 27 | "async": "^1.4.2", 28 | "cache-manager": "^1.1.0", 29 | "extend": "^3.0.0", 30 | "fs-promise": "^0.3.1", 31 | "glob": "^7.0.3", 32 | "streamifier": "^0.1.1", 33 | "uuid": "^2.0.1" 34 | }, 35 | "devDependencies": { 36 | "chai": "^3.3.0", 37 | "istanbul": "^0.4.0", 38 | "mocha": "^2.3.3", 39 | "buffer-equal": "^1.0.0" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /test/index.test.js: -------------------------------------------------------------------------------- 1 | var assert = require('chai').assert; 2 | var store = require('../index.js') 3 | var fs = require('fs'); 4 | var stream =require('stream'); 5 | var cacheDirectory = 'test/customCache'; 6 | var bufferEqual = require('buffer-equal'); 7 | 8 | describe('test for the hde-disk-store module', function () { 9 | 10 | // remove test directory after run 11 | after(function (done) { 12 | // create a test store 13 | var s=store.create({options: {path:cacheDirectory, preventfill:true}}); 14 | 15 | // cleanup all entries in the cache 16 | s.cleancache(function (err) { 17 | assert(err === null); 18 | // and remove test data directory 19 | setTimeout(function () { 20 | 21 | fs.rmdirSync(s.options.path); 22 | done(); 23 | }, 100); 24 | }); 25 | }); 26 | 27 | describe('construction', function () { 28 | 29 | it('simple create cache test', function () 30 | { 31 | // create a store with default values 32 | var s = store.create(); 33 | // remove folder after testrun 34 | after(function () { fs.rmdirSync(s.options.path); }); 35 | // check the creation result 36 | assert.isObject(s); 37 | assert.isObject(s.options); 38 | assert.isTrue(fs.existsSync(s.options.path)); 39 | }); 40 | 41 | it('create cache with option path test', function () { 42 | // create a store 43 | var s = store.create({options: {path:cacheDirectory, preventfill:true}}); 44 | // check path option creation 45 | assert.isObject(s); 46 | assert.isObject(s.options); 47 | assert.isTrue(fs.existsSync(s.options.path)); 48 | assert(s.options.path == cacheDirectory); 49 | }); 50 | }); 51 | 52 | describe('get', function () { 53 | 54 | it('simple get test with not existing key', function (done) 55 | { 56 | var s=store.create({options: {path:cacheDirectory, preventfill:true}}); 57 | s.get('asdf', function (err, data) 58 | { 59 | assert(data === null); 60 | done(); 61 | }); 62 | }); 63 | 64 | describe('test missing file on disk', function() { 65 | it('filename empty', function (done){ 66 | var s=store.create({options: {path:cacheDirectory, preventfill:true}}); 67 | s.set('test','test', function (err) 68 | { 69 | assert(err === null); 70 | var tmpfilename = s.collection['test'].filename; 71 | s.collection['test'].filename = null; 72 | s.get('test', function (err,data) { 73 | assert(err !== null); 74 | assert(data == null); 75 | s.collection['test'].filename = tmpfilename; 76 | s.del('test', function (err) 77 | { 78 | assert(err == null); 79 | done(); 80 | }); 81 | }) 82 | }); 83 | }); 84 | 85 | it('file does not exist', function (done){ 86 | var s=store.create({options: {path:cacheDirectory, preventfill:true}}); 87 | s.set('test','test', function (err) 88 | { 89 | assert(err === null); 90 | var tmpfilename = s.collection['test'].filename; 91 | s.collection['test'].filename = "bla"; 92 | s.get('test', function (err,data) { 93 | assert(err !== null); 94 | assert(data == null); 95 | s.collection['test'].filename = tmpfilename; 96 | s.del('test', function (err) 97 | { 98 | assert(err == null); 99 | done(); 100 | }); 101 | }) 102 | }); 103 | }); 104 | }); 105 | 106 | it('test expired of key (and also ttl option on setting)', function (done) 107 | { 108 | var s=store.create({options: {path:cacheDirectory, preventfill:true}}); 109 | s.set('asdf','blabla', {ttl:-1000}, function (err) 110 | { 111 | assert(err === null) 112 | s.get('asdf',function (err,data){ 113 | assert(err === null, 'error is not null!'+err); 114 | assert(data === null); 115 | done(); 116 | }) 117 | }); 118 | }) 119 | }); 120 | 121 | describe('set', function () { 122 | 123 | it('simple set test', function (done) 124 | { 125 | var s=store.create({options: {path:cacheDirectory, preventfill:true}}); 126 | var data = 'a lot of data in a file' 127 | s.set('asdf',data, function (err,data2) 128 | { 129 | assert(err === null); 130 | assert(data2,'check if entry has been returned on insert'); 131 | s.get('asdf', function (err, data2) 132 | { 133 | assert(data2,'check if entry could be retrieved'); 134 | assert(data === data2); 135 | done(); 136 | }); 137 | }); 138 | }); 139 | }); 140 | 141 | describe('keys', function() { 142 | 143 | it('simple keys test', function (done) { 144 | var s=store.create({options: {path:cacheDirectory, preventfill:true}}); 145 | var data = 'just a string with data'; 146 | s.set('key123', data, function (err, data2) { 147 | assert(err === null); 148 | s.keys(function(err, keys) { 149 | assert(err === null); 150 | assert(keys.length === 1); 151 | assert(keys[0] === 'key123'); 152 | done(); 153 | }); 154 | }); 155 | }); 156 | }); 157 | 158 | describe('del / reset', function () { 159 | 160 | it('simple del test for not existing key', function (done) 161 | { 162 | var s=store.create({options: {path:cacheDirectory, preventfill:true}}); 163 | s.del('not existing', function (err) { 164 | done(); 165 | }); 166 | }); 167 | 168 | it('successfull deletion', function (done) 169 | { 170 | var s=store.create({options: {path:cacheDirectory, preventfill:true}}); 171 | s.set('nix','empty', function (err) { 172 | assert(err === null); 173 | s.reset('nix', function (err) { 174 | done(); 175 | }); 176 | }); 177 | }); 178 | 179 | describe('delete errorhandling', function() { 180 | it('file not exists', function(done) { 181 | var s=store.create({options: {path:cacheDirectory, preventfill:true}}); 182 | s.set('test','empty', function(err) { 183 | assert(err === null); 184 | var fn = s.collection['test'].filename; 185 | s.collection['test'].filename = s.collection['test'].filename+".not_here"; 186 | s.del('test', function(err) { 187 | s.collection['test'].filename = fn; 188 | assert(err==null); 189 | done(); 190 | }); 191 | }) 192 | }); 193 | 194 | 195 | it('filename not set', function(done) { 196 | var s=store.create({options: {path:cacheDirectory, preventfill:true}}); 197 | s.set('test','empty', function(err) { 198 | assert(err === null); 199 | var fn = s.collection['test'].filename; 200 | s.collection['test'].filename = null; 201 | s.del('test', function(err) { 202 | s.collection['test'].filename = fn; 203 | assert(err==null); 204 | done(); 205 | }); 206 | }) 207 | }); 208 | 209 | }) 210 | 211 | it('reset all', function(done) { 212 | var s=store.create({options: {path:cacheDirectory, preventfill:true}}); 213 | s.set('test', 'test', function(err){ 214 | assert(err === null); 215 | 216 | s.set('test2', 'test2', function(err) { 217 | assert(err === null); 218 | s.reset(function(err) { 219 | assert(err === null); 220 | 221 | s.keys(function(err, keys) { 222 | assert(err === null); 223 | assert(keys.length === 0); 224 | done(); 225 | }); 226 | }); 227 | }); 228 | }); 229 | }); 230 | 231 | it('reset callback', function (done) 232 | { 233 | var s=store.create({options: {path:cacheDirectory, preventfill:true}}); 234 | s.set('test','test', function (err) 235 | { 236 | assert(err === null); 237 | s.reset(function (error) { 238 | assert(err === null); 239 | done(); 240 | }) 241 | }); 242 | }); 243 | }); 244 | 245 | describe('isCacheableValue', function () { 246 | 247 | it('works', function () { 248 | var s=store.create({options: {path:cacheDirectory, preventfill:true}}); 249 | assert(!s.isCacheableValue(null)); 250 | assert(!s.isCacheableValue(undefined)); 251 | }); 252 | }); 253 | 254 | describe('zip test', function() { 255 | it('save and load again', function(done) { 256 | // create store 257 | var s=store.create({options: {zip:true, path:cacheDirectory, preventfill:true}}); 258 | var datastring = "bla only for test \n and so on..."; 259 | var dataKey = "zipDataTest"; 260 | s.set(dataKey, datastring, function (err) { 261 | assert(err === null); 262 | s.get(dataKey, function (err, data) { 263 | assert(err === null); 264 | assert(data == datastring); 265 | done(); 266 | }); 267 | }); 268 | }) 269 | }); 270 | 271 | describe('buffers portion', function(){ 272 | 273 | it('saves and loads arbitrary buffers with revival', function(done){ 274 | var s=store.create({options: {zip:true, reviveBuffers: true, path:cacheDirectory, preventfill:true}}); 275 | var dataBufferArray = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; 276 | var dataKey = 'bufferRevivalTest'; 277 | var data2Cache = {arbitrary: {testBuffer: Buffer(dataBufferArray)}}; 278 | s.set(dataKey, data2Cache, function (err) { 279 | assert(err === null); 280 | s.get(dataKey, function (err, data) { 281 | try { 282 | assert(err === null); 283 | assert(bufferEqual(data2Cache.arbitrary.testBuffer, data.arbitrary.testBuffer) === true); 284 | done(); 285 | }catch(e){ 286 | done(e); 287 | } 288 | }); 289 | }); 290 | }); 291 | 292 | it('saves and loads binary key buffers without revival', function(done){ 293 | var s=store.create({options: {zip:true, reviveBuffers: false, path:cacheDirectory, preventfill:true}}); 294 | var dataBufferArray = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; 295 | var dataKey = 'binaryBufferTest'; 296 | var data2Cache = {binary: {testBuffer: Buffer(dataBufferArray)}}; 297 | s.set(dataKey, data2Cache, function (err) { 298 | assert(err === null); 299 | s.get(dataKey, function (err, data) { 300 | try { 301 | assert(err === null); 302 | assert(bufferEqual(data2Cache.binary.testBuffer, data.binary.testBuffer) === true); 303 | done(); 304 | }catch(e){ 305 | done(e); 306 | } 307 | }); 308 | }); 309 | }); 310 | 311 | it('saves binary key buffers and loads as readable stream', function(done){ 312 | var s=store.create({options: {zip:true, reviveBuffers: false, binaryAsStream: true, path:cacheDirectory, preventfill:true}}); 313 | var dataBufferArray = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]; 314 | var dataKey = 'binaryBufferReadableStreamTest'; 315 | var data2Cache = {binary: {testBuffer: Buffer(dataBufferArray)}}; 316 | s.set(dataKey, data2Cache, function (err) { 317 | assert(err === null); 318 | s.get(dataKey, function (err, data) { 319 | try { 320 | assert(err === null); 321 | assert(data.binary.testBuffer instanceof stream.Readable, 'Should be stream, but ' + typeof data.binary.testBuffer + ' returned'); 322 | assert(data2Cache.binary.testBuffer instanceof stream.Readable, 'Should be stream, but ' + typeof data2Cache.binary.testBuffer + ' returned'); 323 | var bufs = []; 324 | data.binary.testBuffer.on('data', function (d) { 325 | bufs.push(Buffer(d)); 326 | }); 327 | data.binary.testBuffer.on('error', function (err) { 328 | done(err); 329 | }); 330 | data.binary.testBuffer.on('end', function () { 331 | bufs = Buffer.concat(bufs); 332 | var bufs2 = []; 333 | data2Cache.binary.testBuffer.on('data', function (d) { 334 | bufs2.push(Buffer(d)); 335 | }); 336 | data2Cache.binary.testBuffer.on('error', function (err) { 337 | done(err); 338 | }); 339 | data2Cache.binary.testBuffer.on('end', function () { 340 | bufs2 = Buffer.concat(bufs2); 341 | assert(bufferEqual(bufs2, bufs) === true); 342 | done(); 343 | }); 344 | }); 345 | }catch(e){ 346 | done(e); 347 | } 348 | }); 349 | }); 350 | }); 351 | 352 | }); 353 | 354 | describe('integrationtests', function () { 355 | 356 | it('cache initialization on start', function (done) { 357 | // create store 358 | var s=store.create({options: {path:cacheDirectory, preventfill:true}}); 359 | // save element 360 | s.set('RestoreDontSurvive', 'data', {ttl:-1}, function (err) { 361 | assert(err === null); 362 | s.set('RestoreTest','test', function (err) 363 | { 364 | var t=store.create({options: {path:cacheDirectory, fillcallback: function () { 365 | //fill complete 366 | t.get('RestoreTest', function (err, data) { 367 | assert(data === 'test'); 368 | t.get('RestoreDontSurvive', function (err,data) { 369 | assert(err === null); 370 | assert(data === null); 371 | assert(s.currentsize > 0, 'current size not correctly initialized - '+s.currentsize); 372 | done(); 373 | }); 374 | }); 375 | } 376 | }}); 377 | }); 378 | }); 379 | }); 380 | 381 | it('cache initialization on start with zip option', function (done) { 382 | // create store 383 | var s=store.create({options: {path:cacheDirectory, zip: true, preventfill:true}}); 384 | // save element 385 | s.set('RestoreDontSurvive', 'data', {ttl:-1}, function (err) { 386 | assert(err === null); 387 | s.set('RestoreTest','test', function (err) 388 | { 389 | var t=store.create({options: {path:cacheDirectory, zip: true, fillcallback: function () { 390 | //fill complete 391 | t.get('RestoreTest', function (err, data) { 392 | assert(data === 'test'); 393 | t.get('RestoreDontSurvive', function (err,data) { 394 | assert(err === null); 395 | assert(data === null); 396 | assert(s.currentsize > 0, 'current size not correctly initialized - '+s.currentsize); 397 | done(); 398 | }); 399 | }); 400 | } 401 | }}); 402 | }); 403 | }); 404 | }); 405 | 406 | it('max size option', function (done) { 407 | 408 | // create store 409 | var s = store.create({ 410 | options: { 411 | path: cacheDirectory, 412 | preventfill: true, 413 | maxsize: 1 414 | } 415 | }); 416 | 417 | s.set('one', 'dataone', {}, function (err, val) { 418 | assert(err.message === 'Item size too big.'); 419 | assert(Object.keys(s.collection).length === 0); 420 | 421 | s.set('x', 'x', { ttl: -1 }, function (err, val) { 422 | assert(err.message === 'Item size too big.'); 423 | assert(Object.keys(s.collection).length === 0); 424 | 425 | s.options.maxsize = 150; 426 | s.set('a', 'a', { ttl: 10000 }, function (err, val) { 427 | assert(err === null); 428 | assert(Object.keys(s.collection).length === 1); 429 | 430 | s.set('b', 'b', { ttl: 100 }, function (err){ 431 | assert(err === null); 432 | 433 | s.set('c', 'c', { ttl: 100 }, function (err){ 434 | assert(err === null); 435 | 436 | // now b should be removed from the cache, a should exists 437 | s.get('a', function (err, data) { 438 | assert(err === null); 439 | assert(data,'a'); 440 | 441 | s.get('b', function (err,data){ 442 | assert(err === null); 443 | assert(data === null); 444 | done(); 445 | }); 446 | }); 447 | }); 448 | }); 449 | }); 450 | }); 451 | }); 452 | }); 453 | }); 454 | }); 455 | --------------------------------------------------------------------------------