├── .jshintignore ├── .coveralls.yml ├── .npmignore ├── .gitignore ├── .travis.yml ├── index.js ├── CONTRIBUTING.md ├── tests ├── mocks │ ├── generateItems.js │ └── StorageMock.js └── unit │ └── StorageLRU-test.js ├── package.json ├── src ├── asyncify.js └── StorageLRU.js ├── LICENSE.md └── README.md /.jshintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.coveralls.yml: -------------------------------------------------------------------------------- 1 | service_name: travis-ci 2 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | /artifacts/ 2 | /examples/ 3 | /tests/ 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /artifacts/ 2 | node_modules 3 | npm-debug.log 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.11" 4 | - "0.10" 5 | after_success: 6 | - "npm run cover" 7 | - "cat artifacts/lcov.info | ./node_modules/coveralls/bin/coveralls.js" 8 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014, Yahoo! Inc. 3 | * Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms. 4 | */ 5 | module.exports = { 6 | StorageLRU: require('./src/StorageLRU'), 7 | asyncify: require('./src/asyncify') 8 | } 9 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing Code to `storage-lru` 2 | ------------------------------- 3 | 4 | Please be sure to sign our [CLA][] before you submit pull requests or otherwise contribute to `storage-lru`. This protects developers, who rely on [BSD license][]. 5 | 6 | [BSD license]: https://github.com/yahoo/storage-lru/blob/master/LICENSE.md 7 | [CLA]: https://yahoocla.herokuapp.com/ 8 | -------------------------------------------------------------------------------- /tests/mocks/generateItems.js: -------------------------------------------------------------------------------- 1 | function nowInSec() { 2 | return Math.floor(new Date().getTime() / 1000); 3 | } 4 | 5 | module.exports = function (keyPrefix, records) { 6 | var items = {}; 7 | var now = nowInSec(); 8 | for (var i = 0, len = records.length; i < len; i++) { 9 | var record = records[i]; 10 | var key = record.key || i; 11 | if (record.bad) { 12 | items[keyPrefix + key] = 'noMetaJunk' + record.value; 13 | } else if (record.key === 'empty') { 14 | items[keyPrefix + key] = ''; 15 | } else { 16 | var metaFields = [ 17 | '1', 18 | now + record.accessDelta, 19 | now + record.expiresDelta, 20 | record.maxAge || 600, 21 | record.stale, 22 | record.priority || 3 23 | ]; 24 | items[keyPrefix + key] = '[' + metaFields.join(':') + ']' + record.value; 25 | } 26 | } 27 | return items; 28 | }; 29 | -------------------------------------------------------------------------------- /tests/mocks/StorageMock.js: -------------------------------------------------------------------------------- 1 | function StorageMock(mockData) { 2 | if (mockData) { 3 | this.data = JSON.parse(JSON.stringify(mockData)); 4 | } else { 5 | this.data = {}; 6 | } 7 | // this.data = mockData || {}; 8 | this.length = Object.keys(this.data).length; 9 | } 10 | 11 | StorageMock.prototype.getItem = function (key) { 12 | if (this.data.hasOwnProperty(key)) { 13 | return this.data[key]; 14 | } 15 | return null; 16 | }; 17 | 18 | StorageMock.prototype.setItem = function (key, value) { 19 | if (key.indexOf('throw_max_quota_error') >= 0) { 20 | throw 'max quota error'; 21 | } 22 | this.data[key] = value; 23 | this.length = Object.keys(this.data).length; 24 | }; 25 | 26 | StorageMock.prototype.removeItem = function (key, value) { 27 | delete this.data[key]; 28 | this.length = Object.keys(this.data).length; 29 | }; 30 | 31 | StorageMock.prototype.key = function (index) { 32 | var keys = Object.keys(this.data); 33 | return (keys && keys[index]) || null; 34 | }; 35 | 36 | module.exports = StorageMock; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "storage-lru", 3 | "version": "0.1.1", 4 | "description": "An LRU implementation that can be used with local storage, or other storage mechanisms that support a similar interface.", 5 | "main": "index.js", 6 | "repository": { 7 | "type": "git", 8 | "url": "git://github.com/yahoo/storage-lru.git" 9 | }, 10 | "scripts": { 11 | "cover": "node node_modules/istanbul/lib/cli.js cover --dir artifacts -- ./node_modules/mocha/bin/_mocha tests/unit/ --recursive --reporter spec", 12 | "lint": "jshint lib tests", 13 | "test": "mocha tests/unit/ --recursive --reporter spec" 14 | }, 15 | "author": "Lingyan Zhu syncObject.length) ? syncObject.length : num; 31 | for (var i = 0, len = limit; i < len; i++) { 32 | arr.push(syncObject.key(i)); 33 | } 34 | callback(null, arr); 35 | }; 36 | } 37 | return retval; 38 | } 39 | 40 | module.exports = asyncify; -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Software License Agreement (BSD License) 2 | ======================================== 3 | 4 | Copyright (c) 2014, Yahoo! Inc. All rights reserved. 5 | ---------------------------------------------------- 6 | 7 | Redistribution and use of this software in source and binary forms, with or 8 | without modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright notice, this 12 | list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above copyright notice, 14 | this list of conditions and the following disclaimer in the documentation 15 | and/or other materials provided with the distribution. 16 | * Neither the name of Yahoo! Inc. nor the names of YUI's contributors may be 17 | used to endorse or promote products derived from this software without 18 | specific prior written permission of Yahoo! Inc. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 21 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 22 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 24 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 25 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 26 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 27 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 29 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # StorageLRU [![Build Status](https://travis-ci.org/yahoo/storage-lru.svg?branch=master)](https://travis-ci.org/yahoo/storage-lru) [![Dependency Status](https://david-dm.org/yahoo/storage-lru.svg)](https://david-dm.org/yahoo/storage-lru) [![Coverage Status](https://coveralls.io/repos/yahoo/storage-lru/badge.png?branch=master)](https://coveralls.io/r/yahoo/storage-lru?branch=master) 2 | 3 | StorageLRU is a LRU implementation that can be used with local storage, or other storage mechanisms that support a similar interface. 4 | 5 | **Note:** This library is written in CommonJS style. To use it in browser, please use tools like [Browserify](http://browserify.org/) and [Webpack](http://webpack.github.io/). 6 | 7 | ## Features 8 | 9 | ### Pluggable Underline Storage 10 | You can use your own storage of choice with StorageLRU, as long as it conforms to an asyncronous API. 11 | 12 | For syncronous storage solutions (such as html5 localStorage), we also provide the `asyncify` util, which simply wraps your syncronous storage object inside an asyncronous interface. 13 | 14 | Following is the async API details: 15 | ```js 16 | getItem : function (key, callback) - get the item with the associated key; 17 | setItem : function (key, item, callback) - set an item to the passed in key; 18 | removeItem : function (key, callback) - remove the item with the associated key; 19 | keys : function (num, callback) - get `num` number of keys of items stored; 20 | ``` 21 | Note that your storage should return 'null' or 'undefined' when a requested key does not exist. 22 | 23 | Examples: 24 | 25 | ```js 26 | // Example 1: async storage 27 | var StorageLRU = require('storage-lru').StorageLRU; 28 | // var myAsyncStorage = ...; 29 | var lru = new StorageLRU(myAsyncStorage); 30 | 31 | // Example 2: localStorage 32 | var StorageLRU = require('storage-lru').StorageLRU; 33 | var asyncify = require('storage-lru').asyncify; 34 | var lru = new StorageLRU(asyncify(localStorage)); 35 | ``` 36 | 37 | ### Max-Age and Stale-While-Revalidate 38 | When you save an item to the StorageLRU, you are required to specify a cache control string with HTTP Cache-Control header syntax, in which `max-age` is required and `stale-while-revalidate` is optional. 39 | 40 | The `max-age` defines when the item will expire. The `stale-while-revalidate` defines a time window after expiration, in which the item is marked as stale but still usable. If the time has passed this time window as well, this item will not be fetchable, and be purged. 41 | 42 | If `getItem()` is called on an item when it is in the `stale-while-revalidate` time window, StorageLRU will try to refresh the data during this time window, assuming a `revalidateFn` was passed when the StorageLRU instance was created. The `revalidateFn` function will be used to fetch the stale item. If a fresh value is fetched successfully, StorageLRU will save the new value to the underline storage. 43 | 44 | The revalidate success/failure count will be recorded in [the Stats](#stats). 45 | 46 | Example: 47 | ```js 48 | var StorageLRU = require('storage-lru').StorageLRU; 49 | var asyncify = require('storage-lru').asyncify; 50 | // Creating a StorageLRU instance with an item revalidation function 51 | var lru = new StorageLRU( 52 | asyncify(localStorage), 53 | { 54 | revalidateFn: function(key, callback) { 55 | var newValue = someFunctionToRefetchFromSomewhere(key); // most likely be async 56 | callback(null, newValue); // make sure callback is invoked 57 | } 58 | }); 59 | // Saving item 'fooJSON', which expires in 5 minutes and has a stale-while-revalidate time window of 1 day after expiration. 60 | lru.setItem( 61 | 'fooJSON', // key 62 | { // value 63 | foo: 'bar' 64 | }, 65 | { // options 66 | json: true, 67 | cacheControl:'max-age=300,stale-while-revalidate=86400' 68 | }, function (err) { 69 | if (err) { 70 | // something went wrong. Item not saved. 71 | console.log('Failed to save item: err=', err); 72 | return; 73 | } 74 | } 75 | ); 76 | ``` 77 | 78 | ### Priority 79 | When you save an item to StorageLRU, you can assign a priority. Lower priority items get purged first, if all other conditions are the same. 80 | 81 | | Priority | Description | 82 | |----------|--------------------------| 83 | | 1 | Critical - Last to purge | 84 | | 2 | Important | 85 | | 3 | Normal | 86 | | 4 | Low - First to purge | 87 | 88 | Example: 89 | ```js 90 | var StorageLRU = require('storage-lru').StorageLRU; 91 | var asyncify = require('storage-lru').asyncify; 92 | var lru = new StorageLRU(asyncify(localStorage)); 93 | lru.setItem('fooJSON', {foo: 'bar'}, {json: true, priority: 1}, function (err) { 94 | if (err) { 95 | // something went wrong. Item not saved. 96 | console.log('Failed to save item: err=', err); 97 | } 98 | }); 99 | ``` 100 | 101 | 102 | ### Automatic Purging 103 | When the storage becomes full, StorageLRU will purge the existing items to make enough space. The default purging precendence order is as following: 104 | 105 | * bad entry (invalid meta info), 106 | * truly stale (passed stale-while-revaliate window), 107 | * lowest priority, 108 | * least recently accessed, 109 | * bigger byte size 110 | 111 | Basically, the bad items will be purged first; next will be the items that have expired and passed stale-while-revaliate window; then the lowest priority items; then the least recently accessed items; if there happen to the two items with the same access time, the one takes more space will be purged first. 112 | 113 | ### Customizable PurgeComparator 114 | You can replace the default purging algorithm with your own, by specifying a purgeComparator function when creating the StorageLRU instance. 115 | 116 | ```js 117 | var StorageLRU = require('storage-lru').StorageLRU; 118 | var asyncify = require('storage-lru').asyncify; 119 | var lru = new StorageLRU(asyncify(localStorage), { 120 | // always purge the largest item first 121 | purgeComparator: function (meta1, meta2) { 122 | if (meta1.size > meta2.size) { 123 | return 1; 124 | } else if (meta1.size === meta2.size){ 125 | return 0; 126 | } else { 127 | return -1; 128 | } 129 | } 130 | }); 131 | ``` 132 | 133 | ### Configurable Purge Factor 134 | You can configure how much extra space to purge, by providing a `purgeFactor` param when instantiating the StorageLRU instance. It should be a positive float number. 135 | 136 | ```js 137 | var StorageLRU = require('storage-lru').StorageLRU; 138 | var asyncify = require('storage-lru').asyncify; 139 | var lru = new StorageLRU(asyncify(localStorage), { 140 | // purgeFactor controls amount of extra space to purge. 141 | // E.g. if space needed for a new item is 1000 characters, StorageLRU will actually 142 | // try to purge (1000 + 1000 * purgeFactor) characters. 143 | purgeFactor: 0.5 144 | }); 145 | ``` 146 | 147 | ### Configurable Purge Attempts 148 | In addition to how much extra space to purge, you can also configure how many items to retrieve from underlying storage in the event that purge is unable to find enough free space. By providing a `maxPurgeAttempts` param you can set how many times purge will attempt to free up space. Each attempt will increase the number of keys looked up by the `purgeLoadIncrease` param. 149 | 150 | ```js 151 | var StorageLRU = require('storage-lru').StorageLRU; 152 | var asyncify = require('storage-lru').asyncify; 153 | var lru = new StorageLRU(asyncify(localStorage), { 154 | // maxPurgeAttempts controls the number of times to try purging, 155 | // each attempt will look through more items. 156 | maxPurgeAttempts: 3, 157 | // purgeLoadIncrease controls the number of additional keys to look up during each 158 | // successive purge attempt. 159 | // E.g. if this is the second additional purge attempt, StorageLRU will attempt to load 160 | // (2 * 500) keys. 161 | purgeLoadIncrease: 500 162 | }); 163 | ``` 164 | 165 | ### Purge Notification 166 | If you want to be notified when items get purged from the storage, you can register a callback function when creating the StorageLRU instance. 167 | 168 | ```js 169 | var StorageLRU = require('storage-lru').StorageLRU; 170 | var asyncify = require('storage-lru').asyncify; 171 | var lru = new StorageLRU(asyncify(localStorage), { 172 | // purgeFactor controls amount of extra space to purge. 173 | // E.g. if space needed for a new item is 1000 characters, StorageLRU will actually 174 | // try to purge (1000 + 1000 * purgeFactor) characters. 175 | purgeFactor: 0.5, 176 | purgedFn: function (purgedKeys) { 177 | console.log('These keys were purged:', purgedKeys); 178 | } 179 | }); 180 | ``` 181 | 182 | 183 | 184 | ### Stats 185 | 186 | StorageLRU collects statistics data for you to tune the LRU to work efficiently with the specific characteristics of your app data. For example, you can customize `purgeFactor` to be a bigger number if your app saves several items in a short time interval. 187 | 188 | Currently stats data collected include the following: 189 | 190 | | Name | Description | 191 | |-------|-------------------------------------------------------------------------------------------------------------------------------------| 192 | | hit | Number of cache hits | 193 | | miss | Number of cache misses | 194 | | stale | Number of times where stale items were returned (cache hit with data that expired but still within stale-while-revalidate window) | 195 | | error | Number of errors occurred during getItem | 196 | | revalidateSuccess | Success count for revalidating a stale item, if `revalidateFn` is provided when the StorageLRU instance is instantiated. | 197 | | revalidateFailure | Failure count for revalidating a stale item, if `revalidateFn` is provided when the StorageLRU instance is instantiated. | 198 | 199 | Example: 200 | ```js 201 | var stats = lru.stats(); 202 | ``` 203 | 204 | 205 | ## Usage 206 | 207 | ```js 208 | var StorageLRU = require('storage-lru').StorageLRU; 209 | var asyncify = require('storage-lru').asyncify; 210 | 211 | var lru = new StorageLRU(asyncify(localStorage), { 212 | purgeFactor: 0.5, // this controls amount of extra space to purge. 213 | purgedFn: function (purgedKeys) { 214 | console.log('These keys were purged:', purgedKeys); 215 | } 216 | }); 217 | console.log(lru.numItems()); // output 0, assuming the storage is clear 218 | 219 | lru.setItem('foo', 'bar', {}, function (err) { 220 | if (err) { 221 | // something went wrong. Item not saved. 222 | console.log('Failed to save item: err=', err); 223 | } 224 | }); 225 | 226 | lru.setItem('fooJSON', {foo: 'bar'}, {json: true}, function (err) { 227 | if (err) { 228 | // something went wrong. Item not saved. 229 | console.log('Failed to save item: err=', err); 230 | } 231 | }); 232 | 233 | lru.getItem('foo', {json: false}, function (err, value) { 234 | if (err) { 235 | // something went wrong, for example, can't deserialize 236 | console.log('Failed to fetch item: err=', err); 237 | return; 238 | } 239 | console.log('The value of "foo" is: ', value); 240 | }); 241 | 242 | lru.removeItem('foo', function (err) { 243 | if (err) { 244 | // something went wrong. Item not removed. 245 | } 246 | }); 247 | 248 | var stats = lru.stats(); 249 | ``` 250 | 251 | ## Error Codes 252 | 253 | | Code | Message | Description | Sources | 254 | |------|--------------------|---------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------| 255 | | 1 | disabled | The underline storage (storage instance passed to StorageLRU) is disabled. | StorageLRU.setItem() | 256 | | 2 | cannot deserialize | Not able to deserialize the stored value. | | 257 | | 3 | cannot serialize | Not able to serialize the value for storage. | StorageLRU.setItem() | 258 | | 4 | bad cacheControl | Invalid cache control string was passed to StorageLRU.setItem(). For example, containing no-store, no-cache, negative max-age. | StorageLRU.setItem() | 259 | | 5 | invalid key | Invalid key was provided, e.g. empty string | StorageLRU.setItem(), StorageLRU.getItem(), StorageLRU.removeItem() | 260 | | 6 | not enough space | The underline storage does not have enough space for the item being saved, even after purging old items. | StorageLRU.setItem() | 261 | | 7 | revalidate failed | Revalidating a stale item failed. (Internal error, not exposed via public API.) | StorageLRU._revalidate() | 262 | 263 | 264 | ## Polyfills 265 | 266 | This library requires the following Polyfill: 267 | 268 | * JSON - See [Modernizr Polyfill Doc](https://github.com/Modernizr/Modernizr/wiki/HTML5-Cross-browser-Polyfills#ecmascript-5) for available JSON polyfills. 269 | * Array.prototype.filter - See [Modernizer Polyfill Doc](https://github.com/Modernizr/Modernizr/wiki/HTML5-Cross-browser-Polyfills#ecmascript-5) for available `Array.prototype` polyfills. 270 | 271 | 272 | ## License 273 | This software is free to use under the Yahoo! Inc. BSD license. 274 | See the [LICENSE file][] for license text and copyright information. 275 | 276 | [LICENSE file]: https://github.com/yahoo/storage-lru/blob/master/LICENSE.md 277 | 278 | Third-pary open source code used are listed in our [package.json file]( https://github.com/yahoo/storage-lru/blob/master/package.json). 279 | -------------------------------------------------------------------------------- /tests/unit/StorageLRU-test.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014, Yahoo! Inc. 3 | * Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms. 4 | */ 5 | /*globals describe,it,beforeEach */ 6 | "use strict"; 7 | 8 | var expect = require('chai').expect, 9 | StorageLRU = require('../../src/StorageLRU'), 10 | asyncify = require('../../src/asyncify'), 11 | StorageMock =require('../mocks/StorageMock'), 12 | generateItems = require('../mocks/generateItems'); 13 | 14 | function findMetaRecord(records, key) { 15 | var record; 16 | for (var i = 0, len = records.length; i < len; i++) { 17 | if (records[i].key === key) { 18 | record = records[i]; 19 | } 20 | } 21 | return record; 22 | } 23 | 24 | describe('StorageLRU', function () { 25 | var storage; 26 | 27 | beforeEach(function () { 28 | var mockData = generateItems('TEST_', [ 29 | { 30 | key: 'fresh-lastAccessed', 31 | expiresDelta: 60, 32 | stale: 0, 33 | accessDelta: -30, 34 | value: 'expires in 1min, stale=0, last accessed 30secs ago' 35 | }, 36 | { 37 | key: 'fresh', 38 | expiresDelta: 60, 39 | stale: 0, 40 | accessDelta: -300, 41 | value: 'expires in 1min, stale=0, last accessed 5mins ago' 42 | }, 43 | { 44 | key: 'fresh-lastAccessed-biggerrecord', 45 | expiresDelta: 60, 46 | stale: 0, 47 | accessDelta: -30, 48 | value: 'expires in 1min, stale=0, last accessed 30secs ago, blahblahblah' 49 | }, 50 | { 51 | key: 'stale-lowpriority', 52 | expiresDelta: -60, 53 | stale: 300, 54 | accessDelta: -600, 55 | priority: 5, 56 | value: 'expired 1min ago, stale=5, last accessed 10mins ago, priority=5' 57 | }, 58 | { 59 | key: 'stale', 60 | expiresDelta: -60, 61 | stale: 300, 62 | accessDelta: -600, 63 | value: 'expired 1min ago, stale=5, last accessed 10mins ago' 64 | }, 65 | { 66 | key: 'trulyStale', 67 | expiresDelta: -60, 68 | stale: 0, 69 | accessDelta: -30, 70 | value: 'expired 1min ago, stale=0, last accessed 30secs ago' 71 | }, 72 | { 73 | key: 'bad', 74 | bad: true, 75 | value: 'invalid format' 76 | }, 77 | { 78 | key: 'empty', 79 | value: '' 80 | } 81 | ]); 82 | storage = asyncify(new StorageMock(mockData)); 83 | }); 84 | 85 | it('constructor', function () { 86 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 87 | expect(lru._storage === storage).to.equal(true, '_storage assigned'); 88 | expect(lru.options.recheckDelay).to.equal(-1, 'options.recheckDelay'); 89 | expect(lru.options.keyPrefix).to.equal('TEST_', 'options.keyPrefix'); 90 | expect(lru._purgeComparator).to.be.a('function', '_purgeComparator assigned'); 91 | }); 92 | 93 | it('stats', function () { 94 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 95 | var stats = lru.stats(); 96 | expect(stats).to.eql({hit: 0, miss: 0, stale: 0, error: 0, revalidateSuccess: 0, revalidateFailure: 0}, 'stats inited'); 97 | stats = lru.stats(); 98 | expect(stats.hit).to.eql(0, 'stats.hit'); 99 | expect(stats.miss).to.eql(0, 'stats.miss'); 100 | expect(stats.stale).to.eql(0, 'stats.stale'); 101 | expect(stats.error).to.eql(0, 'stats.error'); 102 | expect(stats.error).to.eql(0, 'stats.revalidateSuccess'); 103 | expect(stats.error).to.eql(0, 'stats.revalidateFailure'); 104 | }); 105 | 106 | it('get keys', function (done) { 107 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 108 | lru.keys(10, function (err, keys) { 109 | expect(keys[0]).to.equal('TEST_fresh-lastAccessed', 'first key'); 110 | expect(keys[1]).to.equal('TEST_fresh', 'second key'); 111 | done(); 112 | }); 113 | }); 114 | 115 | it('_parseCacheControl', function () { 116 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 117 | 118 | var cc = lru._parseCacheControl('max-age=300,stale-while-revalidate=60'); 119 | expect(cc['max-age']).to.equal(300); 120 | expect(cc['stale-while-revalidate']).to.equal(60); 121 | cc = lru._parseCacheControl('no-cache,no-store'); 122 | expect(cc['no-cache']).to.equal(true); 123 | expect(cc['no-store']).to.equal(true); 124 | cc = lru._parseCacheControl(''); 125 | expect(cc).to.eql({}); 126 | }); 127 | 128 | describe('#getItem', function () { 129 | it('invalid key', function (done) { 130 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 131 | lru.getItem('', {}, function (err, value) { 132 | expect(err.code).to.equal(5, 'expect "invalid key" error'); 133 | done(); 134 | }); 135 | }); 136 | it('cache miss - key does not exist', function (done) { 137 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 138 | lru.getItem('key_does_not_exist', {}, function(err, value) { 139 | expect(lru.stats()).to.include({hit: 0, miss: 1, stale: 0, error: 0}, 'cache miss'); 140 | done(); 141 | }); 142 | }); 143 | it('cache miss - truly stale', function (done) { 144 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 145 | lru._meta.init(8, function initDone() { 146 | var size = lru._meta.numRecords(); 147 | lru.getItem('trulyStale', {}, function(err, value) { 148 | expect(!err).to.equal(true, 'no error'); 149 | expect(!value).to.equal(true, 'no value'); 150 | expect(lru.stats()).to.include({hit: 0, miss: 1, stale: 0, error: 0}, 'cache miss - truly stale'); 151 | expect(lru._meta.numRecords()).to.equal(size - 1, 'truly stale item removed'); 152 | done(); 153 | }); 154 | }); 155 | }); 156 | it('cache hit - meta not yet built', function (done) { 157 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 158 | expect(lru._meta.records.length).to.equal(0); 159 | storage.getItem('TEST_fresh', function(err, value) { 160 | var oldMeta = lru._deserialize(value, {}).meta; 161 | lru.getItem('fresh', {json: false}, function(err, value, meta) { 162 | expect(lru._meta.records.length).to.equal(1); 163 | expect(err).to.equal(null); 164 | expect(value).to.equal('expires in 1min, stale=0, last accessed 5mins ago'); 165 | expect(meta.isStale).to.equal(false); 166 | expect(lru.stats()).to.include({hit: 1, miss: 0, stale: 0, error: 0}, 'cache hit'); 167 | // make sure access timestamp is updated 168 | storage.getItem('TEST_fresh', function(err, item) { 169 | var newMeta = lru._deserialize(item, {}).meta; 170 | expect(newMeta.access > oldMeta.access).to.equal(true, 'access ts updated'); 171 | expect(newMeta.expires).to.equal(oldMeta.expires, 'expires not changed'); 172 | expect(newMeta.stale).to.equal(oldMeta.stale, 'stale not changed'); 173 | expect(newMeta.priority).to.equal(oldMeta.priority, 'priority not changed'); 174 | done(); 175 | }); 176 | }); 177 | }); 178 | }); 179 | it('cache hit - fresh', function (done) { 180 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 181 | storage.getItem('TEST_fresh', function(err, value) { 182 | var oldMeta = lru._deserialize(value, {}).meta; 183 | lru.getItem('fresh', {json: false}, function(err, value, meta) { 184 | expect(err).to.equal(null); 185 | expect(value).to.equal('expires in 1min, stale=0, last accessed 5mins ago'); 186 | expect(meta.isStale).to.equal(false); 187 | expect(lru.stats()).to.include({hit: 1, miss: 0, stale: 0, error: 0}, 'cache hit'); 188 | // make sure access timestamp is updated 189 | storage.getItem('TEST_fresh', function(err, item) { 190 | var newMeta = lru._deserialize(item, {}).meta; 191 | expect(newMeta.access > oldMeta.access).to.equal(true, 'access ts updated'); 192 | expect(newMeta.expires).to.equal(oldMeta.expires, 'expires not changed'); 193 | expect(newMeta.stale).to.equal(oldMeta.stale, 'stale not changed'); 194 | expect(newMeta.priority).to.equal(oldMeta.priority, 'priority not changed'); 195 | done(); 196 | }); 197 | }); 198 | }); 199 | }); 200 | it('cache hit - stale', function (done) { 201 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 202 | lru.getItem('stale', {json: false}, function(err, value, meta) { 203 | expect(err).to.equal(null); 204 | expect(meta.isStale).to.equal(true); 205 | expect(value).to.equal('expired 1min ago, stale=5, last accessed 10mins ago'); 206 | expect(lru.stats()).to.include({hit: 1, miss: 0, stale: 1, error: 0}, 'cache hit - stale'); 207 | done(); 208 | }); 209 | }); 210 | it('cache hit - stale - revalidate success', function (done) { 211 | var lru = new StorageLRU(storage, { 212 | keyPrefix: 'TEST_', 213 | revalidateFn: function (key, callback) { 214 | callback(null, 'revalidated value'); 215 | } 216 | }); 217 | lru._meta.init(10, function initDone() { 218 | var size = lru._meta.numRecords(); 219 | var record = findMetaRecord(lru._meta.records, 'TEST_stale'); 220 | expect(record.key).to.equal('TEST_stale'); 221 | expect(record.size).to.equal(86); 222 | expect(record.stale).to.equal(300); 223 | 224 | lru.getItem('stale', {json: false}, function(err, value, meta) { 225 | expect(!err).to.equal(true, 'no error, but getting: ' + (err && err.message)); 226 | expect(meta.isStale).to.equal(true); 227 | expect(value).to.equal('expired 1min ago, stale=5, last accessed 10mins ago'); 228 | expect(lru.stats()).to.include({hit: 1, miss: 0, stale: 1, error: 0, revalidateSuccess:1, revalidateFailure: 0}, 'cache hit,stale,revalidateSuccess'); 229 | 230 | var updatedRecord = findMetaRecord(lru._meta.records, 'TEST_stale'); 231 | expect(updatedRecord.key).to.equal(record.key, 'key remains the same'); 232 | expect(updatedRecord.size).to.equal(52, 'size is updated'); 233 | expect(updatedRecord.access).to.be.above(record.access, 'access timestamp is updated'); 234 | expect(updatedRecord.expires).to.be.above(record.expires, 'expires timestamp is extended'); 235 | expect(updatedRecord.maxAge).to.equal(record.maxAge, 'maxAge remains the same'); 236 | expect(updatedRecord.stale).to.equal(record.stale, 'stale window size remains the same'); 237 | expect(updatedRecord.priority).to.equal(record.priority, 'priority remains the same'); 238 | done(); 239 | }); 240 | }); 241 | }); 242 | it('cache hit - stale - revalidate failure', function (done) { 243 | var lru = new StorageLRU(storage, { 244 | keyPrefix: 'TEST_', 245 | revalidateFn: function (key, callback) { 246 | callback('not able to revalidate "' + key + '"'); 247 | } 248 | }); 249 | lru._meta.init(10, function initDone() { 250 | var size = lru._meta.numRecords(); 251 | var record = findMetaRecord(lru._meta.records, 'TEST_stale'); 252 | expect(record.key).to.equal('TEST_stale'); 253 | expect(record.size).to.equal(86); 254 | expect(record.stale).to.equal(300); 255 | 256 | lru.getItem('stale', {json: false}, function(err, value, meta) { 257 | expect(!err).to.equal(true, 'no error, but getting: ' + (err && err.message)); 258 | expect(meta.isStale).to.equal(true); 259 | expect(value).to.equal('expired 1min ago, stale=5, last accessed 10mins ago'); 260 | expect(lru.stats()).to.include({hit: 1, miss: 0, stale: 1, error: 0, revalidateSuccess:0, revalidateFailure: 1}, 'cache hit,stale,revalidateFailure'); 261 | 262 | var updatedRecord = findMetaRecord(lru._meta.records, 'TEST_stale'); 263 | expect(updatedRecord.key).to.equal(record.key, 'key remains the same'); 264 | expect(updatedRecord.size).to.equal(record.size, 'size remains the same'); 265 | expect(updatedRecord.access).to.be.above(record.access, 'access timestamp is updated'); 266 | expect(updatedRecord.expires).to.equal(record.expires, 'expires timestamp remains the same'); 267 | expect(updatedRecord.maxAge).to.equal(record.maxAge, 'maxAge remains the same'); 268 | expect(updatedRecord.stale).to.equal(record.stale, 'stale window size remains the same'); 269 | expect(updatedRecord.priority).to.equal(record.priority, 'priority remains the same'); 270 | done(); 271 | }); 272 | }); 273 | }); 274 | it('bad item', function (done) { 275 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 276 | lru.getItem('bad', {json: false}, function(err, value, meta) { 277 | expect(err.code).to.equal(2, 'expect "cannot deserialize" error'); 278 | expect(lru.stats()).to.include({hit: 0, miss: 0, stale: 0, error: 1}, 'cache hit - stale'); 279 | done(); 280 | }); 281 | }); 282 | it('empty item', function (done) { 283 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 284 | lru.getItem('empty', {}, function(err, value) { 285 | expect(err.code).to.equal(2, 'expect deserialize error'); 286 | expect(lru.stats()).to.include({hit: 0, miss: 0, stale: 0, error: 1}, 'cache miss'); 287 | done(); 288 | }); 289 | }); 290 | }); 291 | 292 | describe('#setItem', function () { 293 | it('invalid key', function (done) { 294 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 295 | var size = lru._meta.numRecords(); 296 | lru.setItem('', {foo: 'bar'}, {json: true, cacheControl: 'max-age=300'}, function (err, value) { 297 | expect(err.code).to.equal(5, 'expect "invalid key" error'); 298 | expect(lru._meta.numRecords()).to.equal(size, 'numItems remains the same'); 299 | done(); 300 | }); 301 | }); 302 | it('new item, json=true', function (done) { 303 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 304 | var size = lru._meta.numRecords(); 305 | lru.setItem('new_item', {foo: 'bar'}, {json: true, cacheControl: 'max-age=300'}, function (err) { 306 | var num = lru._meta.numRecords(); 307 | expect(num).to.equal(size + 1, 'numItems should increase by 1'); 308 | var record = findMetaRecord(lru._meta.records, 'TEST_new_item'); 309 | expect(record.key).to.equal('TEST_new_item'); 310 | expect(record.size).to.equal(46); 311 | expect(record.stale).to.equal(0); 312 | lru.getItem('new_item', {}, function (err, value, meta) { 313 | expect(value).to.equal('{"foo":"bar"}'); 314 | expect(meta.isStale).to.equal(false); 315 | done(); 316 | }); 317 | }); 318 | }); 319 | it('new item, json=false', function (done) { 320 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 321 | var size = lru._meta.numRecords(); 322 | lru.setItem('new_item', 'foobar', {json: false, cacheControl: 'max-age=300'}, function (err) { 323 | var num = lru._meta.numRecords(); 324 | expect(num).to.equal(size + 1); 325 | lru.getItem('new_item', {json: false}, function (err, value, meta) { 326 | expect(value).to.equal('foobar'); 327 | expect(meta.isStale).to.equal(false); 328 | done(); 329 | }); 330 | }); 331 | }); 332 | it('new item, json default is false', function (done) { 333 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 334 | var size = lru._meta.numRecords(); 335 | lru.setItem('new_item', '{foo:"bar"}', {cacheControl: 'max-age=300'}, function (err) { 336 | var num = lru._meta.numRecords(); 337 | expect(num).to.equal(size + 1); 338 | lru.getItem('new_item', {}, function (err, value, meta) { 339 | expect(value).to.equal('{foo:"bar"}'); 340 | expect(meta.isStale).to.equal(false); 341 | done(); 342 | }); 343 | }); 344 | }); 345 | it('existing item, json=false', function (done) { 346 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 347 | lru._meta.init(10, function initDone() { 348 | var numItems = lru._meta.numRecords(); 349 | var record = findMetaRecord(lru._meta.records, 'TEST_fresh'); 350 | var access = record.access; 351 | var size = record.size; 352 | lru.setItem('fresh', 'foobar', {json: false, cacheControl: 'max-age=300'}, function (err) { 353 | var num = lru._meta.numRecords(); 354 | expect(num).to.equal(numItems, 'numItems is correct'); 355 | var updatedRecord = findMetaRecord(lru._meta.records, 'TEST_fresh'); 356 | expect(updatedRecord.access > access).to.equal(true, 'access timestamp updated'); 357 | expect(updatedRecord.size < size).to.equal(true, 'size timestamp updated'); 358 | lru.getItem('fresh', {json: false}, function (err, value, meta) { 359 | expect(value).to.equal('foobar'); 360 | expect(meta.isStale).to.equal(false); 361 | done(); 362 | }); 363 | }); 364 | }); 365 | }); 366 | it('disabled', function (done) { 367 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 368 | lru._enabled = false; 369 | lru.setItem('new_item', 'foobar', {json: false, cacheControl: 'max-age=300'}, function (err) { 370 | expect(err.code).to.equal(1); 371 | done(); 372 | }); 373 | }); 374 | it('no-cache', function (done) { 375 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 376 | lru.setItem('new_item', 'foobar', {json: false, cacheControl: 'no-cache'}, function (err) { 377 | expect(err.code).to.equal(4); 378 | done(); 379 | }); 380 | }); 381 | it('no-store', function (done) { 382 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 383 | lru.setItem('new_item', 'foobar', {json: false, cacheControl: 'no-store'}, function (err) { 384 | expect(err.code).to.equal(4); 385 | done(); 386 | }); 387 | }); 388 | it('invalid max-age', function (done) { 389 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 390 | lru.setItem('new_item', 'foobar', {json: false, cacheControl: 'max-age=-1'}, function (err) { 391 | expect(err.code).to.equal(4); 392 | done(); 393 | }); 394 | }); 395 | it('missing cacehControl', function (done) { 396 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 397 | lru.setItem('new_item', 'foobar', {json: false}, function (err) { 398 | expect(err.code).to.equal(4); 399 | done(); 400 | }); 401 | }); 402 | it('disable mode', function (done) { 403 | var emptyStorage = asyncify(new StorageMock()); 404 | var lru = new StorageLRU(emptyStorage, {keyPrefix: 'TEST_'}); 405 | lru.setItem('throw_max_quota_error', 'foobar', {json: false, cacheControl: 'max-age=300'}, function (err) { 406 | expect(err.code).to.equal(1); 407 | done(); 408 | }); 409 | }); 410 | it('disable mode - re-enable', function (done) { 411 | var emptyStorage = asyncify(new StorageMock()); 412 | var lru = new StorageLRU(emptyStorage, {keyPrefix: 'TEST_', recheckDelay: 10}); 413 | lru.setItem('throw_max_quota_error', 'foobar', {json: false, cacheControl: 'max-age=300'}, function (err, val) { 414 | expect(err.code).to.equal(1); 415 | setTimeout(function () { 416 | expect(lru._enabled).to.equal(true, 'renabled'); 417 | done(); 418 | }, 10); 419 | }); 420 | }); 421 | it('try purge', function (done) { 422 | var emptyStorage = asyncify(new StorageMock(generateItems('TEST_', [ 423 | { 424 | key: 'fresh', 425 | expiresDelta: 60, 426 | stale: 0, 427 | accessDelta: -300, 428 | value: 'foobar' 429 | } 430 | ]))); 431 | var lru = new StorageLRU(emptyStorage, {keyPrefix: 'TEST_'}); 432 | lru.setItem('throw_max_quota_error', 'foobarrrrrrr', {json: false, cacheControl: 'max-age=300'}, function (err) { 433 | expect(err.code).to.equal(6, 'expected "not enough space" error'); 434 | done(); 435 | }); 436 | }); 437 | }); 438 | 439 | describe('#purge', function () { 440 | it('all purged spacedNeeded=100000', function (done) { 441 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_', purgedFn: function (purged) { 442 | setTimeout(function () { 443 | expect(purged).to.eql(['bad', 'empty', 'trulyStale', 'stale-lowpriority', 'stale', 'fresh', 'fresh-lastAccessed-biggerrecord', 'fresh-lastAccessed']); 444 | done(); 445 | }, 1); 446 | }}); 447 | var size = lru._meta.numRecords(); 448 | lru.purge(10000, function (err) { 449 | expect(!!err).to.equal(true, 'not enough space'); 450 | expect(lru._meta.numRecords()).to.equal(0); 451 | }); 452 | }); 453 | it('1 purged spacedNeeded=3', function (done) { 454 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_', purgedFn: function (purged) { 455 | setTimeout(function () { 456 | expect(purged).to.eql(['bad']); 457 | }, 1); 458 | }}); 459 | var size = lru._meta.numRecords(); 460 | lru.purge(3, function (err) { 461 | expect(!err).to.eql(true); 462 | expect(lru._meta.numRecords()).to.equal(7); 463 | done(); 464 | }); 465 | }); 466 | it('3 purged spacedNeeded=50', function (done) { 467 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_', purgedFn: function (purged) { 468 | setTimeout(function () { 469 | expect(purged).to.eql(['bad', 'empty', 'trulyStale']); 470 | }, 1); 471 | }}); 472 | lru.purge(50, function (err) { 473 | expect(!err).to.eql(true); 474 | expect(lru._meta.numRecords()).to.equal(5); 475 | done(); 476 | }); 477 | }); 478 | }); 479 | 480 | describe('#_parser.format', function () { 481 | it('valid meta', function () { 482 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 483 | var parser = lru._parser; 484 | expect(parser.format).to.throw('invalid meta'); 485 | var value = parser.format({ 486 | access: 1000, 487 | expires: 1000, 488 | maxAge: 300, 489 | stale: 0, 490 | priority: 4 491 | }, 'aaa'); 492 | expect(value).to.equal('[1:1000:1000:300:0:4]aaa'); 493 | }); 494 | it('negative access', function () { 495 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 496 | var parser = lru._parser; 497 | try { 498 | parser.format({ 499 | access: -1, 500 | expires: 1000, 501 | maxAge: 300, 502 | stale: 1000 503 | }, 'aaa'); 504 | } catch (e) { 505 | expect(e.message).to.equal('invalid meta'); 506 | } 507 | }); 508 | it('negative stale', function () { 509 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 510 | var parser = lru._parser; 511 | try { 512 | parser.format({ 513 | access: 1000, 514 | expires: 1000, 515 | maxAge: 300, 516 | stale: -1 517 | }, 'aaa'); 518 | } catch (e) { 519 | expect(e.message).to.equal('invalid meta'); 520 | } 521 | }); 522 | it('negative expires', function () { 523 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 524 | var parser = lru._parser; 525 | try { 526 | parser.format({ 527 | access: 1000, 528 | expires: -1, 529 | maxAge: 300, 530 | stale: 0 531 | }, 'aaa'); 532 | } catch (e) { 533 | expect(e.message).to.equal('invalid meta'); 534 | } 535 | }); 536 | it('bad priority', function () { 537 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 538 | var parser = lru._parser; 539 | try { 540 | parser.format({ 541 | access: 1000, 542 | expires: 1000, 543 | maxAge: 300, 544 | stale: 0, 545 | priority: 0 546 | }, 'aaa'); 547 | } catch (e) { 548 | expect(e.message).to.equal('invalid meta'); 549 | } 550 | }); 551 | }); 552 | 553 | describe('#_parser.parse', function () { 554 | it('valid format', function () { 555 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 556 | var parser = lru._parser; 557 | expect(parser.parse).to.throw('missing meta'); 558 | var parsed = parser.parse('[1:2000:1000:300:0:1]aaa'); 559 | expect(parsed.meta.version).to.equal('1'); 560 | expect(parsed.meta.access).to.equal(2000); 561 | expect(parsed.meta.expires).to.equal(1000); 562 | expect(parsed.meta.stale).to.equal(0); 563 | expect(parsed.meta.priority).to.equal(1); 564 | expect(parsed.meta.size).to.equal(24); 565 | expect(parsed.value).to.equal('aaa'); 566 | }); 567 | it('negative access field', function () { 568 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 569 | var parser = lru._parser; 570 | try { 571 | parser.parse('[1:-2000:1000:300:0:1]aaa'); 572 | } catch(e) { 573 | expect(e.message).to.equal('invalid meta fields'); 574 | } 575 | }); 576 | }); 577 | 578 | describe('#removeItem', function () { 579 | it('valid key', function (done) { 580 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 581 | lru._meta.init(8, function initDone() { 582 | var size = lru._meta.numRecords(); 583 | lru.removeItem('fresh', function (err) { 584 | expect(!err).to.equal(true, 'expect no error'); 585 | expect(lru._meta.numRecords()).to.equal(size - 1, 'numItems should decrease by 1'); 586 | done(); 587 | }); 588 | }); 589 | }); 590 | it('invalid key', function (done) { 591 | var lru = new StorageLRU(storage, {keyPrefix: 'TEST_'}); 592 | lru._meta.init(8, function initDone() { 593 | var size = lru._meta.numRecords(); 594 | lru.removeItem('', function (err) { 595 | expect(err.code).to.equal(5, 'expect "invalid key" error'); 596 | expect(lru._meta.numRecords()).to.equal(size, 'numItems should not change'); 597 | done(); 598 | }); 599 | }); 600 | }); 601 | }); 602 | 603 | }); 604 | -------------------------------------------------------------------------------- /src/StorageLRU.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014, Yahoo! Inc. 3 | * Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms. 4 | */ 5 | 'use strict'; 6 | 7 | var ERR_DISABLED = {code: 1, message: 'disabled'}; 8 | var ERR_DESERIALIZE = {code: 2, message: 'cannot deserialize'}; 9 | var ERR_SERIALIZE = {code: 3, message: 'cannot serialize'}; 10 | var ERR_CACHECONTROL = {code: 4, message: 'bad cacheControl'}; 11 | var ERR_INVALIDKEY = {code: 5, message: 'invalid key'}; 12 | var ERR_NOTENOUGHSPACE = {code: 6, message: 'not enough space'}; 13 | var ERR_REVALIDATE = {code: 7, message: 'revalidate failed'}; 14 | // cache control fields 15 | var MAX_AGE = 'max-age'; 16 | var STALE_WHILE_REVALIDATE = 'stale-while-revalidate'; 17 | var DEFAULT_KEY_PREFIX = ''; 18 | var DEFAULT_PRIORITY = 3; 19 | var DEFAULT_PURGE_LOAD_INCREASE = 500; 20 | var DEFAULT_PURGE_ATTEMPTS = 2; 21 | var CUR_VERSION = '1'; 22 | 23 | var asyncEachSeries = require('async-each-series'); 24 | require('setimmediate'); 25 | 26 | 27 | function isDefined (x) { return x !== undefined; } 28 | 29 | function getIntegerOrDefault (x, defaultVal) { 30 | if ((typeof x !== 'number') || (x % 1 !== 0)) { 31 | return defaultVal; 32 | } 33 | return x; 34 | } 35 | 36 | function cloneError (err, moreInfo) { 37 | var message = err.message; 38 | if (moreInfo) { 39 | message += ': ' + moreInfo; 40 | } 41 | return {code: err.code, message: message}; 42 | } 43 | 44 | function merge () { 45 | var merged = {}; 46 | for (var i = 0, len = arguments.length; i < len; i++) { 47 | var obj = arguments[i]; 48 | for (var key in obj) { 49 | if (obj.hasOwnProperty(key)) { 50 | merged[key] = obj[key]; 51 | } 52 | } 53 | } 54 | return merged; 55 | } 56 | 57 | function nowInSec () { 58 | return Math.floor(new Date().getTime() / 1000); 59 | } 60 | 61 | /* 62 | * Use this to sort meta records array. Item to be purged first 63 | * should be the first in the array after sort. 64 | */ 65 | function defaultPurgeComparator (meta1, meta2) { 66 | // purge bad entries first 67 | if (meta1.bad !== meta2.bad) { 68 | return meta1.bad ? -1 : 1; 69 | } 70 | // purge truly stale one first 71 | var now = nowInSec(); 72 | var stale1 = now >= (meta1.expires + meta1.stale); 73 | var stale2 = now >= (meta2.expires + meta2.stale); 74 | if (stale1 !== stale2) { 75 | return stale1 ? -1 : 1; 76 | } 77 | 78 | // both fetchable (not truly staled); purge lowest priority one first 79 | if (meta1.priority !== meta2.priority) { 80 | return (meta1.priority > meta2.priority) ? -1 : 1; 81 | } 82 | 83 | // same priority; purge least access one first 84 | if (meta1.access !== meta2.access) { 85 | return (meta1.access < meta2.access) ? -1 : 1; 86 | } 87 | // compare size. big ones go first. 88 | if (meta1.size > meta2.size) { 89 | return -1; 90 | } else if (meta1.size === meta2.size) { 91 | return 0; 92 | } else { 93 | return 1; 94 | } 95 | } 96 | 97 | function Meta (storageInterface, parser, options) { 98 | this.storage = storageInterface; 99 | this.parser = parser; 100 | this.options = options || {}; 101 | this.records = []; 102 | } 103 | 104 | Meta.prototype.getMetaFromItem = function (key, item) { 105 | var meta; 106 | try { 107 | meta = this.parser.parse(item).meta; 108 | meta.key = key; 109 | } catch (ignore) { 110 | // ignore 111 | meta = {key: key, bad: true, size: item.length}; 112 | } 113 | return meta; 114 | }; 115 | 116 | Meta.prototype.updateMetaRecord = function (key, callback) { 117 | var self = this; 118 | 119 | self.storage.getItem(key, function getItemCallback (err, item) { 120 | if (!err) { 121 | self.records.push(self.getMetaFromItem(key, item)); 122 | } 123 | callback && callback(); 124 | }); 125 | }; 126 | 127 | Meta.prototype.generateRecordsHash = function () { 128 | var self = this; 129 | var retval = {}; 130 | self.records.forEach(function recordsIterator (record) { 131 | retval[record.key] = true; 132 | }); 133 | return retval; 134 | }; 135 | 136 | Meta.prototype.init = function (scanSize, callback) { 137 | // expensive operation 138 | // go through all items in storage, get meta data 139 | var self = this; 140 | var storage = self.storage; 141 | var keyPrefix = self.options.keyPrefix; 142 | var doneInserting = 0; 143 | if (scanSize <= 0) { 144 | callback && callback(); 145 | return; 146 | } 147 | 148 | storage.keys(scanSize, function getKeysCallback (err, keys) { 149 | var numKeys = keys.length; 150 | if (numKeys <= 0) { 151 | callback && callback(); 152 | return; 153 | } 154 | // generate the records hash 155 | var recordsHash = self.generateRecordsHash(); 156 | keys.forEach(function keyIterator (key) { 157 | // if the keyPrefix is different from the current options or we already have a record, ignore this key 158 | if (!recordsHash[key] && (!keyPrefix || key.indexOf(keyPrefix) === 0)) { 159 | self.updateMetaRecord(key, function updateMetaRecordCallback () { 160 | doneInserting += 1; 161 | recordsHash[key] = true; 162 | if (doneInserting === numKeys) { 163 | callback && callback(); 164 | } 165 | }); 166 | } else { 167 | doneInserting += 1; 168 | if (doneInserting === numKeys) { 169 | callback && callback(); 170 | } 171 | } 172 | }); 173 | }); 174 | }; 175 | 176 | Meta.prototype.sort = function (comparator) { 177 | this.records.sort(comparator); 178 | }; 179 | Meta.prototype.update = function (key, meta) { 180 | for (var i = 0, len = this.records.length; i < len; i++) { 181 | var record = this.records[i]; 182 | if (record.key === key) { 183 | record.bad = false; // in case it was a bad record before 184 | this.records[i] = merge(record, meta); 185 | return this.records[i]; 186 | } 187 | } 188 | // record does not exist. create a new one. 189 | meta = merge(meta, {key: key}); 190 | this.records.push(meta); 191 | return meta; 192 | }; 193 | Meta.prototype.remove = function (key) { 194 | for (var i = 0, len = this.records.length; i < len; i++) { 195 | if (this.records[i].key === key) { 196 | this.records.splice(i, 1); 197 | return; 198 | } 199 | } 200 | }; 201 | Meta.prototype.numRecords = function () { 202 | return this.records.length; 203 | }; 204 | 205 | function Parser () {} 206 | Parser.prototype.format = function (meta, value) { 207 | if (meta && meta.access > 0 && meta.expires > 0 && meta.stale >= 0 && meta.priority > 0 && meta.maxAge > 0) { 208 | return '[' + [CUR_VERSION, meta.access, meta.expires, meta.maxAge, meta.stale, meta.priority].join(':') + ']' + value; 209 | } 210 | throw new Error('invalid meta'); 211 | }; 212 | Parser.prototype.parse = function (item) { 213 | // format is: 214 | // [:::::] 215 | // in the future, parse version out first; then fields depending on version 216 | var pos = item && item.indexOf(']'); 217 | if (!pos) { 218 | throw new Error('missing meta'); 219 | } 220 | var meta = item.substring(1, pos).split(':'); 221 | if (meta.length !== 6) { 222 | throw new Error('invalid number of meta fields'); 223 | } 224 | meta = { 225 | version: meta[0], 226 | access: parseInt(meta[1], 10), 227 | expires: parseInt(meta[2], 10), 228 | maxAge: parseInt(meta[3], 10), 229 | stale: parseInt(meta[4], 10), 230 | priority: parseInt(meta[5], 10), 231 | size: item.length 232 | }; 233 | if (isNaN(meta.access) || isNaN(meta.expires) || isNaN(meta.maxAge) || isNaN(meta.stale) || meta.access <= 0 || meta.expires <= 0 || meta.maxAge <= 0 || meta.stale < 0 || meta.priority <= 0) { 234 | throw new Error('invalid meta fields'); 235 | } 236 | return { 237 | meta: meta, 238 | value: item.substring(pos + 1) 239 | }; 240 | }; 241 | 242 | function Stats (meta) { 243 | this.hit = 0; 244 | this.miss = 0; 245 | this.stale = 0; 246 | this.error = 0; 247 | this.revalidateSuccess = 0; 248 | this.revalidateFailure = 0; 249 | } 250 | Stats.prototype.toJSON = function () { 251 | var stats = { 252 | hit: this.hit, 253 | miss: this.miss, 254 | stale: this.stale, 255 | error: this.error, 256 | revalidateSuccess: this.revalidateSuccess, 257 | revalidateFailure: this.revalidateFailure 258 | }; 259 | return stats; 260 | }; 261 | 262 | /** 263 | * @class StorageLRU 264 | * @constructor 265 | * @param {Object} storageInterface A storage object (such as window.localStorage, but not limited to localStorage) 266 | * that conforms to the localStorage API. 267 | * @param {Object} [options] 268 | * @param {Number} [options.recheckDelay=-1] If the underline storage is disabled, this option defines the delay time interval 269 | * for re-checking whether the underline storage is re-enabled. Default value is -1, which 270 | * means no re-checking. 271 | * @param {String} [options.keyPrefix=''] Storage key prefix. 272 | * @param {Number} [options.purgeFactor=1] Extra space to purge. E.g. if space needed for a new item is 1000 characters, LRU will actually 273 | * try to purge (1000 + 1000 * purgeFactor) characters. 274 | * @param {Number} [options.maxPurgeAttempts=2] The number of times to load 'purgeLoadIncrease' more keys if purge cannot initially 275 | * find enough space. 276 | * @param {Number} [options.purgeLoadIncrease=500] The number of extra keys to load with each purgeLoadAttempt when purge cannot initially 277 | * find enough space. 278 | * @param {Function} [options.purgedFn] The callback function to be executed, if an item is purged. *Note* This function will be 279 | * asynchronously called, meaning, you won't be able to cancel the purge. 280 | * @param {Function} [options.purgeComparator] If you really want to, you can customize the comparator used to determine items' 281 | * purge order. The default comparator purges in this precendence order (from high to low): 282 | * bad entry (invalid meta info), 283 | * truly stale (passed stale-while-revaliate window), 284 | * lowest priority, 285 | * least recently accessed, 286 | * bigger byte size 287 | * @param {Function} [options.revalidateFn] The function to be executed to refetch the item if it becomes expired but still 288 | * in the stale-while-revalidate window. 289 | */ 290 | function StorageLRU (storageInterface, options) { 291 | var self = this; 292 | options = options || {}; 293 | var callback = options.onInit; 294 | self.options = {}; 295 | self.options.recheckDelay = isDefined(options.recheckDelay) ? options.recheckDelay : -1; 296 | self.options.keyPrefix = options.keyPrefix || DEFAULT_KEY_PREFIX; 297 | self.options.purgeLoadIncrease = getIntegerOrDefault(options.purgeLoadIncrease, DEFAULT_PURGE_LOAD_INCREASE); 298 | self.options.maxPurgeAttempts = getIntegerOrDefault(options.maxPurgeAttempts, DEFAULT_PURGE_ATTEMPTS); 299 | self.options.purgedFn = options.purgedFn; 300 | var metaOptions = { 301 | keyPrefix: self.options.keyPrefix 302 | }; 303 | self._storage = storageInterface; 304 | self._purgeComparator = options.purgeComparator || defaultPurgeComparator; 305 | self._revalidateFn = options.revalidateFn; 306 | self._parser = new Parser(); 307 | self._meta = new Meta(self._storage, self._parser, metaOptions); 308 | self._stats = new Stats(); 309 | self._enabled = true; 310 | } 311 | 312 | /** 313 | * Reports statistics information. 314 | * @method stats 315 | * @return {Object} statistics information, including: 316 | * - hit: Number of cache hits 317 | * - miss: Number of cache misses 318 | * - error: Number of errors occurred during getItem 319 | * - stale: Number of occurrances where stale items were returned (cache hit with data that 320 | * expired but still within stale-while-revalidate window) 321 | */ 322 | StorageLRU.prototype.stats = function () { 323 | return this._stats.toJSON(); 324 | }; 325 | 326 | /** 327 | * Gets a number of the keys of the items in the underline storage 328 | * @method keys 329 | * @param {Number} the number of keys to return 330 | * @param {Funtion} callback 331 | */ 332 | StorageLRU.prototype.keys = function (num, callback) { 333 | return this._storage.keys(num, callback); 334 | }; 335 | 336 | /** 337 | * Gets the item with the given key in the underline storage. Note that if the item has exipired but 338 | * is still in stale-while-revalidate window, its value will be revalidated if revalidateFn is provided 339 | * when the StorageLRU instance was created. 340 | * @method getItem 341 | * @param {String} key The key string 342 | * @param {Object} options 343 | * @param {Boolean} [options.json=false] Whether the value should be deserialized to a JSON object. 344 | * @param {Function} callback The callback function. 345 | * @param {Object} callback.error The error object (an object with code, message fields) if get failed. 346 | * @param {String|Object} callback.value The value. 347 | * @param {Object} callback.meta Meta information. Containing isStale field. isStale=true means this 348 | * item has expired (max-age reached), but still within stale-while-revalidate window. 349 | * isStale=false means this item has not reached its max-age. 350 | */ 351 | StorageLRU.prototype.getItem = function (key, options, callback) { 352 | if (!key) { 353 | callback && callback(cloneError(ERR_INVALIDKEY, key)); 354 | return; 355 | } 356 | var self = this; 357 | var prefixedKey = self._prefix(key); 358 | self._storage.getItem(prefixedKey, function getItemCallback (err, value) { 359 | if (err || value === null || value === undefined) { 360 | self._stats.miss++; 361 | self._meta.remove(prefixedKey); 362 | callback(cloneError(ERR_INVALIDKEY, key)); 363 | return; 364 | } 365 | 366 | var deserialized; 367 | try { 368 | deserialized = self._deserialize(value, options); 369 | } catch (e) { 370 | self._stats.error++; 371 | callback(cloneError(ERR_DESERIALIZE, e.message)); 372 | return; 373 | } 374 | var meta = deserialized.meta, 375 | now = nowInSec(); 376 | 377 | if ((meta.expires + meta.stale) < now) { 378 | // item exists, but expired and passed stale-while-revalidate window. 379 | // count as hit miss. 380 | self._stats.miss++; 381 | self.removeItem(key); 382 | callback(); 383 | return; 384 | } 385 | 386 | // this is a cache hit 387 | self._stats.hit++; 388 | 389 | // update the access timestamp in the underline storage 390 | try { 391 | meta.access = now; 392 | var serializedValue = self._serialize(deserialized.value, meta, options); 393 | self._storage.setItem(prefixedKey, serializedValue, function setItemCallback (err) { 394 | if (!err) { 395 | meta = self._meta.update(prefixedKey, meta); 396 | } 397 | }); 398 | } catch (ignore) {} 399 | 400 | // is the item already expired but still in the stale-while-revalidate window? 401 | var isStale = meta.expires < now; 402 | if (isStale) { 403 | self._stats.stale++; 404 | try { 405 | self._revalidate(key, meta, {json: !!(options && options.json)}); 406 | } catch (ignore) {} 407 | } 408 | callback(null, deserialized.value, {isStale: isStale}); 409 | }); 410 | }; 411 | 412 | /** 413 | * Calls the revalidateFn to fetch a fresh copy of a stale item. 414 | * @method _revalidate 415 | * @param {String} key The item key 416 | * @param {Object} meta The meta record for this item 417 | * @param {Object} options 418 | * @param {Boolean} [options.json=false] Whether the value is a JSON object. 419 | * @param {Function} [callback] 420 | * @param {Object} callback.error The error object (an object with code, message fields) if revalidateFn failed to fetch the item. 421 | * @private 422 | */ 423 | StorageLRU.prototype._revalidate = function (key, meta, options, callback) { 424 | var self = this; 425 | 426 | // if revalidateFn is defined, refetch item and save it to storage 427 | if ('function' !== typeof self._revalidateFn) { 428 | callback && callback(); 429 | return; 430 | } 431 | 432 | self._revalidateFn(key, function revalidated (err, value) { 433 | if (err) { 434 | self._stats.revalidateFailure++; 435 | callback && callback(cloneError(ERR_REVALIDATE, err.message)); 436 | return; 437 | } 438 | try { 439 | var now = nowInSec(); 440 | 441 | // update the size and expires fields, and inherit other fields. 442 | // Especially, do not update access timestamp. 443 | var newMeta = { 444 | access: meta.access, 445 | maxAge: meta.maxAge, 446 | expires: now + meta.maxAge, 447 | stale: meta.stale, 448 | priority: meta.priority 449 | }; 450 | 451 | // save into the underline storage and update meta record 452 | var serializedValue = self._serialize(value, newMeta, options); 453 | var prefixedKey = self._prefix(key); 454 | self._storage.setItem(prefixedKey, serializedValue, function setItemCallback (err) { 455 | if (!err) { 456 | newMeta.size = serializedValue.length; 457 | self._meta.update(prefixedKey, newMeta); 458 | 459 | self._stats.revalidateSuccess++; 460 | } else { 461 | self._stats.revalidateFailure++; 462 | } 463 | }); 464 | } catch (e) { 465 | self._stats.revalidateFailure++; 466 | callback && callback(cloneError(ERR_REVALIDATE, e.message)); 467 | return; 468 | } 469 | callback && callback(); 470 | }); 471 | }; 472 | 473 | /** 474 | * Saves the item with the given key in the underline storage 475 | * @method setItem 476 | * @param {String} key The key string 477 | * @param {String|Object} value The value string or JSON object 478 | * @param {Object} options 479 | * @param {Boolean} options.cacheControl Required. Use the syntax as HTTP Cache-Control header. To be 480 | * able to use LRU, you need to have a positive "max-age" value (in seconds), e.g. "max-age=300". 481 | * Another very useful field is "stale-while-revalidate", e.g. "max-age=300,stale-while-revalidate=6000". 482 | * If an item has expired (max-age reached), but still within stale-while-revalidate window, 483 | * LRU will allow retrieval the item, but tag it with isStale=true in the callback. 484 | * **Note**: 485 | * - LRU does not try to refetch the item when it is stale-while-revaliate. 486 | * - Having "no-cache" or "no-store" will abort the operation with invalid cache control error. 487 | * @param {Boolean} [options.json=false] Whether the value should be serialized to a string before saving. 488 | * @param {Number} [options.priority=3] The priority of the item. Items with lower priority will be purged before 489 | * items with higher priority, assuming other conditions are the same. 490 | * @param {Function} [callback] The callback function. 491 | * @param {Object} callback.error The error object (an object with code, message fields) if setItem failed. 492 | */ 493 | StorageLRU.prototype.setItem = function (key, value, options, callback) { 494 | if (!key) { 495 | callback && callback(cloneError(ERR_INVALIDKEY, key)); 496 | return; 497 | } 498 | 499 | var self = this; 500 | if (!self._enabled) { 501 | callback && callback(cloneError(ERR_DISABLED)); 502 | return; 503 | } 504 | 505 | // parse cache control 506 | var cacheControl = self._parseCacheControl(options && options.cacheControl); 507 | if (cacheControl['no-cache'] || cacheControl['no-store'] || !cacheControl[MAX_AGE] || cacheControl[MAX_AGE] <= 0) { 508 | callback && callback(cloneError(ERR_CACHECONTROL)); 509 | return; 510 | } 511 | 512 | // serialize value (along with meta data) 513 | var now = nowInSec(); 514 | var priority = (options && options.priority) || DEFAULT_PRIORITY; 515 | var meta = { 516 | expires: now + cacheControl[MAX_AGE], 517 | maxAge: cacheControl[MAX_AGE], 518 | stale: cacheControl[STALE_WHILE_REVALIDATE] || 0, 519 | priority: priority, 520 | access: now 521 | }; 522 | var serializedValue; 523 | try { 524 | serializedValue = self._serialize(value, meta, options); 525 | } catch (serializeError) { 526 | callback && callback(cloneError(ERR_SERIALIZE)); 527 | return; 528 | } 529 | 530 | // save into the underline storage and update meta record 531 | var prefixedKey = self._prefix(key); 532 | self._storage.setItem(prefixedKey, serializedValue, function setItemCallback (err) { 533 | if (!err) { 534 | meta.size = serializedValue.length; 535 | self._meta.update(prefixedKey, meta); 536 | callback && callback(); 537 | return; 538 | } else { 539 | //check to see if there is at least 1 valid key 540 | self.keys(1, function getKeysCallback (err, keysArr) { 541 | if (keysArr.length === 0) { 542 | // if numItems is 0, private mode is on or storage is disabled. 543 | // callback with error and return 544 | self._markAsDisabled(); 545 | callback && callback(cloneError(ERR_DISABLED)); 546 | return; 547 | } 548 | // purge and save again 549 | var spaceNeeded = serializedValue.length; 550 | self.purge(spaceNeeded, function purgeCallback (err) { 551 | if (err) { 552 | // not enough space purged 553 | callback && callback(cloneError(ERR_NOTENOUGHSPACE)); 554 | return; 555 | } 556 | // purged enough space, now try to save again 557 | self._storage.setItem(prefixedKey, serializedValue, function setItemCallback (err) { 558 | if (err) { 559 | callback && callback(cloneError(ERR_NOTENOUGHSPACE)); 560 | } else { 561 | self._meta.update(prefixedKey, meta); 562 | // setItem succeeded after the purge 563 | callback && callback(); 564 | } 565 | }); 566 | }); 567 | }); 568 | } 569 | }); 570 | }; 571 | 572 | /** 573 | * @method removeItem 574 | * @param {String} key The key string 575 | * @param {Function} [callback] The callback function. 576 | * @param {Object} callback.error The error object (an object with code, message fields) if removeItem failed. 577 | */ 578 | StorageLRU.prototype.removeItem = function (key, callback) { 579 | if (!key) { 580 | callback && callback(cloneError(ERR_INVALIDKEY, key)); 581 | return; 582 | } 583 | var self = this; 584 | key = self._prefix(key); 585 | self._storage.removeItem(key, function removeItemCallback (err) { 586 | if (err) { 587 | callback && callback(cloneError(ERR_INVALIDKEY, key)); 588 | return; 589 | } 590 | self._meta.remove(key); 591 | callback && callback(); 592 | }); 593 | }; 594 | 595 | /** 596 | * @method _parseCacheControl 597 | * @param {String} str The cache control string, following HTTP Cache-Control header syntax. 598 | * @return {Object} 599 | * @private 600 | */ 601 | StorageLRU.prototype._parseCacheControl = function (str) { 602 | var cacheControl = {}; 603 | if (str) { 604 | var parts = str.toLowerCase().split(','); 605 | for (var i = 0, len = parts.length; i < len; i++) { 606 | var kv = parts[i].split('='); 607 | if (kv.length === 2) { 608 | cacheControl[kv[0]] = kv[1]; 609 | } else if (kv.length === 1) { 610 | cacheControl[kv[0]] = true; 611 | } 612 | } 613 | if (cacheControl[MAX_AGE]) { 614 | cacheControl[MAX_AGE] = parseInt(cacheControl[MAX_AGE], 10) || 0; 615 | } 616 | if (cacheControl[STALE_WHILE_REVALIDATE]) { 617 | cacheControl[STALE_WHILE_REVALIDATE] = parseInt(cacheControl[STALE_WHILE_REVALIDATE], 10) || 0; 618 | } 619 | } 620 | return cacheControl; 621 | }; 622 | 623 | /** 624 | * Prefix the item key with the keyPrefix defined in "options" when LRU instance was created. 625 | * @method _prefix 626 | * @param {String} key The item key. 627 | * @return {String} The prefixed key. 628 | * @private 629 | */ 630 | StorageLRU.prototype._prefix = function (key) { 631 | return this.options.keyPrefix + key; 632 | }; 633 | 634 | /** 635 | * Remove the prefix from the prefixed item key. 636 | * The keyPrefix is defined in "options" when LRU instance was created. 637 | * @method _deprefix 638 | * @param {String} prefixedKey The prefixed item key. 639 | * @return {String} The item key. 640 | * @private 641 | */ 642 | StorageLRU.prototype._deprefix = function (prefixedKey) { 643 | var prefix = this.options.keyPrefix; 644 | return prefix ? prefixedKey.substring(prefix.length) : prefixedKey; 645 | }; 646 | 647 | /** 648 | * Mark the storage as disabled. For example, when in Safari private mode, localStorage 649 | * is disabled. During setItem(), LRU will check whether the underline storage 650 | * is disabled. 651 | * If the LRU was created with a recheckDelay option, LRU will re-check whether the underline 652 | * storage is disabled. after the specified delay time. 653 | * @method _markAsDisabled 654 | * @private 655 | */ 656 | StorageLRU.prototype._markAsDisabled = function () { 657 | var self = this; 658 | self._enabled = false; 659 | // set a timeout to mark the cache back to enabled so that status can be checked again 660 | var recheckDelay = self.options.recheckDelay; 661 | if (recheckDelay > 0) { 662 | setTimeout(function reEnable() { 663 | self._enabled = true; 664 | }, recheckDelay); 665 | } 666 | }; 667 | 668 | /** 669 | * Serializes the item value and meta info into a string. 670 | * @method _serialize 671 | * @param {String|Object} value 672 | * @param {Object} meta Meta info for this item, such as access ts, expire ts, stale-while-revalidate window size 673 | * @param {Object} options 674 | * @param {Boolean} [options.json=false] 675 | * @return {String} the serialized string to store in underline storage 676 | * @private 677 | * @throw Error 678 | */ 679 | StorageLRU.prototype._serialize = function (value, meta, options) { 680 | var v = (options && options.json) ? JSON.stringify(value) : value; 681 | return this._parser.format(meta, v); 682 | }; 683 | 684 | /** 685 | * De-serializes the stored string into item value and meta info. 686 | * @method _deserialize 687 | * @param {String} str The stored string 688 | * @param {Object} options 689 | * @param {Boolean} [options.json=false] 690 | * @return {Object} An object containing "value" (for item value) and "meta" (Meta data object for this item, such as access ts, expire ts, stale-while-revalidate window size). 691 | * @private 692 | * @throw Error 693 | */ 694 | StorageLRU.prototype._deserialize = function (str, options) { 695 | var parsed = this._parser.parse(str); 696 | return { 697 | meta: parsed.meta, 698 | value: options.json? JSON.parse(parsed.value) : parsed.value 699 | }; 700 | }; 701 | 702 | /** 703 | * Purge the underline storage to make room for new data. If options.purgedFn is defined 704 | * when LRU instance was created, this function will invoke it with the array if purged keys asynchronously. 705 | * If the meta data for all objects has not yet been built, then it will occur in this function. 706 | * @method purge 707 | * @param {Number} spaceNeeded The char count of space needed for the new data. Note that 708 | * if options.purgeFactor is defined when LRU instance was created, extra space 709 | * will be purged. E.g. if spaceNeeded is 1000 characters, LRU will actually 710 | * try to purge (1000 + 1000 * purgeFactor) characters. 711 | * @param {Boolean} forcePurge True if we want to ignore un-initialized records, else false; 712 | * @param {Function} callback 713 | * @param {Error} callback.error if the space that we were able to purge was less than spaceNeeded. 714 | */ 715 | StorageLRU.prototype.purge = function (spaceNeeded, callback) { 716 | var self = this; 717 | var factor = Math.max(0, self.options.purgeFactor) || 1; 718 | var padding = Math.round(spaceNeeded * factor); 719 | 720 | var removeData = { 721 | purged: [], 722 | recordsToRemove: [], 723 | size: spaceNeeded + padding 724 | }; 725 | 726 | var attempts = []; 727 | for (var i = 0; i < self.options.maxPurgeAttempts; i++) { 728 | attempts.push((i + 1) * self.options.purgeLoadIncrease); 729 | } 730 | 731 | asyncEachSeries(attempts, function purgeAttempt(loadSize, attemptDone) { 732 | removeData.recordsToRemove = []; 733 | removeData.purged = []; 734 | 735 | self._meta.init(loadSize, function doneInit() { 736 | self._meta.sort(self._purgeComparator); 737 | asyncEachSeries(self._meta.records, function removeItem(record, cb) { 738 | // mark record to remove, to remove in batch later for performance 739 | record.remove = true; 740 | removeData.purged.push(self._deprefix(record.key)); // record purged key 741 | self._storage.removeItem(record.key, function removeItemCallback (err) { 742 | // if there was an error removing, remove the record but assume we still need some space 743 | if (!err) { 744 | removeData.size = removeData.size - record.size; 745 | } 746 | if (removeData.size > 0) { 747 | cb(); // keep removing 748 | } else { 749 | cb(true); // done removing 750 | } 751 | }); 752 | }, function itemsRemoved(ignore) { 753 | // remove records that were marked to remove 754 | self._meta.records = self._meta.records.filter(function shouldKeepRecord(record) { 755 | return record.remove !== true; 756 | }); 757 | 758 | // invoke purgedFn if it is defined 759 | var purgedCallback = self.options.purgedFn; 760 | var purged = removeData.purged; 761 | if (purgedCallback && purged.length > 0) { 762 | // execute the purged callback asynchronously to prevent library users 763 | // from potentially slow down the purge process by executing long tasks 764 | // in this callback. 765 | setImmediate(function purgeTimeout() { 766 | purgedCallback(purged); 767 | }); 768 | } 769 | 770 | if (removeData.size <= padding) { 771 | // removed enough space, stop subsequent purge attempts 772 | attemptDone(true); 773 | } else { 774 | attemptDone(); 775 | } 776 | }); 777 | }); 778 | }, function attemptsDone() { 779 | // async series reached the end, either because all attempts were tried, 780 | // or enough space was already freed. 781 | 782 | // if enough space was made for spaceNeeded, consider purge as success 783 | if (callback) { 784 | if (removeData.size <= padding) { 785 | callback(); 786 | } else { 787 | callback(new Error('still need ' + (removeData.size - padding))); 788 | } 789 | } 790 | }); 791 | }; 792 | 793 | module.exports = StorageLRU; --------------------------------------------------------------------------------