├── .npmignore ├── lib ├── jspack.js ├── buffer_ieee754.js ├── async.js └── underscore.js ├── test ├── large.whisper ├── testcreate.whisper └── hoard.test.coffee ├── package.json ├── LICENSE ├── Cakefile ├── README.md └── src └── hoard.coffee /.npmignore: -------------------------------------------------------------------------------- 1 | test/* 2 | 3 | -------------------------------------------------------------------------------- /lib/jspack.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cgbystrom/hoard/HEAD/lib/jspack.js -------------------------------------------------------------------------------- /test/large.whisper: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cgbystrom/hoard/HEAD/test/large.whisper -------------------------------------------------------------------------------- /test/testcreate.whisper: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cgbystrom/hoard/HEAD/test/testcreate.whisper -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hoard", 3 | "version": "0.1.5", 4 | "description": "node.js lib for storing time series data on disk, similar to RRD.", 5 | "homepage": "https://github.com/cgbystrom/hoard", 6 | "author": "Carl Byström http://cgbystrom.com", 7 | "keywords": ["timeseries", "rrd", "rrdtool", "db", "database", "metric", "stats", "statistics"], 8 | "repository": {"type": "git", "url": "https://github.com/cgbystrom/hoard.git"}, 9 | "bugs": {"web": "https://github.com/cgbystrom/hoard/issues"}, 10 | "directories": {"lib": "./lib"}, 11 | "main": "./lib/hoard.js", 12 | "engines": { 13 | "node": ">= 0.4.0" 14 | }, 15 | "dependencies": { 16 | "put": ">= 0.0.5", 17 | "binary": ">= 0.2.5" 18 | }, 19 | "devDependencies": { 20 | "coffee-script": ">= 1.0.1", 21 | "expresso": ">= 0.8.1" 22 | }, 23 | "licenses": [{"type": "MIT", "url": "https://github.com/cgbystrom/hoard/raw/master/LICENSE"}] 24 | } 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2011 Carl Byström 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Cakefile: -------------------------------------------------------------------------------- 1 | fs = require "fs" 2 | path = require "path" 3 | {spawn, exec} = require "child_process" 4 | stdout = process.stdout 5 | 6 | # Use executables installed with npm bundle. 7 | #process.env["PATH"] = "node_modules/.bin:#{process.env["PATH"]}" 8 | 9 | # ANSI Terminal Colors. 10 | bold = "\033[0;1m" 11 | red = "\033[0;31m" 12 | green = "\033[0;32m" 13 | reset = "\033[0m" 14 | 15 | # Log a message with a color. 16 | log = (message, color, explanation) -> 17 | console.log color + message + reset + ' ' + (explanation or '') 18 | 19 | # Handle error and kill the process. 20 | onError = (err) -> 21 | if err 22 | process.stdout.write "#{red}#{err.stack}#{reset}\n" 23 | process.exit -1 24 | 25 | # Setup development dependencies, not part of runtime dependencies. 26 | task "setup", "Install development dependencies", -> 27 | fs.readFile "package.json", "utf8", (err, package) -> 28 | log "Need runtime dependencies, installing into node_modules ...", green 29 | exec "npm bundle", onError 30 | 31 | log "Need development dependencies, installing ...", green 32 | for name, version of JSON.parse(package).devDependencies 33 | log "Installing #{name} #{version}", green 34 | exec "npm bundle install \"#{name}@#{version}\"", onError 35 | 36 | task "install", "Install Hoard in your local repository", -> 37 | build (err) -> 38 | onError err 39 | log "Installing Hoard ...", green 40 | exec "npm install", (err, stdout, stderr) -> 41 | process.stdout.write stderr 42 | onError err 43 | 44 | build = (callback) -> 45 | log "Compiling CoffeeScript to JavaScript ...", green 46 | exec "coffee -c -l -b -o lib src", (err, stdout) -> callback err 47 | task "build", "Compile CoffeeScript to JavaScript", -> build onError 48 | 49 | runTests = (callback) -> 50 | log "Running test suite ...", green 51 | exec "expresso -I src -I lib test/*.test.coffee", (err, stdout, stderr) -> 52 | process.stdout.write stdout 53 | process.binding('stdio').writeError stderr 54 | callback err if callback 55 | 56 | clean = (callback) -> 57 | log "Removing build files ...", green 58 | exec "rm -f test/*.hoard", (err, stdout) -> callback err 59 | task "clean", "Clean up build files", -> clean onError 60 | 61 | task "test", "Run all tests", -> 62 | runTests (err) -> 63 | process.stdout.on "drain", -> process.exit -1 if err 64 | -------------------------------------------------------------------------------- /lib/buffer_ieee754.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2008, Fair Oaks Labs, Inc. 2 | // All rights reserved. 3 | // 4 | // Redistribution and use in source and binary forms, with or without 5 | // modification, are permitted provided that the following conditions are met: 6 | // 7 | // * Redistributions of source code must retain the above copyright notice, 8 | // this list of conditions and the following disclaimer. 9 | // 10 | // * Redistributions in binary form must reproduce the above copyright notice, 11 | // this list of conditions and the following disclaimer in the documentation 12 | // and/or other materials provided with the distribution. 13 | // 14 | // * Neither the name of Fair Oaks Labs, Inc. nor the names of its contributors 15 | // may be used to endorse or promote products derived from this software 16 | // without specific prior written permission. 17 | // 18 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 22 | // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | // POSSIBILITY OF SUCH DAMAGE. 29 | // 30 | // 31 | // Modifications to writeIEEE754 to support negative zeroes made by Brian White 32 | 33 | exports.readIEEE754 = function(buffer, offset, endian, mLen, nBytes) { 34 | var e, m, 35 | bBE = (endian === 'big'), 36 | eLen = nBytes * 8 - mLen - 1, 37 | eMax = (1 << eLen) - 1, 38 | eBias = eMax >> 1, 39 | nBits = -7, 40 | i = bBE ? 0 : (nBytes - 1), 41 | d = bBE ? 1 : -1, 42 | s = buffer[offset + i]; 43 | 44 | i += d; 45 | 46 | e = s & ((1 << (-nBits)) - 1); 47 | s >>= (-nBits); 48 | nBits += eLen; 49 | for (; nBits > 0; e = e * 256 + buffer[offset + i], i += d, nBits -= 8); 50 | 51 | m = e & ((1 << (-nBits)) - 1); 52 | e >>= (-nBits); 53 | nBits += mLen; 54 | for (; nBits > 0; m = m * 256 + buffer[offset + i], i += d, nBits -= 8); 55 | 56 | if (e === 0) { 57 | e = 1 - eBias; 58 | } else if (e === eMax) { 59 | return m ? NaN : ((s ? -1 : 1) * Infinity); 60 | } else { 61 | m = m + Math.pow(2, mLen); 62 | e = e - eBias; 63 | } 64 | return (s ? -1 : 1) * m * Math.pow(2, e - mLen); 65 | }; 66 | 67 | exports.writeIEEE754 = function(buffer, value, offset, endian, mLen, nBytes) { 68 | var e, m, c, 69 | bBE = (endian === 'big'), 70 | eLen = nBytes * 8 - mLen - 1, 71 | eMax = (1 << eLen) - 1, 72 | eBias = eMax >> 1, 73 | rt = (mLen === 23 ? Math.pow(2, -24) - Math.pow(2, -77) : 0), 74 | i = bBE ? (nBytes-1) : 0, 75 | d = bBE ? -1 : 1, 76 | s = value < 0 || (value === 0 && 1 / value < 0) ? 1 : 0; 77 | 78 | value = Math.abs(value); 79 | 80 | if (isNaN(value) || value === Infinity) { 81 | m = isNaN(value) ? 1 : 0; 82 | e = eMax; 83 | } else { 84 | e = Math.floor(Math.log(value) / Math.LN2); 85 | if (value * (c = Math.pow(2, -e)) < 1) { 86 | e--; 87 | c *= 2; 88 | } 89 | if (e+eBias >= 1) { 90 | value += rt / c; 91 | } else { 92 | value += rt * Math.pow(2, 1 - eBias); 93 | } 94 | if (value * c >= 2) { 95 | e++; 96 | c /= 2; 97 | } 98 | 99 | if (e + eBias >= eMax) { 100 | m = 0; 101 | e = eMax; 102 | } else if (e + eBias >= 1) { 103 | m = (value * c - 1) * Math.pow(2, mLen); 104 | e = e + eBias; 105 | } else { 106 | m = value * Math.pow(2, eBias - 1) * Math.pow(2, mLen); 107 | e = 0; 108 | } 109 | } 110 | 111 | for (; mLen >= 8; buffer[offset + i] = m & 0xff, i += d, m /= 256, mLen -= 8); 112 | 113 | e = (e << mLen) | m; 114 | eLen += mLen; 115 | for (; eLen > 0; buffer[offset + i] = e & 0xff, i += d, e /= 256, eLen -= 8); 116 | 117 | buffer[offset + i - d] |= s * 128; 118 | }; 119 | -------------------------------------------------------------------------------- /test/hoard.test.coffee: -------------------------------------------------------------------------------- 1 | assert = require 'assert' 2 | fs = require 'fs' 3 | path = require 'path' 4 | hoard = require "hoard" 5 | equal = assert.equal 6 | 7 | FILENAME = 'test/large.whisper' 8 | unixTime = -> parseInt(new Date().getTime() / 1000) 9 | 10 | # Tests against Python generated Whisper data file 11 | module.exports = 12 | 'test info()': (beforeExit) -> 13 | called = false 14 | hoard.info FILENAME, (err, header) -> 15 | called = true 16 | assert.equal 94608000, header.maxRetention 17 | assert.equal 0.5, header.xFilesFactor 18 | assert.equal 2, header.archives.length 19 | 20 | archive = header.archives[0] 21 | assert.equal 31536000, archive.retention 22 | assert.equal 3600, archive.secondsPerPoint 23 | assert.equal 8760, archive.points 24 | assert.equal 105120, archive.size 25 | assert.equal 40, archive.offset 26 | 27 | archive = header.archives[1] 28 | assert.equal 94608000, archive.retention 29 | assert.equal 86400, archive.secondsPerPoint 30 | assert.equal 1095, archive.points 31 | assert.equal 13140, archive.size 32 | assert.equal 105160, archive.offset 33 | 34 | beforeExit -> assert.ok called 35 | 36 | 'test fetch()': (beforeExit) -> 37 | called = false 38 | fromTime = 1311161605 39 | toTime = 1311179605 40 | 41 | hoard.fetch FILENAME, fromTime, toTime, (err, timeInfo, values) -> 42 | throw err if err 43 | called = true 44 | assert.equal 1311163200, timeInfo[0] 45 | assert.equal 1311181200, timeInfo[1] 46 | assert.equal 3600, timeInfo[2] 47 | v = [2048, 4546, 794, 805, 4718] 48 | assert.length values, v.length 49 | assert.eql v, values 50 | 51 | beforeExit -> assert.ok called, 'Callback must return' 52 | 53 | 'test create()': (beforeExit) -> 54 | called = false 55 | filename = 'test/testcreate.hoard' 56 | if path.existsSync(filename) then fs.unlinkSync(filename) 57 | 58 | hoard.create filename, [[1, 60], [10, 600]], 0.5, (err) -> 59 | if err then throw err 60 | 61 | hoardFile = fs.readFileSync(filename) 62 | whisperFile = fs.readFileSync('test/testcreate.whisper') 63 | equal whisperFile.length, hoardFile.length, "File lengths must match" 64 | 65 | hoard.info filename, (err, header) -> 66 | called = true 67 | assert.equal 6000, header.maxRetention 68 | assert.equal 0.5, header.xFilesFactor 69 | assert.equal 2, header.archives.length 70 | 71 | archive = header.archives[0] 72 | assert.equal 60, archive.retention 73 | assert.equal 1, archive.secondsPerPoint 74 | assert.equal 60, archive.points 75 | assert.equal 720, archive.size 76 | assert.equal 40, archive.offset 77 | 78 | archive = header.archives[1] 79 | assert.equal 6000, archive.retention 80 | assert.equal 10, archive.secondsPerPoint 81 | assert.equal 600, archive.points 82 | assert.equal 7200, archive.size 83 | assert.equal 760, archive.offset 84 | 85 | # FIXME: Compare to real file, must mock creation timestamp in create() 86 | #assert.eql whisperFile, hoardFile 87 | 88 | beforeExit -> assert.ok called, "Callback must return" 89 | 90 | 'test update()': (beforeExit) -> 91 | called = false 92 | filename = 'test/testupdate.hoard' 93 | if path.existsSync(filename) then fs.unlinkSync(filename) 94 | 95 | hoard.create filename, [[3600, 8760], [86400, 1095]], 0.5, (err) -> 96 | if err then throw err 97 | hoard.update filename, 1337, 1311169605, (err) -> 98 | if err then throw err 99 | hoard.fetch filename, 1311161605, 1311179605, (err, timeInfo, values) -> 100 | if err then throw err 101 | called = true 102 | equal 1311163200, timeInfo[0] 103 | equal 1311181200, timeInfo[1] 104 | equal 3600, timeInfo[2] 105 | assert.length values, 5 106 | equal 1337, values[1] 107 | 108 | beforeExit -> assert.ok called, "Callback must return" 109 | 110 | 'test updateMany()': (beforeExit) -> 111 | called = false 112 | filename = 'test/testupdatemany.hoard' 113 | if path.existsSync(filename) then fs.unlinkSync(filename) 114 | 115 | tsData = JSON.parse(fs.readFileSync('test/timeseriesdata.json', 'utf8')) 116 | console.log tsData[0] 117 | hoard.create filename, [[3600, 8760], [86400, 1095]], 0.5, (err) -> 118 | if err then throw err 119 | hoard.updateMany filename, tsData, (err) -> 120 | if err then throw err 121 | from = 1311277105 122 | to = 1311295105 123 | hoard.fetch filename, from, to, (err, timeInfo, values) -> 124 | if err then throw err 125 | called = true 126 | equal 1311278400, timeInfo[0] 127 | equal 1311296400, timeInfo[1] 128 | equal 3600, timeInfo[2] 129 | assert.length values, 5 130 | assert.eql [1043, 3946, 1692, 899, 2912], values 131 | 132 | beforeExit -> assert.ok called, "Callback must return" 133 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | hoard 2 | ===== 3 | 4 | Hoard is a library for storing time series data data on disk in an efficient way. 5 | The format lends itself very for collecting and recording data over time, for example 6 | temperatures, CPU utilization, bandwidth consumption, requests per second and other metrics. 7 | It is very similar to [RRD][RRD], but comes with a few improvements. 8 | 9 | Background 10 | ---------- 11 | Hoard is based on an existing file format called Whisper. 12 | It was designed by Chris Davis for the [Graphite][Graphite] project and features improvements over the RRD file format. 13 | Whisper is implemented in Python and Hoard is merely a straight-forward port 14 | of that implementation over to node.js. 15 | 16 | [RRD][RRD] is a very well-known file format for storing time series data on disk and has been around for over a decade. 17 | The [Whisper][Whisper] file format tries to overcome a few limitations with RRD that makes it impractical at certain times. 18 | This new file format address the following issues, currently found in RRD: 19 | 20 | * No updates for a timestamp prior the most recent update 21 | This makes it impossible to file old, possibly missed, updates to an RRD archive. 22 | A big limitation when you try to be fault-tolerant and handle metrics arriving out-of-order. 23 | * No batch updates 24 | RRD doesn't support making updates of multiple values in a single batch. 25 | Updating each value separately yields many unneccessary and expensive disk operations 26 | * No irregular updates 27 | When you update an RRD but don't follow up with another update soon, your original update will be lost. 28 | 29 | (These issues were prevalent in RRD at the time [Whisper][Whisper] was designed, it may have changed since then) 30 | 31 | A simple implementation of RRD using C bindings was therefore out of the question for the reasons listed above. 32 | Using the C library would have required another native dependency and lot of glue getting it to work in an asynchronous manner. 33 | The current implementation in CoffeeScript is really straight-forward, checks in at around 600 LOC. 34 | Performance should really not be an issue compared to a native version since A) V8 is really fast and B) You're ultimately disk I/O bound. 35 | In a high-throughput environment you are also very likely to be buffering your data an only write to disk at given intervals. 36 | 37 | The name "Hoard" was selected because of the meaning "A stock or store of money or valued objects, typically one that is secret or carefully guarded". 38 | (See http://en.wikipedia.org/wiki/Hoard) 39 | 40 | 41 | Installing 42 | ---------- 43 | Just use NPM and type: 44 | 45 | npm install hoard 46 | 47 | 48 | Example 49 | ------- 50 | 51 | ```javascript 52 | // Create a Hoard file for storing time series data. 53 | // Inside of it there will be two archives with retention periods: 54 | // 1) 1 second per point for a total of 60 points (60 seconds of data) 55 | // 2) 10 second per point for a total of 600 points (100 minutes of data) 56 | hoard.create('users.hoard', [[1, 60], [10, 600]], 0.5, function(err) { 57 | if (err) throw err; 58 | console.log('Hoard file created!'); 59 | }); 60 | ``` 61 | 62 | ```javascript 63 | // Update an existing Hoard file with value 1337 for timestamp 1311169605 64 | // When doing multiple updates in batch, use updateMany() instead as it's faster 65 | hoard.update('users.hoard', 1337, 1311169605, function(err) { 66 | if (err) throw err; 67 | console.log('Hoard file updated!'); 68 | }); 69 | ``` 70 | 71 | ```javascript 72 | // Update multiple values at once in an existing Hoard file. 73 | // This function is much faster when dealing with multiple values 74 | // that need to be written at once. 75 | hoard.update('users.hoard', [[1312490305, 4976], [1312492105, 3742]], function(err) { 76 | if (err) throw err; 77 | console.log('Hoard file updated!'); 78 | }); 79 | ``` 80 | 81 | ```javascript 82 | // Retrieve data from a Hoard file between timestamps 1311161605 and 1311179605 83 | hoard.fetch('users.hoard', 1311161605, 1311179605, function(err, timeInfo, values) { 84 | if (err) throw err; 85 | console.log('Values', values); // Displays an array of values 86 | }); 87 | ``` 88 | 89 | Implementation details 90 | ---------------------- 91 | Hoard is written for node.js using CoffeeScript. Uses almost the same number of lines as 92 | the Python version. Probably requires some additional lines for async parts but those things certainly 93 | can be reduced by using more/better async/CoffeeScript idioms. It is a line-by-line port so perhaps there's 94 | a more fitting node.js paradigm that can be used to further improve readability and performance of this. 95 | 96 | Some dependencies such as underscore.js and async.js were packaged inside instead as a separate dependency. 97 | Not sure of the best practice of doing this, but depending on these packages through NPM felt unneccesary 98 | since they both are pure JS code. 99 | 100 | The tests are testing the implementation against the Python implementation to ensure 101 | maximum compatibility. They don't require the Python version to be installed but rather uses 102 | files generated by it. The tests were implemented using Expresso after some experimentation with Vows. 103 | Ran into some issues with Vows and decided to use the much simpler (and dumber) Expresso instead. 104 | 105 | 106 | Authors 107 | ------- 108 | 109 | - Carl Byström ([@cgbystrom](http://twitter.com/cgbystrom)) 110 | - [Original file format design][Whisper] by Chris Davis 111 | 112 | License 113 | ------- 114 | 115 | Open-source licensed under the MIT license (see _LICENSE_ file for details). 116 | 117 | 118 | [RRD]: http://oss.oetiker.ch/rrdtool/ 119 | [Graphite]: http://graphite.wikidot.com 120 | [Whisper]: http://graphite.wikidot.com/whisper 121 | -------------------------------------------------------------------------------- /lib/async.js: -------------------------------------------------------------------------------- 1 | /*global setTimeout: false, console: false */ 2 | (function () { 3 | 4 | var async = {}; 5 | 6 | // global on the server, window in the browser 7 | var root = this, 8 | previous_async = root.async; 9 | 10 | if (typeof module !== 'undefined' && module.exports) { 11 | module.exports = async; 12 | } 13 | else { 14 | root.async = async; 15 | } 16 | 17 | async.noConflict = function () { 18 | root.async = previous_async; 19 | return async; 20 | }; 21 | 22 | //// cross-browser compatiblity functions //// 23 | 24 | var _forEach = function (arr, iterator) { 25 | if (arr.forEach) { 26 | return arr.forEach(iterator); 27 | } 28 | for (var i = 0; i < arr.length; i += 1) { 29 | iterator(arr[i], i, arr); 30 | } 31 | }; 32 | 33 | var _map = function (arr, iterator) { 34 | if (arr.map) { 35 | return arr.map(iterator); 36 | } 37 | var results = []; 38 | _forEach(arr, function (x, i, a) { 39 | results.push(iterator(x, i, a)); 40 | }); 41 | return results; 42 | }; 43 | 44 | var _reduce = function (arr, iterator, memo) { 45 | if (arr.reduce) { 46 | return arr.reduce(iterator, memo); 47 | } 48 | _forEach(arr, function (x, i, a) { 49 | memo = iterator(memo, x, i, a); 50 | }); 51 | return memo; 52 | }; 53 | 54 | var _keys = function (obj) { 55 | if (Object.keys) { 56 | return Object.keys(obj); 57 | } 58 | var keys = []; 59 | for (var k in obj) { 60 | if (obj.hasOwnProperty(k)) { 61 | keys.push(k); 62 | } 63 | } 64 | return keys; 65 | }; 66 | 67 | var _indexOf = function (arr, item) { 68 | if (arr.indexOf) { 69 | return arr.indexOf(item); 70 | } 71 | for (var i = 0; i < arr.length; i += 1) { 72 | if (arr[i] === item) { 73 | return i; 74 | } 75 | } 76 | return -1; 77 | }; 78 | 79 | //// exported async module functions //// 80 | 81 | //// nextTick implementation with browser-compatible fallback //// 82 | if (typeof process === 'undefined' || !(process.nextTick)) { 83 | async.nextTick = function (fn) { 84 | setTimeout(fn, 0); 85 | }; 86 | } 87 | else { 88 | async.nextTick = process.nextTick; 89 | } 90 | 91 | async.forEach = function (arr, iterator, callback) { 92 | if (!arr.length) { 93 | return callback(); 94 | } 95 | var completed = 0; 96 | _forEach(arr, function (x) { 97 | iterator(x, function (err) { 98 | if (err) { 99 | callback(err); 100 | callback = function () {}; 101 | } 102 | else { 103 | completed += 1; 104 | if (completed === arr.length) { 105 | callback(); 106 | } 107 | } 108 | }); 109 | }); 110 | }; 111 | 112 | async.forEachSeries = function (arr, iterator, callback) { 113 | if (!arr.length) { 114 | return callback(); 115 | } 116 | var completed = 0; 117 | var iterate = function () { 118 | iterator(arr[completed], function (err) { 119 | if (err) { 120 | callback(err); 121 | callback = function () {}; 122 | } 123 | else { 124 | completed += 1; 125 | if (completed === arr.length) { 126 | callback(); 127 | } 128 | else { 129 | iterate(); 130 | } 131 | } 132 | }); 133 | }; 134 | iterate(); 135 | }; 136 | 137 | 138 | var doParallel = function (fn) { 139 | return function () { 140 | var args = Array.prototype.slice.call(arguments); 141 | return fn.apply(null, [async.forEach].concat(args)); 142 | }; 143 | }; 144 | var doSeries = function (fn) { 145 | return function () { 146 | var args = Array.prototype.slice.call(arguments); 147 | return fn.apply(null, [async.forEachSeries].concat(args)); 148 | }; 149 | }; 150 | 151 | 152 | var _asyncMap = function (eachfn, arr, iterator, callback) { 153 | var results = []; 154 | arr = _map(arr, function (x, i) { 155 | return {index: i, value: x}; 156 | }); 157 | eachfn(arr, function (x, callback) { 158 | iterator(x.value, function (err, v) { 159 | results[x.index] = v; 160 | callback(err); 161 | }); 162 | }, function (err) { 163 | callback(err, results); 164 | }); 165 | }; 166 | async.map = doParallel(_asyncMap); 167 | async.mapSeries = doSeries(_asyncMap); 168 | 169 | 170 | // reduce only has a series version, as doing reduce in parallel won't 171 | // work in many situations. 172 | async.reduce = function (arr, memo, iterator, callback) { 173 | async.forEachSeries(arr, function (x, callback) { 174 | iterator(memo, x, function (err, v) { 175 | memo = v; 176 | callback(err); 177 | }); 178 | }, function (err) { 179 | callback(err, memo); 180 | }); 181 | }; 182 | // inject alias 183 | async.inject = async.reduce; 184 | // foldl alias 185 | async.foldl = async.reduce; 186 | 187 | async.reduceRight = function (arr, memo, iterator, callback) { 188 | var reversed = _map(arr, function (x) { 189 | return x; 190 | }).reverse(); 191 | async.reduce(reversed, memo, iterator, callback); 192 | }; 193 | // foldr alias 194 | async.foldr = async.reduceRight; 195 | 196 | var _filter = function (eachfn, arr, iterator, callback) { 197 | var results = []; 198 | arr = _map(arr, function (x, i) { 199 | return {index: i, value: x}; 200 | }); 201 | eachfn(arr, function (x, callback) { 202 | iterator(x.value, function (v) { 203 | if (v) { 204 | results.push(x); 205 | } 206 | callback(); 207 | }); 208 | }, function (err) { 209 | callback(_map(results.sort(function (a, b) { 210 | return a.index - b.index; 211 | }), function (x) { 212 | return x.value; 213 | })); 214 | }); 215 | }; 216 | async.filter = doParallel(_filter); 217 | async.filterSeries = doSeries(_filter); 218 | // select alias 219 | async.select = async.filter; 220 | async.selectSeries = async.filterSeries; 221 | 222 | var _reject = function (eachfn, arr, iterator, callback) { 223 | var results = []; 224 | arr = _map(arr, function (x, i) { 225 | return {index: i, value: x}; 226 | }); 227 | eachfn(arr, function (x, callback) { 228 | iterator(x.value, function (v) { 229 | if (!v) { 230 | results.push(x); 231 | } 232 | callback(); 233 | }); 234 | }, function (err) { 235 | callback(_map(results.sort(function (a, b) { 236 | return a.index - b.index; 237 | }), function (x) { 238 | return x.value; 239 | })); 240 | }); 241 | }; 242 | async.reject = doParallel(_reject); 243 | async.rejectSeries = doSeries(_reject); 244 | 245 | var _detect = function (eachfn, arr, iterator, main_callback) { 246 | eachfn(arr, function (x, callback) { 247 | iterator(x, function (result) { 248 | if (result) { 249 | main_callback(x); 250 | main_callback = function () {}; 251 | } 252 | else { 253 | callback(); 254 | } 255 | }); 256 | }, function (err) { 257 | main_callback(); 258 | }); 259 | }; 260 | async.detect = doParallel(_detect); 261 | async.detectSeries = doSeries(_detect); 262 | 263 | async.some = function (arr, iterator, main_callback) { 264 | async.forEach(arr, function (x, callback) { 265 | iterator(x, function (v) { 266 | if (v) { 267 | main_callback(true); 268 | main_callback = function () {}; 269 | } 270 | callback(); 271 | }); 272 | }, function (err) { 273 | main_callback(false); 274 | }); 275 | }; 276 | // any alias 277 | async.any = async.some; 278 | 279 | async.every = function (arr, iterator, main_callback) { 280 | async.forEach(arr, function (x, callback) { 281 | iterator(x, function (v) { 282 | if (!v) { 283 | main_callback(false); 284 | main_callback = function () {}; 285 | } 286 | callback(); 287 | }); 288 | }, function (err) { 289 | main_callback(true); 290 | }); 291 | }; 292 | // all alias 293 | async.all = async.every; 294 | 295 | async.sortBy = function (arr, iterator, callback) { 296 | async.map(arr, function (x, callback) { 297 | iterator(x, function (err, criteria) { 298 | if (err) { 299 | callback(err); 300 | } 301 | else { 302 | callback(null, {value: x, criteria: criteria}); 303 | } 304 | }); 305 | }, function (err, results) { 306 | if (err) { 307 | return callback(err); 308 | } 309 | else { 310 | var fn = function (left, right) { 311 | var a = left.criteria, b = right.criteria; 312 | return a < b ? -1 : a > b ? 1 : 0; 313 | }; 314 | callback(null, _map(results.sort(fn), function (x) { 315 | return x.value; 316 | })); 317 | } 318 | }); 319 | }; 320 | 321 | async.auto = function (tasks, callback) { 322 | callback = callback || function () {}; 323 | var keys = _keys(tasks); 324 | if (!keys.length) { 325 | return callback(null); 326 | } 327 | 328 | var completed = []; 329 | 330 | var listeners = []; 331 | var addListener = function (fn) { 332 | listeners.unshift(fn); 333 | }; 334 | var removeListener = function (fn) { 335 | for (var i = 0; i < listeners.length; i += 1) { 336 | if (listeners[i] === fn) { 337 | listeners.splice(i, 1); 338 | return; 339 | } 340 | } 341 | }; 342 | var taskComplete = function () { 343 | _forEach(listeners, function (fn) { 344 | fn(); 345 | }); 346 | }; 347 | 348 | addListener(function () { 349 | if (completed.length === keys.length) { 350 | callback(null); 351 | } 352 | }); 353 | 354 | _forEach(keys, function (k) { 355 | var task = (tasks[k] instanceof Function) ? [tasks[k]]: tasks[k]; 356 | var taskCallback = function (err) { 357 | if (err) { 358 | callback(err); 359 | // stop subsequent errors hitting callback multiple times 360 | callback = function () {}; 361 | } 362 | else { 363 | completed.push(k); 364 | taskComplete(); 365 | } 366 | }; 367 | var requires = task.slice(0, Math.abs(task.length - 1)) || []; 368 | var ready = function () { 369 | return _reduce(requires, function (a, x) { 370 | return (a && _indexOf(completed, x) !== -1); 371 | }, true); 372 | }; 373 | if (ready()) { 374 | task[task.length - 1](taskCallback); 375 | } 376 | else { 377 | var listener = function () { 378 | if (ready()) { 379 | removeListener(listener); 380 | task[task.length - 1](taskCallback); 381 | } 382 | }; 383 | addListener(listener); 384 | } 385 | }); 386 | }; 387 | 388 | async.waterfall = function (tasks, callback) { 389 | if (!tasks.length) { 390 | return callback(); 391 | } 392 | callback = callback || function () {}; 393 | var wrapIterator = function (iterator) { 394 | return function (err) { 395 | if (err) { 396 | callback(err); 397 | callback = function () {}; 398 | } 399 | else { 400 | var args = Array.prototype.slice.call(arguments, 1); 401 | var next = iterator.next(); 402 | if (next) { 403 | args.push(wrapIterator(next)); 404 | } 405 | else { 406 | args.push(callback); 407 | } 408 | async.nextTick(function () { 409 | iterator.apply(null, args); 410 | }); 411 | } 412 | }; 413 | }; 414 | wrapIterator(async.iterator(tasks))(); 415 | }; 416 | 417 | async.parallel = function (tasks, callback) { 418 | callback = callback || function () {}; 419 | if (tasks.constructor === Array) { 420 | async.map(tasks, function (fn, callback) { 421 | if (fn) { 422 | fn(function (err) { 423 | var args = Array.prototype.slice.call(arguments, 1); 424 | if (args.length <= 1) { 425 | args = args[0]; 426 | } 427 | callback.call(null, err, args); 428 | }); 429 | } 430 | }, callback); 431 | } 432 | else { 433 | var results = {}; 434 | async.forEach(_keys(tasks), function (k, callback) { 435 | tasks[k](function (err) { 436 | var args = Array.prototype.slice.call(arguments, 1); 437 | if (args.length <= 1) { 438 | args = args[0]; 439 | } 440 | results[k] = args; 441 | callback(err); 442 | }); 443 | }, function (err) { 444 | callback(err, results); 445 | }); 446 | } 447 | }; 448 | 449 | async.series = function (tasks, callback) { 450 | callback = callback || function () {}; 451 | if (tasks.constructor === Array) { 452 | async.mapSeries(tasks, function (fn, callback) { 453 | if (fn) { 454 | fn(function (err) { 455 | var args = Array.prototype.slice.call(arguments, 1); 456 | if (args.length <= 1) { 457 | args = args[0]; 458 | } 459 | callback.call(null, err, args); 460 | }); 461 | } 462 | }, callback); 463 | } 464 | else { 465 | var results = {}; 466 | async.forEachSeries(_keys(tasks), function (k, callback) { 467 | tasks[k](function (err) { 468 | var args = Array.prototype.slice.call(arguments, 1); 469 | if (args.length <= 1) { 470 | args = args[0]; 471 | } 472 | results[k] = args; 473 | callback(err); 474 | }); 475 | }, function (err) { 476 | callback(err, results); 477 | }); 478 | } 479 | }; 480 | 481 | async.iterator = function (tasks) { 482 | var makeCallback = function (index) { 483 | var fn = function () { 484 | if (tasks.length) { 485 | tasks[index].apply(null, arguments); 486 | } 487 | return fn.next(); 488 | }; 489 | fn.next = function () { 490 | return (index < tasks.length - 1) ? makeCallback(index + 1): null; 491 | }; 492 | return fn; 493 | }; 494 | return makeCallback(0); 495 | }; 496 | 497 | async.apply = function (fn) { 498 | var args = Array.prototype.slice.call(arguments, 1); 499 | return function () { 500 | return fn.apply( 501 | null, args.concat(Array.prototype.slice.call(arguments)) 502 | ); 503 | }; 504 | }; 505 | 506 | var _concat = function (eachfn, arr, fn, callback) { 507 | var r = []; 508 | eachfn(arr, function (x, cb) { 509 | fn(x, function (err, y) { 510 | r = r.concat(y || []); 511 | cb(err); 512 | }); 513 | }, function (err) { 514 | callback(err, r); 515 | }); 516 | }; 517 | async.concat = doParallel(_concat); 518 | async.concatSeries = doSeries(_concat); 519 | 520 | async.whilst = function (test, iterator, callback) { 521 | if (test()) { 522 | iterator(function (err) { 523 | if (err) { 524 | return callback(err); 525 | } 526 | async.whilst(test, iterator, callback); 527 | }); 528 | } 529 | else { 530 | callback(); 531 | } 532 | }; 533 | 534 | async.until = function (test, iterator, callback) { 535 | if (!test()) { 536 | iterator(function (err) { 537 | if (err) { 538 | return callback(err); 539 | } 540 | async.until(test, iterator, callback); 541 | }); 542 | } 543 | else { 544 | callback(); 545 | } 546 | }; 547 | 548 | async.queue = function (worker, concurrency) { 549 | var workers = 0; 550 | var tasks = []; 551 | var q = { 552 | concurrency: concurrency, 553 | saturated: null, 554 | empty: null, 555 | drain: null, 556 | push: function (data, callback) { 557 | tasks.push({data: data, callback: callback}); 558 | if(q.saturated && tasks.length == concurrency) q.saturated(); 559 | async.nextTick(q.process); 560 | }, 561 | process: function () { 562 | if (workers < q.concurrency && tasks.length) { 563 | var task = tasks.shift(); 564 | if(q.empty && tasks.length == 0) q.empty(); 565 | workers += 1; 566 | worker(task.data, function () { 567 | workers -= 1; 568 | if (task.callback) { 569 | task.callback.apply(task, arguments); 570 | } 571 | if(q.drain && tasks.length + workers == 0) q.drain(); 572 | q.process(); 573 | }); 574 | } 575 | }, 576 | length: function () { 577 | return tasks.length; 578 | }, 579 | running: function () { 580 | return workers; 581 | } 582 | }; 583 | return q; 584 | }; 585 | 586 | var _console_fn = function (name) { 587 | return function (fn) { 588 | var args = Array.prototype.slice.call(arguments, 1); 589 | fn.apply(null, args.concat([function (err) { 590 | var args = Array.prototype.slice.call(arguments, 1); 591 | if (typeof console !== 'undefined') { 592 | if (err) { 593 | if (console.error) { 594 | console.error(err); 595 | } 596 | } 597 | else if (console[name]) { 598 | _forEach(args, function (x) { 599 | console[name](x); 600 | }); 601 | } 602 | } 603 | }])); 604 | }; 605 | }; 606 | async.log = _console_fn('log'); 607 | async.dir = _console_fn('dir'); 608 | /*async.info = _console_fn('info'); 609 | async.warn = _console_fn('warn'); 610 | async.error = _console_fn('error');*/ 611 | 612 | async.memoize = function (fn, hasher) { 613 | var memo = {}; 614 | hasher = hasher || function (x) { 615 | return x; 616 | }; 617 | return function () { 618 | var args = Array.prototype.slice.call(arguments); 619 | var callback = args.pop(); 620 | var key = hasher.apply(null, args); 621 | if (key in memo) { 622 | callback.apply(null, memo[key]); 623 | } 624 | else { 625 | fn.apply(null, args.concat([function () { 626 | memo[key] = arguments; 627 | callback.apply(null, arguments); 628 | }])); 629 | } 630 | }; 631 | }; 632 | 633 | }()); 634 | -------------------------------------------------------------------------------- /src/hoard.coffee: -------------------------------------------------------------------------------- 1 | fs = require 'fs' 2 | Buffer = require('buffer').Buffer 3 | Binary = require 'binary' 4 | underscore = _ = require '../lib/underscore' 5 | async = require '../lib/async' 6 | pack = require('../lib/jspack').jspack 7 | path = require 'path' 8 | Put = require 'put' 9 | 10 | # Monkey patch since modulo operator is broken in JS 11 | Number.prototype.mod = (n) -> ((@ % n) + n) % n 12 | 13 | longFormat = "!L" 14 | longSize = pack.CalcLength(longFormat) 15 | floatFormat = "!f" 16 | floatSize = pack.CalcLength(floatFormat) 17 | timestampFormat = "!L" 18 | timestampSize = pack.CalcLength(timestampFormat) 19 | valueFormat = "!d" 20 | valueSize = pack.CalcLength(valueFormat) 21 | pointFormat = "!Ld" 22 | pointSize = pack.CalcLength(pointFormat) 23 | metadataFormat = "!2LfL" 24 | metadataSize = pack.CalcLength(metadataFormat) 25 | archiveInfoFormat = "!3L" 26 | archiveInfoSize = pack.CalcLength(archiveInfoFormat) 27 | 28 | unixTime = -> parseInt(new Date().getTime() / 1000) 29 | 30 | create = (filename, archives, xFilesFactor, cb) -> 31 | # FIXME: Check parameters 32 | # FIXME: Check that values are correctly formatted 33 | archives.sort (a, b) -> a[0] - b[0] 34 | 35 | if path.existsSync(filename) 36 | cb new Error('File ' + filename + ' already exists') 37 | 38 | oldest = (a[0] * a[1] for a in archives).sort().reverse()[0] 39 | 40 | encodeFloat = (value) -> 41 | # Dirty hack. 42 | # Using 'buffer_ieee754' from node 0.5.x 43 | # as no libraries had a working IEEE754 encoder 44 | buffer = new Buffer(4) 45 | require('./buffer_ieee754').writeIEEE754(buffer, 0.5, 0, 'big', 23, 4); 46 | buffer 47 | 48 | buffer = Put() 49 | .word32be(unixTime()) # last update 50 | .word32be(oldest) # max retention 51 | .put(encodeFloat(xFilesFactor)) 52 | .word32be(archives.length) 53 | 54 | headerSize = metadataSize + (archiveInfoSize * archives.length) 55 | archiveOffset = headerSize 56 | 57 | for archive in archives 58 | secondsPerPoint = archive[0]; points = archive[1] 59 | buffer.word32be(archiveOffset) 60 | buffer.word32be(secondsPerPoint) 61 | buffer.word32be(points) 62 | archiveOffset += (points * pointSize) 63 | 64 | # Pad archive data itself with zeroes 65 | buffer.pad(archiveOffset - headerSize) 66 | 67 | # FIXME: Check file lock? 68 | # FIXME: fsync this? 69 | fs.writeFile filename, buffer.buffer(), 'binary', cb 70 | 71 | propagate = (fd, timestamp, xff, higher, lower, cb) -> 72 | lowerIntervalStart = timestamp - timestamp.mod(lower.secondsPerPoint) 73 | lowerIntervalEnd = lowerIntervalStart + lower.secondsPerPoint 74 | 75 | packedPoint = new Buffer(pointSize) 76 | fs.read fd, packedPoint, 0, pointSize, higher.offset, (err, written, buffer) -> 77 | cb(err) if err 78 | [higherBaseInterval, higherBaseValue] = pack.Unpack(pointFormat, packedPoint) 79 | 80 | if higherBaseInterval == 0 81 | higherFirstOffset = higher.offset 82 | else 83 | timeDistance = lowerIntervalStart - higherBaseInterval 84 | pointDistance = timeDistance / higher.secondsPerPoint 85 | byteDistance = pointDistance * pointSize 86 | higherFirstOffset = higher.offset + byteDistance.mod(higher.size) 87 | 88 | higherPoints = lower.secondsPerPoint / higher.secondsPerPoint 89 | higherSize = higherPoints * pointSize 90 | relativeFirstOffset = higherFirstOffset - higher.offset 91 | relativeLastOffset = (relativeFirstOffset + higherSize).mod(higher.size) 92 | higherLastOffset = relativeLastOffset + higher.offset 93 | 94 | if higherFirstOffset < higherLastOffset 95 | # We don't wrap the archive 96 | seriesSize = higherLastOffset - higherFirstOffset 97 | seriesString = new Buffer(seriesSize) 98 | 99 | fs.read fd, seriesString, 0, seriesSize, higherFirstOffset, (err, written, buffer) -> 100 | parseSeries(seriesString) 101 | else 102 | # We do wrap the archive 103 | higherEnd = higher.offset + higher.size 104 | firstSeriesSize = higherEnd - higherFirstOffset 105 | secondSeriesSize = higherLastOffset - higher.offset 106 | 107 | seriesString = new Buffer(firstSeriesSize + secondSeriesSize) 108 | 109 | fs.read fd, seriesString, 0, firstSeriesSize, higherFirstOffset, (err, written, buffer) -> 110 | cb(err) if err 111 | if secondSeriesSize > 0 112 | fs.read fd, seriesString, firstSeriesSize, secondSeriesSize, higher.offset, (err, written, buffer) -> 113 | cb(err) if err 114 | parseSeries(seriesString) 115 | else 116 | ret = new Buffer(firstSeriesSize) 117 | seriesString.copy(ret, 0, 0, firstSeriesSize) 118 | parseSeries(ret) 119 | 120 | parseSeries = (seriesString) -> 121 | # Now we unpack the series data we just read 122 | [byteOrder, pointTypes] = [pointFormat[0], pointFormat.slice(1)] 123 | points = seriesString.length / pointSize 124 | 125 | seriesFormat = byteOrder + (pointTypes for f in [0...points]).join("") 126 | unpackedSeries = pack.Unpack(seriesFormat, seriesString, 0) 127 | 128 | # And finally we construct a list of values 129 | neighborValues = (null for f in [0...points]) 130 | currentInterval = lowerIntervalStart 131 | step = higher.secondsPerPoint 132 | 133 | for i in [0...unpackedSeries.length] by 2 134 | pointTime = unpackedSeries[i] 135 | if pointTime == currentInterval 136 | neighborValues[i/2] = unpackedSeries[i+1] 137 | currentInterval += step 138 | 139 | 140 | 141 | # Propagate aggregateValue to propagate from neighborValues if we have enough known points 142 | knownValues = (v for v in neighborValues when v isnt null) 143 | if knownValues.length == 0 144 | cb null, false 145 | return 146 | 147 | sum = (list) -> 148 | s = 0 149 | for x in list 150 | s += x 151 | s 152 | 153 | knownPercent = knownValues.length / neighborValues.length 154 | if knownPercent >= xff 155 | # We have enough data to propagate a value! 156 | aggregateValue = sum(knownValues) / knownValues.length # TODO: Another CF besides average? 157 | myPackedPoint = pack.Pack(pointFormat, [lowerIntervalStart, aggregateValue]) 158 | 159 | # !!!!!!!!!!!!!!!!! 160 | packedPoint = new Buffer(pointSize) 161 | fs.read fd, packedPoint, 0, pointSize, lower.offset, (err) -> 162 | [lowerBaseInterval, lowerBaseValue] = pack.Unpack(pointFormat, packedPoint) 163 | 164 | if lowerBaseInterval == 0 165 | # First propagated update to this lower archive 166 | offset = lower.offset 167 | else 168 | # Not our first propagated update to this lower archive 169 | timeDistance = lowerIntervalStart - lowerBaseInterval 170 | pointDistance = timeDistance / lower.secondsPerPoint 171 | byteDistance = pointDistance * pointSize 172 | offset = lower.offset + byteDistance.mod(lower.size) 173 | 174 | mypp = new Buffer(myPackedPoint) 175 | fs.write fd, mypp, 0, pointSize, offset, (err) -> 176 | cb(null, true) 177 | else 178 | cb(null, false) 179 | 180 | 181 | update = (filename, value, timestamp, cb) -> 182 | # FIXME: Check file lock? 183 | # FIXME: Don't use info(), re-use fd between internal functions 184 | info filename, (err, header) -> 185 | cb(err) if err 186 | now = unixTime() 187 | diff = now - timestamp 188 | if not (diff < header.maxRetention and diff >= 0) 189 | cb(new Error('Timestamp not covered by any archives in this database.')) 190 | return 191 | 192 | # Find the highest-precision archive that covers timestamp 193 | for i in [0...header.archives.length] 194 | archive = header.archives[i] 195 | continue if archive.retention < diff 196 | # We'll pass on the update to these lower precision archives later 197 | lowerArchives = header.archives.slice(i + 1) 198 | break 199 | 200 | fs.open filename, 'r+', (err, fd) -> 201 | cb(err) if err 202 | # First we update the highest-precision archive 203 | myInterval = timestamp - timestamp.mod(archive.secondsPerPoint) 204 | myPackedPoint = new Buffer(pack.Pack(pointFormat, [myInterval, value])) 205 | 206 | packedPoint = new Buffer(pointSize) 207 | fs.read fd, packedPoint, 0, pointSize, archive.offset, (err, bytesRead, buffer) -> 208 | cb(err) if err 209 | [baseInterval, baseValue] = pack.Unpack(pointFormat, packedPoint) 210 | 211 | if baseInterval == 0 212 | # This file's first update 213 | fs.write fd, myPackedPoint, 0, pointSize, archive.offset, (err, written, buffer) -> 214 | cb(err) if err 215 | [baseInterval, baseValue] = [myInterval, value] 216 | propagateLowerArchives() 217 | else 218 | # File has been updated before 219 | timeDistance = myInterval - baseInterval 220 | pointDistance = timeDistance / archive.secondsPerPoint 221 | byteDistance = pointDistance * pointSize 222 | myOffset = archive.offset + byteDistance.mod(archive.size) 223 | fs.write fd, myPackedPoint, 0, pointSize, myOffset, (err, written, buffer) -> 224 | cb(err) if err 225 | propagateLowerArchives() 226 | 227 | propagateLowerArchives = -> 228 | # Propagate the update to lower-precision archives 229 | #higher = archive 230 | #for lower in lowerArchives: 231 | # if not __propagate(fd, myInterval, header.xFilesFactor, higher, lower): 232 | # break 233 | # higher = lower 234 | 235 | #__changeLastUpdate(fh) 236 | 237 | # FIXME: Also fsync here? 238 | fs.close fd, cb 239 | return 240 | 241 | updateMany = (filename, points, cb) -> 242 | points.sort((a, b) -> a[0] - b[0]).reverse() 243 | # FIXME: Check lock 244 | info filename, (err, header) -> 245 | cb err if err 246 | fs.open filename, 'r+', (err, fd) -> 247 | now = unixTime() 248 | archives = header.archives 249 | currentArchiveIndex = 0 250 | currentArchive = header.archives[currentArchiveIndex] 251 | currentPoints = [] 252 | 253 | updateArchiveCalls = [] 254 | for point in points 255 | age = now - point[0] 256 | 257 | while currentArchive.retention < age # We can't fit any more points in this archive 258 | if currentPoints 259 | # Commit all the points we've found that it can fit 260 | currentPoints.reverse() # Put points in chronological order 261 | do (header, currentArchive, currentPoints) -> 262 | f = (cb) -> updateManyArchive fd, header, currentArchive, currentPoints, cb 263 | updateArchiveCalls.push(f) 264 | currentPoints = [] 265 | 266 | if currentArchiveIndex < (archives.length - 1) 267 | currentArchiveIndex++ 268 | currentArchive = archives[currentArchiveIndex] 269 | else 270 | # Last archive 271 | currentArchive = null 272 | break 273 | 274 | if not currentArchive 275 | break # Drop remaining points that don't fit in the database 276 | 277 | currentPoints.push(point) 278 | 279 | async.series updateArchiveCalls, (err, results) -> 280 | throw err if err 281 | if currentArchive and currentPoints.length > 0 282 | # Don't forget to commit after we've checked all the archives 283 | currentPoints.reverse() 284 | updateManyArchive fd, header, currentArchive, currentPoints, (err) -> 285 | throw err if err 286 | fs.close fd, cb 287 | else 288 | fs.close fd, cb 289 | 290 | # FIXME: touch last update 291 | # FIXME: fsync here? 292 | # FIXME: close fd fh.close() 293 | #cb(null) 294 | 295 | updateManyArchive = (fd, header, archive, points, cb) -> 296 | step = archive.secondsPerPoint 297 | alignedPoints = [] 298 | for p in points 299 | [timestamp, value] = p 300 | alignedPoints.push([timestamp - timestamp.mod(step), value]) 301 | 302 | # Create a packed string for each contiguous sequence of points 303 | packedStrings = [] 304 | previousInterval = null 305 | currentString = [] 306 | 307 | for ap in alignedPoints 308 | [interval, value] = ap 309 | 310 | if !previousInterval or (interval == previousInterval + step) 311 | currentString.concat(pack.Pack(pointFormat, [interval, value])) 312 | previousInterval = interval 313 | else 314 | numberOfPoints = currentString.length / pointSize 315 | startInterval = previousInterval - (step * (numberOfPoints - 1)) 316 | packedStrings.push([startInterval, new Buffer(currentString)]) 317 | currentString = pack.Pack(pointFormat, [interval, value]) 318 | previousInterval = interval 319 | 320 | if currentString.length > 0 321 | numberOfPoints = currentString.length / pointSize 322 | startInterval = previousInterval - (step * (numberOfPoints - 1)) 323 | packedStrings.push([startInterval, new Buffer(currentString, 'binary')]) 324 | 325 | # Read base point and determine where our writes will start 326 | packedBasePoint = new Buffer(pointSize) 327 | fs.read fd, packedBasePoint, 0, pointSize, archive.offset, (err) -> 328 | cb err if err 329 | [baseInterval, baseValue] = pack.Unpack(pointFormat, packedBasePoint) 330 | 331 | if baseInterval == 0 332 | # This file's first update 333 | # Use our first string as the base, so we start at the start 334 | baseInterval = packedStrings[0][0] 335 | 336 | # Write all of our packed strings in locations determined by the baseInterval 337 | 338 | writePackedString = (ps, callback) -> 339 | [interval, packedString] = ps 340 | timeDistance = interval - baseInterval 341 | pointDistance = timeDistance / step 342 | byteDistance = pointDistance * pointSize 343 | myOffset = archive.offset + byteDistance.mod(archive.size) 344 | archiveEnd = archive.offset + archive.size 345 | bytesBeyond = (myOffset + packedString.length) - archiveEnd 346 | 347 | if bytesBeyond > 0 348 | fs.write fd, packedString, 0, packedString.length - bytesBeyond, myOffset, (err) -> 349 | cb err if err 350 | assert.equal archiveEnd, myOffset + packedString.length - bytesBeyond 351 | #assert fh.tell() == archiveEnd, "archiveEnd=%d fh.tell=%d bytesBeyond=%d len(packedString)=%d" % (archiveEnd,fh.tell(),bytesBeyond,len(packedString)) 352 | # Safe because it can't exceed the archive (retention checking logic above) 353 | fs.write fd, packedString, packedString.length - bytesBeyond, bytesBeyond, archive.offset, (err) -> 354 | cb err if err 355 | callback() 356 | else 357 | fs.write fd, packedString, 0, packedString.length, myOffset, (err) -> 358 | callback() 359 | 360 | async.forEachSeries packedStrings, writePackedString, (err) -> 361 | throw err if err 362 | propagateLowerArchives() 363 | 364 | propagateLowerArchives = -> 365 | # Now we propagate the updates to lower-precision archives 366 | higher = archive 367 | lowerArchives = (arc for arc in header.archives when arc.secondsPerPoint > archive.secondsPerPoint) 368 | 369 | if lowerArchives.length > 0 370 | # Collect a list of propagation calls to make 371 | # This is easier than doing async looping 372 | propagateCalls = [] 373 | for lower in lowerArchives 374 | fit = (i) -> i - i.mod(lower.secondsPerPoint) 375 | lowerIntervals = (fit(p[0]) for p in alignedPoints) 376 | uniqueLowerIntervals = _.uniq(lowerIntervals) 377 | for interval in uniqueLowerIntervals 378 | propagateCalls.push {interval: interval, header: header, higher: higher, lower: lower} 379 | higher = lower 380 | 381 | callPropagate = (args, callback) -> 382 | propagate fd, args.interval, args.header.xFilesFactor, args.higher, args.lower, (err, result) -> 383 | cb err if err 384 | callback err, result 385 | 386 | async.forEachSeries propagateCalls, callPropagate, (err, result) -> 387 | throw err if err 388 | cb null 389 | else 390 | cb null 391 | 392 | info = (path, cb) -> 393 | # FIXME: Close this stream? 394 | # FIXME: Signal errors to callback? 395 | 396 | # FIXME: Stream parsing with node-binary dies 397 | # Looks like an issue, see their GitHub 398 | # Using fs.readFile() instead of read stream for now 399 | fs.readFile path, (err, data) -> 400 | cb err if err 401 | archives = []; metadata = {} 402 | 403 | Binary.parse(data) 404 | .word32bu('lastUpdate') 405 | .word32bu('maxRetention') 406 | .buffer('xff', 4) # Must decode separately since node-binary can't handle floats 407 | .word32bu('archiveCount') 408 | .tap (vars) -> 409 | metadata = vars 410 | metadata.xff = pack.Unpack('!f', vars.xff, 0)[0] 411 | @flush() 412 | for index in [0...metadata.archiveCount] 413 | @word32bu('offset').word32bu('secondsPerPoint').word32bu('points') 414 | @tap (archive) -> 415 | @flush() 416 | archive.retention = archive.secondsPerPoint * archive.points 417 | archive.size = archive.points * pointSize 418 | archives.push(archive) 419 | .tap -> 420 | cb null, 421 | maxRetention: metadata.maxRetention 422 | xFilesFactor: metadata.xff 423 | archives: archives 424 | return 425 | 426 | fetch = (path, from, to, cb) -> 427 | info path, (err, header) -> 428 | now = unixTime() 429 | oldestTime = now - header.maxRetention 430 | from = oldestTime if from < oldestTime 431 | throw new Error('Invalid time interval') unless from < to 432 | to = now if to > now or to < from 433 | diff = now - from 434 | fd = null 435 | 436 | # Find closest archive to look in, that iwll contain our information 437 | for archive in header.archives 438 | break if archive.retention >= diff 439 | 440 | fromInterval = parseInt(from - from.mod(archive.secondsPerPoint)) + archive.secondsPerPoint 441 | toInterval = parseInt(to - to.mod(archive.secondsPerPoint)) + archive.secondsPerPoint 442 | 443 | file = fs.createReadStream(path) 444 | 445 | Binary.stream(file) 446 | .skip(archive.offset) 447 | .word32bu('baseInterval') 448 | .word32bu('baseValue') 449 | .tap (vars) -> 450 | if vars.baseInterval == 0 451 | # Nothing has been written to this hoard 452 | step = archive.secondsPerPoint 453 | points = (toInterval - fromInterval) / step 454 | timeInfo = [fromInterval, toInterval, step] 455 | values = (null for n in [0...points]) 456 | cb(null, timeInfo, values) 457 | else 458 | # We have data in this hoard, let's read it 459 | getOffset = (interval) -> 460 | timeDistance = interval - vars.baseInterval 461 | pointDistance = timeDistance / archive.secondsPerPoint 462 | byteDistance = pointDistance * pointSize 463 | a = archive.offset + byteDistance.mod(archive.size) 464 | a 465 | 466 | fromOffset = getOffset(fromInterval) 467 | toOffset = getOffset(toInterval) 468 | 469 | fs.open path, 'r', (err, fd) -> 470 | if err then throw err 471 | if fromOffset < toOffset 472 | # We don't wrap around, can everything in a single read 473 | size = toOffset - fromOffset 474 | seriesBuffer = new Buffer(size) 475 | fs.read fd, seriesBuffer, 0, size, fromOffset, (err, num) -> 476 | cb(err) if err 477 | fs.close fd, (err) -> 478 | cb(err) if err 479 | unpack(seriesBuffer) # We have read it, go unpack! 480 | else 481 | # We wrap around the archive, we need two reads 482 | archiveEnd = archive.offset + archive.size 483 | size1 = archiveEnd - fromOffset 484 | size2 = toOffset - archive.offset 485 | seriesBuffer = new Buffer(size1 + size2) 486 | fs.read fd, seriesBuffer, 0, size1, fromOffset, (err, num) -> 487 | cb(err) if err 488 | fs.read fd, seriesBuffer, size1, size2, archive.offset, (err, num) -> 489 | cb(err) if err 490 | unpack(seriesBuffer) # We have read it, go unpack! 491 | fs.close(fd) 492 | 493 | unpack = (seriesData) -> 494 | # Optmize this? 495 | numPoints = seriesData.length / pointSize 496 | seriesFormat = "!" + ('Ld' for f in [0...numPoints]).join("") 497 | unpackedSeries = pack.Unpack(seriesFormat, seriesData) 498 | 499 | # Use buffer/pre-allocate? 500 | valueList = (null for f in [0...numPoints]) 501 | currentInterval = fromInterval 502 | step = archive.secondsPerPoint 503 | 504 | for i in [0...unpackedSeries.length] by 2 505 | pointTime = unpackedSeries[i] 506 | if pointTime == currentInterval 507 | pointValue = unpackedSeries[i + 1] 508 | valueList[i / 2] = pointValue 509 | currentInterval += step 510 | 511 | timeInfo = [fromInterval, toInterval, step] 512 | cb(null, timeInfo, valueList) 513 | return 514 | 515 | exports.create = create 516 | exports.update = update 517 | exports.updateMany = updateMany 518 | exports.info = info 519 | exports.fetch = fetch 520 | 521 | -------------------------------------------------------------------------------- /lib/underscore.js: -------------------------------------------------------------------------------- 1 | // Underscore.js 1.1.7 2 | // (c) 2011 Jeremy Ashkenas, DocumentCloud Inc. 3 | // Underscore is freely distributable under the MIT license. 4 | // Portions of Underscore are inspired or borrowed from Prototype, 5 | // Oliver Steele's Functional, and John Resig's Micro-Templating. 6 | // For all details and documentation: 7 | // http://documentcloud.github.com/underscore 8 | 9 | (function() { 10 | 11 | // Baseline setup 12 | // -------------- 13 | 14 | // Establish the root object, `window` in the browser, or `global` on the server. 15 | var root = this; 16 | 17 | // Save the previous value of the `_` variable. 18 | var previousUnderscore = root._; 19 | 20 | // Establish the object that gets returned to break out of a loop iteration. 21 | var breaker = {}; 22 | 23 | // Save bytes in the minified (but not gzipped) version: 24 | var ArrayProto = Array.prototype, ObjProto = Object.prototype, FuncProto = Function.prototype; 25 | 26 | // Create quick reference variables for speed access to core prototypes. 27 | var slice = ArrayProto.slice, 28 | unshift = ArrayProto.unshift, 29 | toString = ObjProto.toString, 30 | hasOwnProperty = ObjProto.hasOwnProperty; 31 | 32 | // All **ECMAScript 5** native function implementations that we hope to use 33 | // are declared here. 34 | var 35 | nativeForEach = ArrayProto.forEach, 36 | nativeMap = ArrayProto.map, 37 | nativeReduce = ArrayProto.reduce, 38 | nativeReduceRight = ArrayProto.reduceRight, 39 | nativeFilter = ArrayProto.filter, 40 | nativeEvery = ArrayProto.every, 41 | nativeSome = ArrayProto.some, 42 | nativeIndexOf = ArrayProto.indexOf, 43 | nativeLastIndexOf = ArrayProto.lastIndexOf, 44 | nativeIsArray = Array.isArray, 45 | nativeKeys = Object.keys, 46 | nativeBind = FuncProto.bind; 47 | 48 | // Create a safe reference to the Underscore object for use below. 49 | var _ = function(obj) { return new wrapper(obj); }; 50 | 51 | // Export the Underscore object for **CommonJS**, with backwards-compatibility 52 | // for the old `require()` API. If we're not in CommonJS, add `_` to the 53 | // global object. 54 | if (typeof module !== 'undefined' && module.exports) { 55 | module.exports = _; 56 | _._ = _; 57 | } else { 58 | // Exported as a string, for Closure Compiler "advanced" mode. 59 | root['_'] = _; 60 | } 61 | 62 | // Current version. 63 | _.VERSION = '1.1.7'; 64 | 65 | // Collection Functions 66 | // -------------------- 67 | 68 | // The cornerstone, an `each` implementation, aka `forEach`. 69 | // Handles objects with the built-in `forEach`, arrays, and raw objects. 70 | // Delegates to **ECMAScript 5**'s native `forEach` if available. 71 | var each = _.each = _.forEach = function(obj, iterator, context) { 72 | if (obj == null) return; 73 | if (nativeForEach && obj.forEach === nativeForEach) { 74 | obj.forEach(iterator, context); 75 | } else if (obj.length === +obj.length) { 76 | for (var i = 0, l = obj.length; i < l; i++) { 77 | if (i in obj && iterator.call(context, obj[i], i, obj) === breaker) return; 78 | } 79 | } else { 80 | for (var key in obj) { 81 | if (hasOwnProperty.call(obj, key)) { 82 | if (iterator.call(context, obj[key], key, obj) === breaker) return; 83 | } 84 | } 85 | } 86 | }; 87 | 88 | // Return the results of applying the iterator to each element. 89 | // Delegates to **ECMAScript 5**'s native `map` if available. 90 | _.map = function(obj, iterator, context) { 91 | var results = []; 92 | if (obj == null) return results; 93 | if (nativeMap && obj.map === nativeMap) return obj.map(iterator, context); 94 | each(obj, function(value, index, list) { 95 | results[results.length] = iterator.call(context, value, index, list); 96 | }); 97 | return results; 98 | }; 99 | 100 | // **Reduce** builds up a single result from a list of values, aka `inject`, 101 | // or `foldl`. Delegates to **ECMAScript 5**'s native `reduce` if available. 102 | _.reduce = _.foldl = _.inject = function(obj, iterator, memo, context) { 103 | var initial = memo !== void 0; 104 | if (obj == null) obj = []; 105 | if (nativeReduce && obj.reduce === nativeReduce) { 106 | if (context) iterator = _.bind(iterator, context); 107 | return initial ? obj.reduce(iterator, memo) : obj.reduce(iterator); 108 | } 109 | each(obj, function(value, index, list) { 110 | if (!initial) { 111 | memo = value; 112 | initial = true; 113 | } else { 114 | memo = iterator.call(context, memo, value, index, list); 115 | } 116 | }); 117 | if (!initial) throw new TypeError("Reduce of empty array with no initial value"); 118 | return memo; 119 | }; 120 | 121 | // The right-associative version of reduce, also known as `foldr`. 122 | // Delegates to **ECMAScript 5**'s native `reduceRight` if available. 123 | _.reduceRight = _.foldr = function(obj, iterator, memo, context) { 124 | if (obj == null) obj = []; 125 | if (nativeReduceRight && obj.reduceRight === nativeReduceRight) { 126 | if (context) iterator = _.bind(iterator, context); 127 | return memo !== void 0 ? obj.reduceRight(iterator, memo) : obj.reduceRight(iterator); 128 | } 129 | var reversed = (_.isArray(obj) ? obj.slice() : _.toArray(obj)).reverse(); 130 | return _.reduce(reversed, iterator, memo, context); 131 | }; 132 | 133 | // Return the first value which passes a truth test. Aliased as `detect`. 134 | _.find = _.detect = function(obj, iterator, context) { 135 | var result; 136 | any(obj, function(value, index, list) { 137 | if (iterator.call(context, value, index, list)) { 138 | result = value; 139 | return true; 140 | } 141 | }); 142 | return result; 143 | }; 144 | 145 | // Return all the elements that pass a truth test. 146 | // Delegates to **ECMAScript 5**'s native `filter` if available. 147 | // Aliased as `select`. 148 | _.filter = _.select = function(obj, iterator, context) { 149 | var results = []; 150 | if (obj == null) return results; 151 | if (nativeFilter && obj.filter === nativeFilter) return obj.filter(iterator, context); 152 | each(obj, function(value, index, list) { 153 | if (iterator.call(context, value, index, list)) results[results.length] = value; 154 | }); 155 | return results; 156 | }; 157 | 158 | // Return all the elements for which a truth test fails. 159 | _.reject = function(obj, iterator, context) { 160 | var results = []; 161 | if (obj == null) return results; 162 | each(obj, function(value, index, list) { 163 | if (!iterator.call(context, value, index, list)) results[results.length] = value; 164 | }); 165 | return results; 166 | }; 167 | 168 | // Determine whether all of the elements match a truth test. 169 | // Delegates to **ECMAScript 5**'s native `every` if available. 170 | // Aliased as `all`. 171 | _.every = _.all = function(obj, iterator, context) { 172 | var result = true; 173 | if (obj == null) return result; 174 | if (nativeEvery && obj.every === nativeEvery) return obj.every(iterator, context); 175 | each(obj, function(value, index, list) { 176 | if (!(result = result && iterator.call(context, value, index, list))) return breaker; 177 | }); 178 | return result; 179 | }; 180 | 181 | // Determine if at least one element in the object matches a truth test. 182 | // Delegates to **ECMAScript 5**'s native `some` if available. 183 | // Aliased as `any`. 184 | var any = _.some = _.any = function(obj, iterator, context) { 185 | iterator = iterator || _.identity; 186 | var result = false; 187 | if (obj == null) return result; 188 | if (nativeSome && obj.some === nativeSome) return obj.some(iterator, context); 189 | each(obj, function(value, index, list) { 190 | if (result |= iterator.call(context, value, index, list)) return breaker; 191 | }); 192 | return !!result; 193 | }; 194 | 195 | // Determine if a given value is included in the array or object using `===`. 196 | // Aliased as `contains`. 197 | _.include = _.contains = function(obj, target) { 198 | var found = false; 199 | if (obj == null) return found; 200 | if (nativeIndexOf && obj.indexOf === nativeIndexOf) return obj.indexOf(target) != -1; 201 | any(obj, function(value) { 202 | if (found = value === target) return true; 203 | }); 204 | return found; 205 | }; 206 | 207 | // Invoke a method (with arguments) on every item in a collection. 208 | _.invoke = function(obj, method) { 209 | var args = slice.call(arguments, 2); 210 | return _.map(obj, function(value) { 211 | return (method.call ? method || value : value[method]).apply(value, args); 212 | }); 213 | }; 214 | 215 | // Convenience version of a common use case of `map`: fetching a property. 216 | _.pluck = function(obj, key) { 217 | return _.map(obj, function(value){ return value[key]; }); 218 | }; 219 | 220 | // Return the maximum element or (element-based computation). 221 | _.max = function(obj, iterator, context) { 222 | if (!iterator && _.isArray(obj)) return Math.max.apply(Math, obj); 223 | var result = {computed : -Infinity}; 224 | each(obj, function(value, index, list) { 225 | var computed = iterator ? iterator.call(context, value, index, list) : value; 226 | computed >= result.computed && (result = {value : value, computed : computed}); 227 | }); 228 | return result.value; 229 | }; 230 | 231 | // Return the minimum element (or element-based computation). 232 | _.min = function(obj, iterator, context) { 233 | if (!iterator && _.isArray(obj)) return Math.min.apply(Math, obj); 234 | var result = {computed : Infinity}; 235 | each(obj, function(value, index, list) { 236 | var computed = iterator ? iterator.call(context, value, index, list) : value; 237 | computed < result.computed && (result = {value : value, computed : computed}); 238 | }); 239 | return result.value; 240 | }; 241 | 242 | // Sort the object's values by a criterion produced by an iterator. 243 | _.sortBy = function(obj, iterator, context) { 244 | return _.pluck(_.map(obj, function(value, index, list) { 245 | return { 246 | value : value, 247 | criteria : iterator.call(context, value, index, list) 248 | }; 249 | }).sort(function(left, right) { 250 | var a = left.criteria, b = right.criteria; 251 | return a < b ? -1 : a > b ? 1 : 0; 252 | }), 'value'); 253 | }; 254 | 255 | // Groups the object's values by a criterion produced by an iterator 256 | _.groupBy = function(obj, iterator) { 257 | var result = {}; 258 | each(obj, function(value, index) { 259 | var key = iterator(value, index); 260 | (result[key] || (result[key] = [])).push(value); 261 | }); 262 | return result; 263 | }; 264 | 265 | // Use a comparator function to figure out at what index an object should 266 | // be inserted so as to maintain order. Uses binary search. 267 | _.sortedIndex = function(array, obj, iterator) { 268 | iterator || (iterator = _.identity); 269 | var low = 0, high = array.length; 270 | while (low < high) { 271 | var mid = (low + high) >> 1; 272 | iterator(array[mid]) < iterator(obj) ? low = mid + 1 : high = mid; 273 | } 274 | return low; 275 | }; 276 | 277 | // Safely convert anything iterable into a real, live array. 278 | _.toArray = function(iterable) { 279 | if (!iterable) return []; 280 | if (iterable.toArray) return iterable.toArray(); 281 | if (_.isArray(iterable)) return slice.call(iterable); 282 | if (_.isArguments(iterable)) return slice.call(iterable); 283 | return _.values(iterable); 284 | }; 285 | 286 | // Return the number of elements in an object. 287 | _.size = function(obj) { 288 | return _.toArray(obj).length; 289 | }; 290 | 291 | // Array Functions 292 | // --------------- 293 | 294 | // Get the first element of an array. Passing **n** will return the first N 295 | // values in the array. Aliased as `head`. The **guard** check allows it to work 296 | // with `_.map`. 297 | _.first = _.head = function(array, n, guard) { 298 | return (n != null) && !guard ? slice.call(array, 0, n) : array[0]; 299 | }; 300 | 301 | // Returns everything but the first entry of the array. Aliased as `tail`. 302 | // Especially useful on the arguments object. Passing an **index** will return 303 | // the rest of the values in the array from that index onward. The **guard** 304 | // check allows it to work with `_.map`. 305 | _.rest = _.tail = function(array, index, guard) { 306 | return slice.call(array, (index == null) || guard ? 1 : index); 307 | }; 308 | 309 | // Get the last element of an array. 310 | _.last = function(array) { 311 | return array[array.length - 1]; 312 | }; 313 | 314 | // Trim out all falsy values from an array. 315 | _.compact = function(array) { 316 | return _.filter(array, function(value){ return !!value; }); 317 | }; 318 | 319 | // Return a completely flattened version of an array. 320 | _.flatten = function(array) { 321 | return _.reduce(array, function(memo, value) { 322 | if (_.isArray(value)) return memo.concat(_.flatten(value)); 323 | memo[memo.length] = value; 324 | return memo; 325 | }, []); 326 | }; 327 | 328 | // Return a version of the array that does not contain the specified value(s). 329 | _.without = function(array) { 330 | return _.difference(array, slice.call(arguments, 1)); 331 | }; 332 | 333 | // Produce a duplicate-free version of the array. If the array has already 334 | // been sorted, you have the option of using a faster algorithm. 335 | // Aliased as `unique`. 336 | _.uniq = _.unique = function(array, isSorted) { 337 | return _.reduce(array, function(memo, el, i) { 338 | if (0 == i || (isSorted === true ? _.last(memo) != el : !_.include(memo, el))) memo[memo.length] = el; 339 | return memo; 340 | }, []); 341 | }; 342 | 343 | // Produce an array that contains the union: each distinct element from all of 344 | // the passed-in arrays. 345 | _.union = function() { 346 | return _.uniq(_.flatten(arguments)); 347 | }; 348 | 349 | // Produce an array that contains every item shared between all the 350 | // passed-in arrays. (Aliased as "intersect" for back-compat.) 351 | _.intersection = _.intersect = function(array) { 352 | var rest = slice.call(arguments, 1); 353 | return _.filter(_.uniq(array), function(item) { 354 | return _.every(rest, function(other) { 355 | return _.indexOf(other, item) >= 0; 356 | }); 357 | }); 358 | }; 359 | 360 | // Take the difference between one array and another. 361 | // Only the elements present in just the first array will remain. 362 | _.difference = function(array, other) { 363 | return _.filter(array, function(value){ return !_.include(other, value); }); 364 | }; 365 | 366 | // Zip together multiple lists into a single array -- elements that share 367 | // an index go together. 368 | _.zip = function() { 369 | var args = slice.call(arguments); 370 | var length = _.max(_.pluck(args, 'length')); 371 | var results = new Array(length); 372 | for (var i = 0; i < length; i++) results[i] = _.pluck(args, "" + i); 373 | return results; 374 | }; 375 | 376 | // If the browser doesn't supply us with indexOf (I'm looking at you, **MSIE**), 377 | // we need this function. Return the position of the first occurrence of an 378 | // item in an array, or -1 if the item is not included in the array. 379 | // Delegates to **ECMAScript 5**'s native `indexOf` if available. 380 | // If the array is large and already in sort order, pass `true` 381 | // for **isSorted** to use binary search. 382 | _.indexOf = function(array, item, isSorted) { 383 | if (array == null) return -1; 384 | var i, l; 385 | if (isSorted) { 386 | i = _.sortedIndex(array, item); 387 | return array[i] === item ? i : -1; 388 | } 389 | if (nativeIndexOf && array.indexOf === nativeIndexOf) return array.indexOf(item); 390 | for (i = 0, l = array.length; i < l; i++) if (array[i] === item) return i; 391 | return -1; 392 | }; 393 | 394 | 395 | // Delegates to **ECMAScript 5**'s native `lastIndexOf` if available. 396 | _.lastIndexOf = function(array, item) { 397 | if (array == null) return -1; 398 | if (nativeLastIndexOf && array.lastIndexOf === nativeLastIndexOf) return array.lastIndexOf(item); 399 | var i = array.length; 400 | while (i--) if (array[i] === item) return i; 401 | return -1; 402 | }; 403 | 404 | // Generate an integer Array containing an arithmetic progression. A port of 405 | // the native Python `range()` function. See 406 | // [the Python documentation](http://docs.python.org/library/functions.html#range). 407 | _.range = function(start, stop, step) { 408 | if (arguments.length <= 1) { 409 | stop = start || 0; 410 | start = 0; 411 | } 412 | step = arguments[2] || 1; 413 | 414 | var len = Math.max(Math.ceil((stop - start) / step), 0); 415 | var idx = 0; 416 | var range = new Array(len); 417 | 418 | while(idx < len) { 419 | range[idx++] = start; 420 | start += step; 421 | } 422 | 423 | return range; 424 | }; 425 | 426 | // Function (ahem) Functions 427 | // ------------------ 428 | 429 | // Create a function bound to a given object (assigning `this`, and arguments, 430 | // optionally). Binding with arguments is also known as `curry`. 431 | // Delegates to **ECMAScript 5**'s native `Function.bind` if available. 432 | // We check for `func.bind` first, to fail fast when `func` is undefined. 433 | _.bind = function(func, obj) { 434 | if (func.bind === nativeBind && nativeBind) return nativeBind.apply(func, slice.call(arguments, 1)); 435 | var args = slice.call(arguments, 2); 436 | return function() { 437 | return func.apply(obj, args.concat(slice.call(arguments))); 438 | }; 439 | }; 440 | 441 | // Bind all of an object's methods to that object. Useful for ensuring that 442 | // all callbacks defined on an object belong to it. 443 | _.bindAll = function(obj) { 444 | var funcs = slice.call(arguments, 1); 445 | if (funcs.length == 0) funcs = _.functions(obj); 446 | each(funcs, function(f) { obj[f] = _.bind(obj[f], obj); }); 447 | return obj; 448 | }; 449 | 450 | // Memoize an expensive function by storing its results. 451 | _.memoize = function(func, hasher) { 452 | var memo = {}; 453 | hasher || (hasher = _.identity); 454 | return function() { 455 | var key = hasher.apply(this, arguments); 456 | return hasOwnProperty.call(memo, key) ? memo[key] : (memo[key] = func.apply(this, arguments)); 457 | }; 458 | }; 459 | 460 | // Delays a function for the given number of milliseconds, and then calls 461 | // it with the arguments supplied. 462 | _.delay = function(func, wait) { 463 | var args = slice.call(arguments, 2); 464 | return setTimeout(function(){ return func.apply(func, args); }, wait); 465 | }; 466 | 467 | // Defers a function, scheduling it to run after the current call stack has 468 | // cleared. 469 | _.defer = function(func) { 470 | return _.delay.apply(_, [func, 1].concat(slice.call(arguments, 1))); 471 | }; 472 | 473 | // Internal function used to implement `_.throttle` and `_.debounce`. 474 | var limit = function(func, wait, debounce) { 475 | var timeout; 476 | return function() { 477 | var context = this, args = arguments; 478 | var throttler = function() { 479 | timeout = null; 480 | func.apply(context, args); 481 | }; 482 | if (debounce) clearTimeout(timeout); 483 | if (debounce || !timeout) timeout = setTimeout(throttler, wait); 484 | }; 485 | }; 486 | 487 | // Returns a function, that, when invoked, will only be triggered at most once 488 | // during a given window of time. 489 | _.throttle = function(func, wait) { 490 | return limit(func, wait, false); 491 | }; 492 | 493 | // Returns a function, that, as long as it continues to be invoked, will not 494 | // be triggered. The function will be called after it stops being called for 495 | // N milliseconds. 496 | _.debounce = function(func, wait) { 497 | return limit(func, wait, true); 498 | }; 499 | 500 | // Returns a function that will be executed at most one time, no matter how 501 | // often you call it. Useful for lazy initialization. 502 | _.once = function(func) { 503 | var ran = false, memo; 504 | return function() { 505 | if (ran) return memo; 506 | ran = true; 507 | return memo = func.apply(this, arguments); 508 | }; 509 | }; 510 | 511 | // Returns the first function passed as an argument to the second, 512 | // allowing you to adjust arguments, run code before and after, and 513 | // conditionally execute the original function. 514 | _.wrap = function(func, wrapper) { 515 | return function() { 516 | var args = [func].concat(slice.call(arguments)); 517 | return wrapper.apply(this, args); 518 | }; 519 | }; 520 | 521 | // Returns a function that is the composition of a list of functions, each 522 | // consuming the return value of the function that follows. 523 | _.compose = function() { 524 | var funcs = slice.call(arguments); 525 | return function() { 526 | var args = slice.call(arguments); 527 | for (var i = funcs.length - 1; i >= 0; i--) { 528 | args = [funcs[i].apply(this, args)]; 529 | } 530 | return args[0]; 531 | }; 532 | }; 533 | 534 | // Returns a function that will only be executed after being called N times. 535 | _.after = function(times, func) { 536 | return function() { 537 | if (--times < 1) { return func.apply(this, arguments); } 538 | }; 539 | }; 540 | 541 | 542 | // Object Functions 543 | // ---------------- 544 | 545 | // Retrieve the names of an object's properties. 546 | // Delegates to **ECMAScript 5**'s native `Object.keys` 547 | _.keys = nativeKeys || function(obj) { 548 | if (obj !== Object(obj)) throw new TypeError('Invalid object'); 549 | var keys = []; 550 | for (var key in obj) if (hasOwnProperty.call(obj, key)) keys[keys.length] = key; 551 | return keys; 552 | }; 553 | 554 | // Retrieve the values of an object's properties. 555 | _.values = function(obj) { 556 | return _.map(obj, _.identity); 557 | }; 558 | 559 | // Return a sorted list of the function names available on the object. 560 | // Aliased as `methods` 561 | _.functions = _.methods = function(obj) { 562 | var names = []; 563 | for (var key in obj) { 564 | if (_.isFunction(obj[key])) names.push(key); 565 | } 566 | return names.sort(); 567 | }; 568 | 569 | // Extend a given object with all the properties in passed-in object(s). 570 | _.extend = function(obj) { 571 | each(slice.call(arguments, 1), function(source) { 572 | for (var prop in source) { 573 | if (source[prop] !== void 0) obj[prop] = source[prop]; 574 | } 575 | }); 576 | return obj; 577 | }; 578 | 579 | // Fill in a given object with default properties. 580 | _.defaults = function(obj) { 581 | each(slice.call(arguments, 1), function(source) { 582 | for (var prop in source) { 583 | if (obj[prop] == null) obj[prop] = source[prop]; 584 | } 585 | }); 586 | return obj; 587 | }; 588 | 589 | // Create a (shallow-cloned) duplicate of an object. 590 | _.clone = function(obj) { 591 | return _.isArray(obj) ? obj.slice() : _.extend({}, obj); 592 | }; 593 | 594 | // Invokes interceptor with the obj, and then returns obj. 595 | // The primary purpose of this method is to "tap into" a method chain, in 596 | // order to perform operations on intermediate results within the chain. 597 | _.tap = function(obj, interceptor) { 598 | interceptor(obj); 599 | return obj; 600 | }; 601 | 602 | // Perform a deep comparison to check if two objects are equal. 603 | _.isEqual = function(a, b) { 604 | // Check object identity. 605 | if (a === b) return true; 606 | // Different types? 607 | var atype = typeof(a), btype = typeof(b); 608 | if (atype != btype) return false; 609 | // Basic equality test (watch out for coercions). 610 | if (a == b) return true; 611 | // One is falsy and the other truthy. 612 | if ((!a && b) || (a && !b)) return false; 613 | // Unwrap any wrapped objects. 614 | if (a._chain) a = a._wrapped; 615 | if (b._chain) b = b._wrapped; 616 | // One of them implements an isEqual()? 617 | if (a.isEqual) return a.isEqual(b); 618 | if (b.isEqual) return b.isEqual(a); 619 | // Check dates' integer values. 620 | if (_.isDate(a) && _.isDate(b)) return a.getTime() === b.getTime(); 621 | // Both are NaN? 622 | if (_.isNaN(a) && _.isNaN(b)) return false; 623 | // Compare regular expressions. 624 | if (_.isRegExp(a) && _.isRegExp(b)) 625 | return a.source === b.source && 626 | a.global === b.global && 627 | a.ignoreCase === b.ignoreCase && 628 | a.multiline === b.multiline; 629 | // If a is not an object by this point, we can't handle it. 630 | if (atype !== 'object') return false; 631 | // Check for different array lengths before comparing contents. 632 | if (a.length && (a.length !== b.length)) return false; 633 | // Nothing else worked, deep compare the contents. 634 | var aKeys = _.keys(a), bKeys = _.keys(b); 635 | // Different object sizes? 636 | if (aKeys.length != bKeys.length) return false; 637 | // Recursive comparison of contents. 638 | for (var key in a) if (!(key in b) || !_.isEqual(a[key], b[key])) return false; 639 | return true; 640 | }; 641 | 642 | // Is a given array or object empty? 643 | _.isEmpty = function(obj) { 644 | if (_.isArray(obj) || _.isString(obj)) return obj.length === 0; 645 | for (var key in obj) if (hasOwnProperty.call(obj, key)) return false; 646 | return true; 647 | }; 648 | 649 | // Is a given value a DOM element? 650 | _.isElement = function(obj) { 651 | return !!(obj && obj.nodeType == 1); 652 | }; 653 | 654 | // Is a given value an array? 655 | // Delegates to ECMA5's native Array.isArray 656 | _.isArray = nativeIsArray || function(obj) { 657 | return toString.call(obj) === '[object Array]'; 658 | }; 659 | 660 | // Is a given variable an object? 661 | _.isObject = function(obj) { 662 | return obj === Object(obj); 663 | }; 664 | 665 | // Is a given variable an arguments object? 666 | _.isArguments = function(obj) { 667 | return !!(obj && hasOwnProperty.call(obj, 'callee')); 668 | }; 669 | 670 | // Is a given value a function? 671 | _.isFunction = function(obj) { 672 | return !!(obj && obj.constructor && obj.call && obj.apply); 673 | }; 674 | 675 | // Is a given value a string? 676 | _.isString = function(obj) { 677 | return !!(obj === '' || (obj && obj.charCodeAt && obj.substr)); 678 | }; 679 | 680 | // Is a given value a number? 681 | _.isNumber = function(obj) { 682 | return !!(obj === 0 || (obj && obj.toExponential && obj.toFixed)); 683 | }; 684 | 685 | // Is the given value `NaN`? `NaN` happens to be the only value in JavaScript 686 | // that does not equal itself. 687 | _.isNaN = function(obj) { 688 | return obj !== obj; 689 | }; 690 | 691 | // Is a given value a boolean? 692 | _.isBoolean = function(obj) { 693 | return obj === true || obj === false; 694 | }; 695 | 696 | // Is a given value a date? 697 | _.isDate = function(obj) { 698 | return !!(obj && obj.getTimezoneOffset && obj.setUTCFullYear); 699 | }; 700 | 701 | // Is the given value a regular expression? 702 | _.isRegExp = function(obj) { 703 | return !!(obj && obj.test && obj.exec && (obj.ignoreCase || obj.ignoreCase === false)); 704 | }; 705 | 706 | // Is a given value equal to null? 707 | _.isNull = function(obj) { 708 | return obj === null; 709 | }; 710 | 711 | // Is a given variable undefined? 712 | _.isUndefined = function(obj) { 713 | return obj === void 0; 714 | }; 715 | 716 | // Utility Functions 717 | // ----------------- 718 | 719 | // Run Underscore.js in *noConflict* mode, returning the `_` variable to its 720 | // previous owner. Returns a reference to the Underscore object. 721 | _.noConflict = function() { 722 | root._ = previousUnderscore; 723 | return this; 724 | }; 725 | 726 | // Keep the identity function around for default iterators. 727 | _.identity = function(value) { 728 | return value; 729 | }; 730 | 731 | // Run a function **n** times. 732 | _.times = function (n, iterator, context) { 733 | for (var i = 0; i < n; i++) iterator.call(context, i); 734 | }; 735 | 736 | // Add your own custom functions to the Underscore object, ensuring that 737 | // they're correctly added to the OOP wrapper as well. 738 | _.mixin = function(obj) { 739 | each(_.functions(obj), function(name){ 740 | addToWrapper(name, _[name] = obj[name]); 741 | }); 742 | }; 743 | 744 | // Generate a unique integer id (unique within the entire client session). 745 | // Useful for temporary DOM ids. 746 | var idCounter = 0; 747 | _.uniqueId = function(prefix) { 748 | var id = idCounter++; 749 | return prefix ? prefix + id : id; 750 | }; 751 | 752 | // By default, Underscore uses ERB-style template delimiters, change the 753 | // following template settings to use alternative delimiters. 754 | _.templateSettings = { 755 | evaluate : /<%([\s\S]+?)%>/g, 756 | interpolate : /<%=([\s\S]+?)%>/g 757 | }; 758 | 759 | // JavaScript micro-templating, similar to John Resig's implementation. 760 | // Underscore templating handles arbitrary delimiters, preserves whitespace, 761 | // and correctly escapes quotes within interpolated code. 762 | _.template = function(str, data) { 763 | var c = _.templateSettings; 764 | var tmpl = 'var __p=[],print=function(){__p.push.apply(__p,arguments);};' + 765 | 'with(obj||{}){__p.push(\'' + 766 | str.replace(/\\/g, '\\\\') 767 | .replace(/'/g, "\\'") 768 | .replace(c.interpolate, function(match, code) { 769 | return "'," + code.replace(/\\'/g, "'") + ",'"; 770 | }) 771 | .replace(c.evaluate || null, function(match, code) { 772 | return "');" + code.replace(/\\'/g, "'") 773 | .replace(/[\r\n\t]/g, ' ') + "__p.push('"; 774 | }) 775 | .replace(/\r/g, '\\r') 776 | .replace(/\n/g, '\\n') 777 | .replace(/\t/g, '\\t') 778 | + "');}return __p.join('');"; 779 | var func = new Function('obj', tmpl); 780 | return data ? func(data) : func; 781 | }; 782 | 783 | // The OOP Wrapper 784 | // --------------- 785 | 786 | // If Underscore is called as a function, it returns a wrapped object that 787 | // can be used OO-style. This wrapper holds altered versions of all the 788 | // underscore functions. Wrapped objects may be chained. 789 | var wrapper = function(obj) { this._wrapped = obj; }; 790 | 791 | // Expose `wrapper.prototype` as `_.prototype` 792 | _.prototype = wrapper.prototype; 793 | 794 | // Helper function to continue chaining intermediate results. 795 | var result = function(obj, chain) { 796 | return chain ? _(obj).chain() : obj; 797 | }; 798 | 799 | // A method to easily add functions to the OOP wrapper. 800 | var addToWrapper = function(name, func) { 801 | wrapper.prototype[name] = function() { 802 | var args = slice.call(arguments); 803 | unshift.call(args, this._wrapped); 804 | return result(func.apply(_, args), this._chain); 805 | }; 806 | }; 807 | 808 | // Add all of the Underscore functions to the wrapper object. 809 | _.mixin(_); 810 | 811 | // Add all mutator Array functions to the wrapper. 812 | each(['pop', 'push', 'reverse', 'shift', 'sort', 'splice', 'unshift'], function(name) { 813 | var method = ArrayProto[name]; 814 | wrapper.prototype[name] = function() { 815 | method.apply(this._wrapped, arguments); 816 | return result(this._wrapped, this._chain); 817 | }; 818 | }); 819 | 820 | // Add all accessor Array functions to the wrapper. 821 | each(['concat', 'join', 'slice'], function(name) { 822 | var method = ArrayProto[name]; 823 | wrapper.prototype[name] = function() { 824 | return result(method.apply(this._wrapped, arguments), this._chain); 825 | }; 826 | }); 827 | 828 | // Start chaining a wrapped Underscore object. 829 | wrapper.prototype.chain = function() { 830 | this._chain = true; 831 | return this; 832 | }; 833 | 834 | // Extracts the result from a wrapped and chained object. 835 | wrapper.prototype.value = function() { 836 | return this._wrapped; 837 | }; 838 | 839 | })(); 840 | --------------------------------------------------------------------------------