├── .gitignore ├── LICENSE ├── README.md ├── details.md ├── examples ├── example.js ├── http-example.js ├── package.json └── user-scope-http-example.js ├── metaparticle-file-storage.js ├── metaparticle-mysql-storage.js ├── metaparticle-redis-storage.js ├── metaparticle-storage-interface.js ├── metaparticle-storage.js ├── package.json └── test └── metaparticle-file-storage_test.js /.gitignore: -------------------------------------------------------------------------------- 1 | examples/node_modules/* 2 | node_modules/* 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Alexander Staubo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Metaparticle/storage 2 | Easy implicit, concurrent persistence for [Node.js](https://nodejs.org) 3 | 4 | Metaparticle/storage makes distributed storage easy. 5 | 6 | ## Implicit, automatic persistence 7 | With Metaparticle/storage you can interact with local variables in a defined _scope_. Any changes 8 | that you make are automatically persisted, _even if your server crashes or restarts_. 9 | 10 | ## Cloud native, built for distributed systems. 11 | Metaparticle/storage is defined for distributed systems running on multiple machines. In particular 12 | Metaparticle/storage ensures [read/update/write consistency](https://en.wikipedia.org/wiki/Read-modify-write). 13 | If Metaparticle/storage detects a conflict due to multiple concurrent writers it automatically rolls back 14 | modifications and re-applies the complete function. 15 | 16 | ## How does it work? 17 | Metaparticle/storage works by defining a collection of _scopes_. Each scope is a named value which can be 18 | obtained by a call to `metaparticle.scoped(scopeName, handlerFunction)`. _scopeName_ provides a unique name 19 | for this scoped data. The data itself is passed on to the handlerFunction. Any modifications to the _scope_ 20 | are automatically detected and persisted. 21 | 22 | ## Example 23 | Here is a simple web-server which increments a counter and returns the number of requests. Despite looking 24 | like a local variable, the counter is kept globally, and persists despite server restarts, scaling or failures. 25 | Furthermore, the read/update/write of the counter is automatically atomic, even under concurrent load the 26 | system maintains the correct count of requests. 27 | 28 | ```js 29 | ... 30 | var mp = require('@metaparticle/storage'); 31 | mp.setStorage('file'); 32 | 33 | var server = http.createServer((request, response) => { 34 | // Define a scope for storage, in this case it is a shared 'global' scope. 35 | mp.scoped('global', (scope) => { 36 | if (!scope.count) { 37 | scope.count = 0; 38 | } 39 | scope.count++; 40 | return scope.count; 41 | }).then((count) => { 42 | response.end("There have been " + count + (count == 1 ? ' request.' : ' requests.')); 43 | }); 44 | }); 45 | ``` 46 | 47 | ## Selecting different storage backends 48 | Metaparticle/storage supports several different storage backends. To select a particular storage 49 | implementation, you need to select it with the `setStorage(driver, config)` function. 50 | 51 | Currently Metaparticle supports the following drivers: 52 | * `file`: Filesystem-local, useful for testing, not much else. 53 | * `redis`: The Redis key/value store 54 | * `mysql`: Everone's favorite relational database. 55 | 56 | ## Defining multiple scopes 57 | The total throughput of an application is related to the total number of concurrent requersts for a particular 58 | scope. Often it makes sense to partition scopes based on user name, location or other variables. Here is a simple 59 | script that partitions the scope by user-name: 60 | 61 | ```js 62 | var urlObj = url.parse(request.url, true); 63 | var scopeName = urlObj.query.user; 64 | if (!scopeName) { 65 | scopeName = 'anonymous' 66 | } 67 | mp.scoped(scopeName, (scope) => { 68 | if (!scope.count) { 69 | scope.count = 0; 70 | } 71 | scope.count++; 72 | return scope.count; 73 | }).then((count) => { 74 | response.end("There have been " + count + (count == 1 ? ' request.' : ' requests.')); 75 | }); 76 | ``` 77 | 78 | ## Caveats 79 | There's no free lunch. To ensure atmocity, Metaparticle/storage may run your code more than once. Though the data input to the function is reset each time, any other side-effects can not be rolled back. 80 | 81 | This means that you _must not_ have side-effects in your code. 82 | 83 | Examples of side-effects include: 84 | * Returning data over an HTTP channel 85 | * Writing to a file 86 | * Writing to a non-scoped variable 87 | 88 | Metaparticle/storage does not currently detect these side-effects. This means that it is up to you to police your usage and ensure that the code within a scope is side-effect free. 89 | 90 | ## Deep Dive 91 | There is a much deeper dive [here](details.md), it contains a deeper description of the problems Metaparticle/storage is trying to solve and examples of it in operation. 92 | 93 | ## Bugs 94 | There are probably some 95 | -------------------------------------------------------------------------------- /details.md: -------------------------------------------------------------------------------- 1 | # Metaparticle/storage 2 | Easy implicit, concurrent persistence for [Node.js](https://nodejs.org) 3 | 4 | ## But what does it do for me? 5 | 6 | Metaparticle/storage lets you treat persistent (e.g. MongoDB, Redis or MySQL) storage the same as you treat local storage. 7 | Furthermore it manages concurrent storage access, it automatically detects conflicts, rolls back data transformations and 8 | resolves the conflicts. 9 | 10 | ## Hrm, I'm not sure what that really means, can you say it in a different way? 11 | Metaparticle/storage makes distributed storage easy. 12 | 13 | ## Great, but how does it work? 14 | Metaparticle/storage works by defining chunks of data called _scopes_. You define a scope by giving Metaparticle/storage a 15 | scope _name_. Metaparticle/storage gives you back that scope and ensures that all reads/writes within that scope occur 16 | in a single transaction. This means that you are guaranteed that within a scope, data consistency is guaranteed. 17 | 18 | ## Sounds fancy, can I get an example? 19 | To understand how this works, consider the task of multi-threaded increment of an integer `i`. We have all seen the data race 20 | that can occur: 21 | 22 | 1. Thread-1 reads `i`, `i == 0` 23 | 2. Thread-2 reads `i`, `i == 0` 24 | 3. Thread-1 writes `i + 1`, `i == 1` 25 | 4. Thread-2 writes `i + 1`, `i == 1` 26 | 27 | You can see that despite two increment operations, the value of the variable only increases by one. 28 | 29 | To solve this via Metaparticle/storage, you write: 30 | 31 | ```js 32 | metaparticle.scoped('global', (data) => { 33 | data.i++; 34 | ... 35 | } 36 | ``` 37 | 38 | Metaparticle ensures both that the local modifications to the value `i` are preserved in persistent storage, as well as that 39 | the changes are only preserved if there have been no other concurrent modifications to the data. If Metaparticle detects other 40 | data modifications, it automatically rolls back any of its current data modifications and re-runs the function. 41 | 42 | ## Give me more! 43 | There are some additional walkthroughs: 44 | * Basic Example 45 | * Basic Web Server 46 | * User-specific scopes 47 | 48 | ## Details 49 | 50 | ### Goals 51 | In distributed systems, persistent state management is often one of the largest design challenges. 52 | The goals of Metaparticle storage are to make this easy. Metaparticle storage achieves this by moving from _explicit_ storage 53 | to _implicit_ storage, and by managing concurrent operations for the user. 54 | 55 | ### Implicit storage 56 | When we say _implicit_ storage what we mean is that the persistent storage of data occurs _implicitly_ without 57 | user interaction. This contrasts to _explicit_ storage which is what most people write today. 58 | 59 | To explain this more completely, we can see that in-memory storage is _implicit_ already in most programming languages: 60 | ```js 61 | var i = 0; 62 | ... 63 | i = i + 1; 64 | ``` 65 | 66 | But when we encounter a persistence layer, the storage becomes _explicit_: 67 | 68 | ```js 69 | client.set("i", 0); 70 | ... 71 | var value = client.get("i"); 72 | client.update("i", value + 1); 73 | ``` 74 | 75 | The differences between the two code snippets aren't eggregious, but the later introduces a bunch of cognitive 76 | overhead to understanding and writing the code. What is this `client`? Why do I use an explicit `set` method instead of just 77 | assignment? What was that `value` variable anyway? 78 | 79 | Especially since we all learn in-memory _implicit_ storage first, the cognitive overhead is sufficient to make distributed 80 | systems more challenging to more novice programmers. Metaparticle storage resolves this by making persistent storage implicit: 81 | 82 | ```js 83 | var mp = require('@metaparticle/storage'); 84 | 85 | mp.scoped('scope', function(data) { 86 | data.i = 0; 87 | ... 88 | data.i = data.i + 1; 89 | }); 90 | ``` 91 | 92 | At the end of this code, `i` will have been incremented and stored persistently so that even if the program exits and restarts, 93 | the new value of `i` will be preserved. 94 | 95 | ### Concurrent Access 96 | The other reason that storage is challenging in distributed systems is that often times there are multiple replicas of 97 | the system and storage operations are being performed simultaneously on multiple different machines or servers. The result 98 | of this is that conflicts can occur and data can be lost or corrupted. 99 | 100 | The most common of these patterns is one where two servers simultaneously read the value for `i`. The both increment the value 101 | and then they both store it. In this case, despite two different increment operations, only `i + 1` is actually stored back 102 | into the persistent storage. 103 | 104 | Metaparticle storage makes this easy as well, since it takes care of conflict detection, rollback and resolution. The only caveat 105 | of this (there is no free lunch after all). Is that your function has to be side-effect free other than it's interactions with 106 | storage. The reason for this is that the function may be executed multiple time before it can be successfully committed. 107 | -------------------------------------------------------------------------------- /examples/example.js: -------------------------------------------------------------------------------- 1 | var mp = require('@metaparticle/storage'); 2 | 3 | mp.setStorage('file', { directory: '/tmp' }); 4 | mp.scoped('global', function(scope) { 5 | if (!scope.calls) { 6 | scope.calls = 0; 7 | } 8 | scope.calls++; 9 | return scope.calls; 10 | }).then(function(calls) { 11 | console.log(calls); 12 | mp.shutdown(); 13 | }); 14 | -------------------------------------------------------------------------------- /examples/http-example.js: -------------------------------------------------------------------------------- 1 | // Simple HTTP Server example, keeps track of the number of requests 2 | // and reports back over HTTP 3 | var http = require('http'); 4 | var mp = require('@metaparticle/storage'); 5 | 6 | mp.setStorage('file'); 7 | 8 | var server = http.createServer((request, response) => { 9 | mp.scoped('global', (scope) => { 10 | if (!scope.count) { 11 | scope.count = 0; 12 | } 13 | scope.count++; 14 | return scope.count; 15 | }).then((count) => { 16 | response.end("There have been " + count + (count == 1 ? ' request.' : ' requests.')); 17 | }); 18 | }); 19 | 20 | server.listen(8090, (err) => { 21 | if (err) { 22 | console.log('error starting server', err) 23 | } 24 | 25 | console.log(`server is listening on http://localhost:8090`) 26 | }); -------------------------------------------------------------------------------- /examples/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@metaparticle/storage-examples", 3 | "version": "0.1.0", 4 | "description": "Scoped storage library examples", 5 | "repository": { 6 | "type": "git", 7 | "url": "git+https://github.com/metaparticle-io/storage.git" 8 | }, 9 | "files": [ 10 | "*.js" 11 | ], 12 | "main": "example.js", 13 | "scripts": { 14 | "clean": "rm -Rf node_modules/" 15 | }, 16 | "author": "Brendan Burns", 17 | "license": "Apache-2.0", 18 | "dependencies": { 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /examples/user-scope-http-example.js: -------------------------------------------------------------------------------- 1 | // Simple HTTP Server example, keeps track of the number of requests 2 | // for each user and reports back over HTTP 3 | var http = require('http'); 4 | var url = require('url'); 5 | var mp = require('@metaparticle/storage'); 6 | 7 | mp.setStorage('file'); 8 | 9 | var server = http.createServer((request, response) => { 10 | var urlObj = url.parse(request.url, true); 11 | var scopeName = urlObj.query.user; 12 | if (!scopeName) { 13 | scopeName = 'anonymous' 14 | } 15 | mp.scoped(scopeName, (scope) => { 16 | if (!scope.count) { 17 | scope.count = 0; 18 | } 19 | scope.count++; 20 | return scope.count; 21 | }).then((count) => { 22 | response.end("There have been " + count + (count == 1 ? ' request.' : ' requests.')); 23 | }); 24 | }); 25 | 26 | server.listen(8090, (err) => { 27 | if (err) { 28 | console.log('error starting server', err) 29 | } 30 | 31 | console.log(`server is listening on http://localhost:8090`) 32 | }); -------------------------------------------------------------------------------- /metaparticle-file-storage.js: -------------------------------------------------------------------------------- 1 | /** 2 | * File based storage layer for metaparticle. Not for use in production, test/experiment only! 3 | */ 4 | (function () { 5 | var q = require('q'); 6 | var log = require('loglevel'); 7 | var pathlib = require('path'); 8 | 9 | var file = null; 10 | var fs = function () { 11 | if (file == null) { 12 | file = require('fs'); 13 | } 14 | return file; 15 | }; 16 | 17 | var directory = process.cwd(); 18 | 19 | module.exports.configure = function(config) { 20 | if (config && config.directory) { 21 | directory = config.directory; 22 | } 23 | }; 24 | 25 | /** 26 | * A note on the 'data' package: 27 | * This library expects to store and load objects of the form: 28 | * { 29 | * 'data': , 30 | * 'version': 31 | * } 32 | * 33 | * Where 'version' is a string that is used for optimistic concurrency. 34 | */ 35 | 36 | /** 37 | * Load a particular data scope and return it 38 | * @param {string} scope The scope to load 39 | * @returns A promise for data for that scope 40 | */ 41 | module.exports.load = function (scope) { 42 | var deferred = q.defer(); 43 | var path = pathlib.join(directory, scope + ".json"); 44 | // TODO: make this async too 45 | var stats = null; 46 | try { 47 | stats = fs().statSync(path); 48 | } catch (err) { 49 | if (err.code != 'ENOENT') { 50 | deferred.reject(err); 51 | return deferred.promise; 52 | } 53 | } 54 | fs().readFile(path, 'utf-8', function (err, data) { 55 | if (err) { 56 | if (err.code == 'ENOENT') { 57 | deferred.resolve({ 58 | 'data': {}, 59 | 'version': 'empty' 60 | }); 61 | } else { 62 | deferred.reject(err); 63 | } 64 | } else { 65 | try { 66 | try { 67 | var obj = JSON.parse(data); 68 | } catch (error) { 69 | deferred.reject(error); 70 | } 71 | deferred.resolve({ 72 | 'data': obj, 73 | 'version': stats.mtime, 74 | }); 75 | } catch (ex) { 76 | deferred.reject(ex); 77 | } 78 | } 79 | }); 80 | return deferred.promise; 81 | } 82 | 83 | /** 84 | * Store the data to persistent storage. 85 | * @param {string} scope The scope to store 86 | * @param {data} data The data package 87 | * @returns A promise that resolves to true if the storage succeeded, false otherwise. 88 | */ 89 | module.exports.store = function (scope, data) { 90 | var deferred = q.defer(); 91 | // TODO: make this async too 92 | var stats = null; 93 | var empty = false; 94 | var path = pathlib.join(directory, scope + '.json'); 95 | try { 96 | stats = fs().statSync(path); 97 | } catch (err) { 98 | if (err.code != 'ENOENT') { 99 | deferred.reject(err); 100 | return deferred.promise; 101 | } else { 102 | empty = true; 103 | } 104 | } 105 | 106 | if (empty) { 107 | if (data.version != 'empty') { 108 | log.warn('version is not empty but file does not exist'); 109 | deferred.resolve(false); 110 | } 111 | } else { 112 | // for some reason straight object compare wasn't working, so stringify then compare 113 | var versionStr = JSON.stringify(data.version); 114 | var statStr = JSON.stringify(stats.mtime); 115 | if (versionStr != statStr) { 116 | log.debug('versions do not match: %' + data.version + '% vs %' + stats.mtime + '%'); 117 | deferred.resolve(false); 118 | } 119 | } 120 | 121 | var str = JSON.stringify(data.data); 122 | fs().writeFile(path, str, function (err) { 123 | if (err) { 124 | deferred.reject(err); 125 | } else { 126 | deferred.resolve(true); 127 | } 128 | }); 129 | return deferred.promise; 130 | } 131 | } ()); 132 | -------------------------------------------------------------------------------- /metaparticle-mysql-storage.js: -------------------------------------------------------------------------------- 1 | var mysql = require('mysql'); 2 | var q = require('q'); 3 | 4 | var connection = null; 5 | 6 | module.exports.configure = function (config) { 7 | if (config) { 8 | connection = mysql.createConnection(config); 9 | } else { 10 | connection = mysql.createConnection({ 11 | host : 'localhost', 12 | user : 'root', 13 | database : 'data' 14 | }); 15 | } 16 | 17 | connection.connect(); 18 | 19 | connection.query('create table if not exists records (name varchar(256) primary key, version int, data text);', (error) => { 20 | if (error) { 21 | throw error; 22 | } 23 | }); 24 | }; 25 | 26 | 27 | module.exports.load = function (scope) { 28 | var deferred = q.defer(); 29 | connection.query('select version, data from records where name=\'' + scope + '\'', (error, results) => { 30 | if (error) { 31 | throw error; 32 | } 33 | if (results.length == 0) { 34 | deferred.resolve({ 35 | 'data': {}, 36 | 'version': 'empty' 37 | }); 38 | return; 39 | } 40 | var obj = JSON.parse(results[0].data); 41 | obj.version = results[0].version; 42 | deferred.resolve(obj); 43 | }); 44 | return deferred.promise; 45 | } 46 | 47 | /** 48 | * Store the data to persistent storage. 49 | * @param {string} scope The scope to store 50 | * @param {data} data The data package 51 | * @returns A promise that resolves to true if the storage succeeded, false otherwise. 52 | */ 53 | module.exports.store = function (scope, data) { 54 | var deferred = q.defer(); 55 | if (data.version == 'empty') { 56 | var sql = mysql.format('insert into records (name, version, data) values (?, ?, ?);', [scope, 1, JSON.stringify(data)]); 57 | connection.query(sql, (error) => { 58 | if (error) { 59 | if (error.code == 'ER_DUP_ENTRY') { 60 | deferred.resolve(false); 61 | return; 62 | } 63 | throw(error); 64 | } 65 | deferred.resolve(true); 66 | }); 67 | } else { 68 | var oldVersion = data.version; 69 | data.version++; 70 | var sql = mysql.format('update records set data=?, version=? where (name=? and version=?) ', [JSON.stringify(data), data.version, scope, oldVersion]); 71 | connection.query(sql, (error, results) => { 72 | if (error) { 73 | throw(error); 74 | } 75 | if (results.affectedRows != 1) { 76 | deferred.resolve(false); 77 | return; 78 | } 79 | deferred.resolve(true); 80 | }) 81 | } 82 | return deferred.promise; 83 | }; 84 | 85 | module.exports.shutdown = () => { 86 | connection.end(); 87 | } -------------------------------------------------------------------------------- /metaparticle-redis-storage.js: -------------------------------------------------------------------------------- 1 | (function () { 2 | var q = require('q'); 3 | var redis = require('node-redis-client'); 4 | var log = require('loglevel').getLogger('metaparticle-redis'); 5 | var host = process.env['REDIS_HOST']; 6 | 7 | var makeRedisClient = function () { 8 | var opts = { 9 | host: host 10 | }; 11 | c = new redis(opts); 12 | c.on('connect', function () { 13 | log.info('connected'); 14 | }); 15 | return c; 16 | } 17 | 18 | //var generator = op.generator(makeRedisClient); 19 | var injectClient = null; 20 | var client = function () { 21 | if (injectClient != null) { 22 | return injectClient; 23 | } 24 | return makeRedisClient(); 25 | }; 26 | 27 | module.exports.configure = function(config) { 28 | if (config && config.host) { 29 | host = config.host; 30 | } 31 | } 32 | 33 | module.exports.injectClientForTesting = function (client) { 34 | injectClient = client; 35 | } 36 | 37 | /** 38 | * A note on the 'data' package: 39 | * This library expects to store and load objects of the form: 40 | * { 41 | * 'data': , 42 | * 'version': 43 | * } 44 | * 45 | * Where 'version' is a string that is used for optimistic concurrency. 46 | */ 47 | 48 | /** 49 | * Load a particular data scope and return it 50 | * @param {string} scope The scope to load 51 | * @returns A promsie for the data for that scope. 52 | */ 53 | module.exports.load = function (scope) { 54 | var deferred = q.defer(); 55 | var obj = {}; 56 | 57 | var c = client(); 58 | // TODO: Probably need a MULTI here 59 | c.call('GET', 'value', function (err, res) { 60 | if (err) { 61 | deferred.reject(err); 62 | return; 63 | } 64 | log.debug(res); 65 | if (res != null) { 66 | deferred.resolve(JSON.parse(res)); 67 | } else { 68 | log.debug('resolving null'); 69 | deferred.resolve({ 70 | 'data': {}, 71 | 'version': 0 72 | }); 73 | } 74 | }); 75 | 76 | return deferred.promise; 77 | } 78 | 79 | /** 80 | * Store the data to persistent storage. 81 | * @param {string} scope The scope to store 82 | * @param {data} data The data package 83 | * @returns A promise that resolves with true if the storage succeeded, false otherwise. 84 | */ 85 | module.exports.store = function (scope, data) { 86 | var deferred = q.defer(); 87 | 88 | // TODO: NEED TO MAKE SURE THIS IS ALL ONE ATOMIC BLOCK IN THE JS 89 | // ALTERNATELY, JUST CREATE A NEW CLIENT FOR EACH STORE. 90 | 91 | var id = Math.floor(Math.random() * 1000); 92 | log.debug(id + " redis store"); 93 | var c = client(); 94 | // TODO: Turn this into it's own function that returns a promise, and then release in that 95 | // promise. 96 | c.call('WATCH', 'value', function (err) { 97 | log.debug(id + " redis store 2 " + err); 98 | 99 | if (err) { 100 | deferred.reject(err); 101 | return 102 | } 103 | 104 | c.call('GET', 'value', function (err, res) { 105 | log.debug(id + " redis store 3 " + err); 106 | 107 | try { 108 | if (err) { 109 | deferred.reject(err); 110 | return; 111 | } 112 | if (res == null && data.version != "0") { 113 | //deferred.reject(new Error("version mismatch")); 114 | deferred.resolve(false); 115 | return; 116 | } 117 | log.debug(id + " redis store 3.5"); 118 | if (res != null) { 119 | var obj = JSON.parse(res); 120 | log.debug(id + ' checking: ' + obj.version + " vs " + data.version); 121 | if (obj.version != data.version) { 122 | deferred.resolve(false); 123 | return; 124 | //deferred.reject(new Error('version mismatch: ' + data.version + ' vs. ' + res)); 125 | } 126 | } 127 | data.version = data.version + 1; 128 | 129 | c.call('MULTI', function () { 130 | c.call('SET', 'value', JSON.stringify(data), function () { 131 | c.call('EXEC', function (err, res) { 132 | log.debug(id + " redis store 4 " + err); 133 | if (err) { 134 | log.debug(id + ' rejected: ' + err); 135 | //deferred.reject(err); 136 | deferred.resolve(false); 137 | return; 138 | } 139 | if (res == null) { 140 | log.debug(id + ' rejected: null'); 141 | deferred.resolve(false); 142 | return; 143 | } 144 | log.debug(id + ' result was: ' + res); 145 | log.debug(id + ' resolving true'); 146 | deferred.resolve(true); 147 | }); 148 | }); 149 | }); 150 | } catch (ex) { 151 | log.debug(ex); 152 | } 153 | }); 154 | }); 155 | 156 | return deferred.promise; 157 | } 158 | } ()); 159 | -------------------------------------------------------------------------------- /metaparticle-storage-interface.js: -------------------------------------------------------------------------------- 1 | (function() { 2 | var q = require('q'); 3 | /** 4 | * A note on the 'data' package: 5 | * This library expects to store and load objects of the form: 6 | * { 7 | * 'data': , 8 | * 'version': 9 | * } 10 | * 11 | * Where 'version' is a string that is used for optimistic concurrency. 12 | */ 13 | 14 | /** 15 | * Load a particular data scope and return it 16 | * @param {string} scope The scope to load 17 | * @returns A promise for the data for that scope. 18 | */ 19 | module.exports.load = function(scope) { 20 | return q.fcall(function() { 21 | return { 22 | 'data': {}, 23 | 'version': '1' 24 | }; 25 | }); 26 | } 27 | 28 | /** 29 | * Store the data to persistent storage. 30 | * @param {string} scope The scope to store 31 | * @param {data} data The data package 32 | * @returns A promise that resolves with true if the storage succeeded, false otherwise. 33 | */ 34 | module.exports.store = function(scope, data) { 35 | return q.fcall(function() { return false; }); 36 | } 37 | }()); 38 | -------------------------------------------------------------------------------- /metaparticle-storage.js: -------------------------------------------------------------------------------- 1 | (function () { 2 | var storage = null; 3 | var q = require('q'); 4 | var storageImpls = { 5 | "file": './metaparticle-file-storage.js', 6 | "redis": './metaparticle-redis-storage', 7 | "mysql": './metaparticle-mysql-storage.js' 8 | } 9 | 10 | module.exports.setStorage = function(name, config) { 11 | // Does this storage method exist? 12 | const file = storageImpls[name]; 13 | if ('undefined' !== typeof (file)) { 14 | // Yes, initialise it. 15 | storage = require(file); 16 | storage.configure(config); 17 | } 18 | } 19 | 20 | module.exports.shutdown = () => { 21 | if (storage && storage.shutdown) { 22 | storage.shutdown(); 23 | } 24 | } 25 | 26 | module.exports.scoped = function(name, fn) { 27 | if (storage == null) { 28 | throw("You must initialize a storage implementation via setStorage first."); 29 | } 30 | var promise = q.defer(); 31 | loadExecuteStore(name, fn, promise); 32 | return promise.promise; 33 | } 34 | 35 | var loadExecuteStore = function(name, fn, promise) { 36 | storage.load(name).then(function(data) { 37 | var dirty = false; 38 | var Proxy = require('harmony-proxy'); 39 | var obj = new Proxy(data.data, { 40 | set: function (target, property, value) { 41 | data.data[property] = value; 42 | dirty = true; 43 | return true; 44 | } 45 | }); 46 | var result = fn(obj); 47 | if (dirty) { 48 | storage.store(name, data).then(function(success) { 49 | if (!success) { 50 | setTimeout(function() { 51 | loadExecuteStore(name, fn, promise); 52 | }, 100); 53 | } else { 54 | promise.resolve(result); 55 | } 56 | }).catch(function(error) { 57 | console.log(error); 58 | }); 59 | } 60 | }).catch(function(error) { 61 | console.log(error); 62 | }); 63 | }; 64 | }()); 65 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@metaparticle/storage", 3 | "version": "0.1.0", 4 | "description": "Scoped storage library", 5 | "repository": { 6 | "type": "git", 7 | "url": "git+https://github.com/metaparticle-io/storage.git" 8 | }, 9 | "files": [ 10 | "*.js" 11 | ], 12 | "main": "metaparticle-storage.js", 13 | "scripts": { 14 | "clean": "rm -Rf node_modules/ dist/", 15 | "test": "mocha test/*_test.js" 16 | }, 17 | "author": "Brendan Burns", 18 | "license": "Apache-2.0", 19 | "dependencies": { 20 | "harmony-proxy": "^1.0.1", 21 | "loglevel": "^1.4.1", 22 | "mysql": "^2.15.0", 23 | "node-redis-client": "0.0.10", 24 | "q": "^1.5.0" 25 | }, 26 | "devDependencies": { 27 | "unit.js": "^2.0.0", 28 | "mocha": "^3.4.2" 29 | }, 30 | "bugs": { 31 | "url": "https://github.com/metaparticle-io/storage/issues" 32 | }, 33 | "homepage": "https://github.com/metaparticle-io/storage", 34 | "keywords": [ 35 | "storage", 36 | "persistence" 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /test/metaparticle-file-storage_test.js: -------------------------------------------------------------------------------- 1 | var test = require('unit.js'); 2 | 3 | describe('file-storage', function() { 4 | var mfs; 5 | var scope; 6 | 7 | before(function(done) { 8 | this.timeout(10000); 9 | mp = require('../metaparticle-file-storage'); 10 | scope = 'scope-' + Math.floor(Math.random() * 10000); 11 | done(); 12 | }); 13 | 14 | afterEach(function(done) { 15 | require('fs').unlink(scope + '.json', function() { 16 | done(); 17 | }); 18 | }) 19 | 20 | it('should return empty if no such scope', function(done) { 21 | var promise = mp.load('no-such-scope'); 22 | promise.then(function(data) { 23 | test.object(data.data).is({}); 24 | test.string(data.version).is('empty'); 25 | done(); 26 | }, 27 | function(err) { 28 | // This should never be called. 29 | test.assert.fail(err); 30 | done(); 31 | }).done(); 32 | }); 33 | 34 | it('should be able to write if it was empty', function(done) { 35 | var promise = mp.store(scope, { 36 | data: {}, 37 | version: 'empty' 38 | }); 39 | promise.then(function(success) { 40 | test.bool(success).isTrue(); 41 | done(); 42 | }, 43 | function(err) { 44 | test.assert.fail(err); 45 | done(); 46 | }).done(); 47 | }); 48 | 49 | it('should round trip successfully', function(done) { 50 | var obj = { 51 | 'foo': 'bar', 52 | 'baz': 'blah' 53 | }; 54 | 55 | var promise = mp.store(scope, { 56 | 'data': obj, 57 | 'version': 'empty' 58 | }); 59 | 60 | promise.then(function() { 61 | mp.load(scope).then(function(loaded) { 62 | test.object(loaded.data).is(obj); 63 | done(); 64 | }).done(); 65 | }).done(); 66 | }); 67 | 68 | it('should fail if there is a conflict non-exist', function(done) { 69 | var obj = { 70 | 'foo': 'bar' 71 | }; 72 | var promise = mp.store(scope, { 73 | 'data': obj, 74 | 'version': 'non-empty-version' 75 | }); 76 | 77 | promise.then(function(success) { 78 | test.bool(success).isFalse(); 79 | // TODO: test here that no file was written 80 | done(); 81 | }, 82 | function(err) { 83 | test.assert.fail(err); 84 | done(); 85 | }).done(); 86 | }); 87 | 88 | it('should fail if there is a conflict exist', function(done) { 89 | var obj = { 90 | 'foo': 'bar', 91 | 'baz': 'blah' 92 | }; 93 | 94 | var data = { 95 | 'data': obj, 96 | 'version': 'empty' 97 | }; 98 | 99 | var promise = mp.store(scope, data); 100 | 101 | promise.then(function(success) { 102 | test.bool(success).isTrue(); 103 | return mp.store(scope, data); 104 | }).then(function(success) { 105 | test.bool(success).isFalse() 106 | done() 107 | }).done(); 108 | }); 109 | 110 | it('should fail if there is a conflict different version', function(done) { 111 | var obj = { 112 | 'foo': 'bar', 113 | 'baz': 'blah' 114 | }; 115 | 116 | var data = { 117 | 'data': obj, 118 | 'version': 'empty' 119 | }; 120 | 121 | var promise = mp.store(scope, data); 122 | 123 | promise.then(function(success) { 124 | test.bool(success).isTrue(); 125 | return mp.load(scope, data); 126 | }).then(function(data) { 127 | data.version = 'no-such-version'; 128 | data.data['foo'] = 'baz'; 129 | return mp.store(scope, data); 130 | }).then(function(success) { 131 | test.bool(success).isFalse(); 132 | done() 133 | }).done(); 134 | }); 135 | }); 136 | --------------------------------------------------------------------------------