├── couchreqest.js ├── tests ├── photoshare.js └── test-cloud.js ├── docstate.js ├── device.js ├── README.md └── cloud.js /couchreqest.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/photoshare.js: -------------------------------------------------------------------------------- 1 | // for testing against the photoshare example 2 | var db_host = "http://jchrisa:jchrisa@127.0.0.1:5984" 3 | , db_name = "photoshare-control-device" 4 | , cloud = require("../cloud") 5 | ; 6 | 7 | cloud.start(db_host, db_name); 8 | -------------------------------------------------------------------------------- /docstate.js: -------------------------------------------------------------------------------- 1 | var follow = require("follow") 2 | , stately = require("stately") 3 | ; 4 | 5 | exports.connect = function(db_host, db_name) { 6 | var feed = new follow.Feed({db : [db_host, db_name].join('/')}) 7 | , safeMachine, safeStates = {} 8 | , cautiousMachine, unsafeStates = {} 9 | ; 10 | 11 | feed.include_docs = true; 12 | feed.on("change", function(change) { 13 | if (change.doc.type && change.doc.state) 14 | console.log(change.doc.type, change.doc.state) 15 | safeMachine.handle(change.doc); 16 | cautiousMachine.handle(change.doc); 17 | }); 18 | 19 | function start() { 20 | safeMachine = stately.define(safeStates); 21 | cautiousMachine = stately.define(unsafeStates); 22 | feed.follow(); 23 | } 24 | 25 | function registerSafeCallback(type, state, cb) { 26 | safeStates[type] = safeStates[type] || {}; 27 | safeStates[type][state] = cb; 28 | } 29 | 30 | function registerUnsafeCallback(type, state, cb) { 31 | unsafeStates[type] = unsafeStates[type] || {}; 32 | unsafeStates[type][state] = cb; 33 | } 34 | 35 | return { 36 | start : start, 37 | safe : registerSafeCallback, 38 | unsafe : registerUnsafeCallback 39 | }; 40 | }; 41 | -------------------------------------------------------------------------------- /device.js: -------------------------------------------------------------------------------- 1 | var stately = require("stately"); 2 | 3 | var LOCALHOST = "http://127.0.0.1:5984/" 4 | 5 | var control = stately.follow(LOCALHOST + "control"); 6 | 7 | var docState = control.docState; 8 | 9 | function userTag(doc) { 10 | doc.owner = user_email; 11 | doc._readers = { 12 | names : [user_email] 13 | }; 14 | return doc; 15 | }; 16 | 17 | docState("device", "active", function() { 18 | // might not need to do anything 19 | // what we really needed was replication access to the control db (for our owner's stuff) 20 | }); 21 | 22 | docState("subscription", "active", function(doc) { 23 | // configure replication for each active subscription 24 | var local = "sub-" + doc._id 25 | , remote = doc.syncpoint 26 | ; 27 | startReplication(local, remote) 28 | }); 29 | 30 | docState("subscription", ["stopped", "paused"], function() { 31 | // cancel replication for each stopped or paused subscription 32 | }); 33 | 34 | function startReplication(local, remote) { 35 | var repl = { 36 | source : remote, 37 | target : local, 38 | continuous : true, 39 | create_target : true 40 | }; 41 | db.server.replicate(repl, errLog); 42 | } 43 | 44 | 45 | function normalizeSyncConfig() { 46 | db.view("channels/active-subscriptions", function(err, view) { 47 | replicatorDb.allDocs({include_docs:true},function(err, docs) { 48 | // for each active subscription, make sure there is a replicator doc 49 | }); 50 | }); 51 | }; 52 | 53 | // channels are private by default 54 | // auto-subscribe to my channels, not to other folks 55 | // todo, implement public : true for channels 56 | function createChannel(channelName) { 57 | db.save(userTag({ 58 | type : "channel", 59 | state : "new", 60 | name : channelName 61 | }), errLog); 62 | } 63 | 64 | // auto-subscribe to my channels 65 | // use well-known id here so user choices across devices converge 66 | docState("channel", ["new","ready"], function(doc) { 67 | var sub_id = doc._id + "-sub-" + device_owner_email; 68 | if (doc.owner != device_owner_email) return; 69 | db.open(sub_id, function(err, sub) { 70 | if (err) { 71 | sub = userTag({ 72 | _id : sub_id, 73 | type : "subscription", 74 | state : "active" 75 | }); 76 | } 77 | sub.remote = doc.syncpoint; 78 | db.save(sub, errLog); 79 | }); 80 | }); 81 | 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This code is one implementation of a process I expect will be common among mobile CouchApps: coordinating sync for millions of mobile devices. 2 | 3 | The first target is just registration of devices via an email confirmation loop. The second target is automatic synchronization of all of a users Channels across devices. 4 | 5 | A channel is just a CouchDB database, except it's distributed across potentially multiple users and synchronized via Couchbase CouchSync. Couchbase uses standard CouchDB replication as the communication backbone. All messages between the cloud and devices take place over replication, so the device can always configure its relation to the cloud, even when it is offline. And the cloud can continue to process device updates even if they come in via a non standard route (the photo galleries you created on vacation got synced to your laptop before they got synced to the cloud, but it doesn't matter, all 3 endpoints can still manage the same dataset.) 6 | 7 | Couchbase Channels takes the guesswork out of manages lots of databases per user. Currently, today, Aug 22nd, the code barely works, I'm just happy to have a proceedural framework for writing them in place, and some meaningful tests. The tests are in a weird style. Feel free to out compete them in other tests for weirder style. 8 | 9 | This repo really contains an application. The application has already spawned an extension to some other frameworks I started on some time ago. So here is how it works. 10 | 11 | The app code looks like this: 12 | 13 | control.safe("channel", "ready", function(doc) { 14 | var channel_db = urlDb(doc.syncpoint); 15 | channel_db.insert({ 16 | _id : 'description', 17 | name : doc.name 18 | }, errLog); 19 | }); 20 | 21 | control.unsafe("device", "new", function(doc) { 22 | var confirm_code = Math.random().toString().split('.').pop(); // todo better entropy 23 | sendEmail(doc.owner, confirm_code, function(err) { 24 | if (err) { 25 | errLog(err) 26 | } else { 27 | doc.state = "confirming"; 28 | doc.confirm_code = confirm_code; 29 | db.insert(doc, errLog); . 30 | } 31 | }); 32 | }); 33 | 34 | So basically it listens to the Couch `_changes` feed and pattern matches against `doc.type` and `doc.state`. It runs your code on the documents that match it. 35 | 36 | There are two modes: **safe** and **unsafe**. Safe mode is for functions which are safe to be run twice by accident. This can happen if you have multiple workers on the same database. Safe functions are safe because if they run in multiple processes concurrently (or they crash in the middle and are re-run) they will not create unwanted side-effects. In fancy talk, they are "idempotent". 37 | 38 | Unsafe mode is for functions that must not be run twice by racing bots, for instance if you are sending an email use unsafe mode or there's a chance you might send it twice concurrently. 39 | 40 | In workloads where you have a high degree of concurrent activity, and the work each transaction does is expensive, you are more likely to want to use unsafe mode for everything. But for normal stuff if you just keep your functions idempotent and lightweight, you are just as well off having some bots duplicate work that gets thrown out, than trying to coordinate bots via Couch MVCC. 41 | 42 | It is expected that your functions will, as part of their operation, save their triggering document back to the database, usually with a new state. In this example I'm using [nano](https://github.com/dscape/nano) 43 | 44 | The goal is to keep your call backs for each individual state as small as possible. Each should be a transaction. This makes the scope for errors introduced by retries limited. 45 | 46 | 47 | ## TODO: Basically Everything 48 | 49 | * refactor code for readability 50 | * package some of the libraries in a proper npm embedded way (maybe docstate just rides with stately) 51 | * come up with a better way to test it 52 | * spec out the remaining doc workflows 53 | * build the parts the depend on being in an app 54 | (or document the API the app must meet) 55 | 56 | ## License: 57 | 58 | Apache 2.0 59 | copyright 2011 Couchbase Inc 60 | author Chris Anderson jchris@couchbase.com 61 | -------------------------------------------------------------------------------- /tests/test-cloud.js: -------------------------------------------------------------------------------- 1 | var assert = require("assert") 2 | , cloud = require("../cloud") 3 | , request = require("request") 4 | , nano = require("nano") 5 | , follow = require("follow") 6 | ; 7 | 8 | // mini test framework 9 | var testNames = {}; 10 | function start(name) { 11 | console.log("start", name) 12 | testNames[name] = new Date(); 13 | }; 14 | function finish(name) { 15 | var start = testNames[name]; 16 | delete testNames[name]; 17 | console.log("finish", name, new Date() - start); 18 | if (Object.keys(testNames).length == 0) { 19 | setTimeout(function() { 20 | if (Object.keys(testNames).length == 0) { 21 | console.log("all tests done") 22 | process.exit(0) 23 | } 24 | }, 50); 25 | } 26 | }; 27 | // end mini test framework 28 | 29 | var user_email = 'drspaceman@30rock.com'; 30 | 31 | function userTag(doc) { 32 | doc.owner = user_email; 33 | // doc._readers = { 34 | // names : [user_email] 35 | // }; 36 | return doc; 37 | }; 38 | 39 | function errLog(err) { 40 | if (err) { 41 | console.error(err.status_code, err.error, err.message) 42 | } 43 | }; 44 | 45 | // only works on urls like http://example.com/foobar 46 | function urlDb(url) { 47 | url = url.split("/"); 48 | db = url.pop(); 49 | return nano(url.join('/')).use(db); 50 | }; 51 | 52 | var db_host = "http://jchrisa:jchrisa@127.0.0.1:5984" 53 | , db_name = "test-control" 54 | , couch = nano(db_host) 55 | , db = couch.use(db_name) 56 | , userDb = couch.use("_users") // todo the tests should use a test user db 57 | ; 58 | 59 | userDb.get("org.couchdb.user:"+user_email, function(err, r, doc) { 60 | if (!err) { 61 | userDb.destroy(doc._id, doc._rev, errLog) 62 | } 63 | // it's OK to do this out of order as we are not counting seq's in the users db 64 | }); 65 | 66 | function fixtureDb(fun) { 67 | couch.db.destroy(db_name, function(err, resp) { 68 | couch.db.create(db_name, function(err, resp) { 69 | if (err && err.status_code != 412) { 70 | errLog(err, resp) 71 | } else { 72 | fun() 73 | } 74 | }); 75 | }); 76 | }; 77 | 78 | 79 | // test that a new channel doc gets made ready and a database is created for it 80 | fixtureDb(function() { 81 | // start cloud server 82 | cloud.start(db_host, db_name); 83 | start("send new device email"); 84 | 85 | // create a new device doc 86 | // this would ordinarily be done by the device UI on first launch 87 | db.insert({ 88 | owner : user_email, 89 | type : "device", 90 | state : "new", 91 | device_code : "random-code", 92 | oauth_creds : { 93 | // TODO test that duplicate keys or tokens result in an error, not an overwrite 94 | consumer_key: "randConsumerKey", 95 | consumer_secret: "consumerSecret", 96 | token_secret: "tokenSecret", 97 | token: "randToken" 98 | } 99 | }, errLog); 100 | 101 | var feed = new follow.Feed({db : [db_host, db_name].join('/')}); 102 | feed.since = 0; 103 | feed.include_docs = true; 104 | feed.on('error', function() {}); 105 | var confirm_code, device_code; 106 | feed.on('change', function(change) { 107 | // wait for states to change 108 | // console.log("change", change) 109 | var doc = change.doc; 110 | switch (change.seq) { 111 | case 1: 112 | assert.ok(doc.type == "device") 113 | assert.ok(doc.state == "new") 114 | break; 115 | case 2: 116 | // when we see a new device we email the owner 117 | assert.ok(doc.type == "device") 118 | assert.ok(doc.state == "confirming") 119 | confirm_code = doc.confirm_code; 120 | device_code = doc.device_code; 121 | finish("send new device email"); 122 | // next test 123 | start("create a new channel"); 124 | db.insert(userTag({ 125 | type : "channel", 126 | state : "new", 127 | name : "My Channel Name" 128 | }), errLog); 129 | break; 130 | case 3: 131 | assert.ok(doc.type == "channel") 132 | assert.ok(doc.state == "new") 133 | break; 134 | case 4: 135 | assert.ok(doc.type == "channel") 136 | assert.ok(doc.state == "ready"); 137 | feed2 = new follow.Feed({db : doc.syncpoint, include_docs : true}) 138 | feed2.on('error', function() {}); 139 | feed2.on('change', function(change) { 140 | if (change.id == "description") { 141 | // assert that the db exists and it contains a description doc 142 | assert.ok(change.doc.name == "My Channel Name") 143 | finish("create a new channel"); 144 | } 145 | }); 146 | feed2.follow(); 147 | start("confirm device user") 148 | // we are testing that if you create a clicked confirm doc that matches the code, 149 | // your device will be activated 150 | db.insert({ 151 | type : "confirm", 152 | state : "clicked", 153 | confirm_code : confirm_code, 154 | device_code : device_code 155 | }, errLog); 156 | break; 157 | case 6: 158 | assert.ok(doc.type == "device") 159 | assert.ok(doc.state == "active") 160 | assert.ok(doc.confirm_code == confirm_code) 161 | var userDb = couch.use("_users"); 162 | userDb.get("org.couchdb.user:"+user_email, function(err, r, doc) { 163 | assert.ok(!err) 164 | assert.equal(doc.oauth.consumer_keys["randConsumerKey"], "consumerSecret"); 165 | assert.equal(doc.oauth.tokens["randToken"], "tokenSecret"); 166 | 167 | couch.request({ 168 | db : "_config", 169 | }, function(err, resp, data) { 170 | // these assertions can go away once 171 | // https://github.com/fdmanana/couchdb/compare/oauth_users_db 172 | // is merged. 173 | assert.ok(!err) 174 | assert.ok(data.oauth_consumer_secrets) 175 | assert.ok(data.oauth_token_users) 176 | assert.ok(data.oauth_token_secrets) 177 | assert.equal(data.oauth_consumer_secrets["randConsumerKey"], "consumerSecret"); 178 | assert.equal(data.oauth_token_users["randToken"], user_email); 179 | assert.equal(data.oauth_token_secrets["randToken"], "tokenSecret"); 180 | finish("confirm device user") 181 | }); 182 | }); 183 | break; 184 | } 185 | }); 186 | feed.follow(); 187 | }); 188 | 189 | 190 | 191 | -------------------------------------------------------------------------------- /cloud.js: -------------------------------------------------------------------------------- 1 | var docstate = require("./docstate") 2 | , nano = require("nano") 3 | ; 4 | 5 | var PUBLIC_HOST_URL = "http://localhost:5984/" 6 | 7 | function errLog(err, resp) { 8 | if (err) { 9 | if (err.message) { 10 | console.error(err.status_code, err.error, err.message) 11 | } else { 12 | console.error(err, resp) 13 | } 14 | } 15 | }; 16 | 17 | // todo move to nano 18 | // only works on urls like http://example.com/foobar 19 | function urlDb(url) { 20 | url = url.split("/"); 21 | db = url.pop(); 22 | return nano(url.join('/')).use(db); 23 | }; 24 | 25 | 26 | function sendEmail(address, code, cb) { 27 | console.warn("not actually sending an email", address, code) 28 | cb(false); 29 | } 30 | 31 | 32 | function ensureUserDoc(userDb, name, fun) { 33 | var user_doc_id = "org.couchdb.user:"+name; 34 | userDb.get(user_doc_id, function(err, r, userDoc) { 35 | if (err && err.status_code == 404) { 36 | fun(false, { 37 | _id : user_doc_id, 38 | type : "user", 39 | name : name, 40 | roles : [] 41 | }); 42 | } else { 43 | fun(false, userDoc); 44 | } 45 | }); 46 | } 47 | 48 | function setOAuthConfig(userDoc, id, creds, server, cb) { 49 | var rc = 0, ops = [ 50 | ["oauth_consumer_secrets", creds.consumer_key, creds.consumer_secret], 51 | ["oauth_token_users", creds.token, userDoc.name], 52 | ["oauth_token_secrets", creds.token, creds.token_secret] 53 | ]; 54 | for (var i=0; i < ops.length; i++) { 55 | var op = ops[i]; 56 | server.request({ 57 | method : "PUT", 58 | db : "_config", doc : op[0], att : op[1], body : op[2] 59 | }, function(err) { 60 | if (err) { 61 | cb(err) 62 | } else { 63 | rc += 1; 64 | if (rc == ops.length) { 65 | cb(false) 66 | } 67 | } 68 | }); 69 | }; 70 | } 71 | 72 | 73 | function applyOAuth(userDoc, id, creds) { 74 | userDoc.oauth = userDoc.oauth || { 75 | consumer_keys : {}, 76 | tokens : {}, 77 | }; 78 | userDoc.oauth.devices = userDoc.oauth.devices || {}; 79 | if (userDoc.oauth.consumer_keys[creds.consumer_key] || userDoc.oauth.tokens[creds.token]) { 80 | throw({error : "token_used", message : "device_id "+id}) 81 | } 82 | userDoc.oauth.devices[id] = [creds.consumer_key, creds.token]; 83 | userDoc.oauth.consumer_keys[creds.consumer_key] = creds.consumer_secret; 84 | userDoc.oauth.tokens[creds.token] = creds.token_secret; 85 | return userDoc; 86 | }; 87 | 88 | function handleDevices(control, db, server) { 89 | var userDb = server.use("_users"); 90 | control.safe("confirm","clicked", function(doc) { 91 | var confirm_code = doc.confirm_code; 92 | var device_code = doc.device_code; 93 | // load the device doc with confirm_code == code 94 | // TODO use a real view 95 | db.list({include_docs:true}, function(err, r, view) { 96 | var deviceDoc; 97 | view.rows.forEach(function(row) { 98 | if (row.doc.confirm_code && row.doc.confirm_code == confirm_code && 99 | row.doc.device_code && row.doc.device_code == device_code && 100 | row.doc.type && row.doc.type == "device") { 101 | deviceDoc = row.doc; 102 | } 103 | }); 104 | if (deviceDoc) { 105 | deviceDoc.state = "confirmed"; 106 | db.insert(deviceDoc, function(err, ok) { 107 | doc.state = "used"; 108 | db.insert(doc, errLog); 109 | }); 110 | } else { 111 | doc.state = "error"; 112 | doc.error = "no matching device"; 113 | db.insert(doc, errLog); 114 | } 115 | }); 116 | }); 117 | 118 | control.safe("device", "confirmed", function(deviceDoc) { 119 | // now we need to ensure the user exists and make sure the device has a delegate on it 120 | // move device_creds to user document, now the device can use them to auth as the user 121 | ensureUserDoc(userDb, deviceDoc.owner, function(err, userDoc) { 122 | userDoc = applyOAuth(userDoc, deviceDoc._id, deviceDoc.oauth_creds); 123 | userDb.insert(userDoc, function(err) { 124 | if (err) { 125 | errLog(err, deviceDoc.owner) 126 | } else { 127 | // set the config that we need with oauth user doc capability 128 | setOAuthConfig(userDoc, deviceDoc._id, deviceDoc.oauth_creds, server, function(err) { 129 | if (!err) { 130 | deviceDoc.state = "active"; 131 | db.insert(deviceDoc, errLog); 132 | } 133 | }); 134 | } 135 | }) 136 | }); 137 | }); 138 | 139 | control.unsafe("device", "new", function(doc) { 140 | var confirm_code = Math.random().toString().split('.').pop(); // todo better entropy 141 | sendEmail(doc.owner, confirm_code, function(err) { 142 | if (err) { 143 | errLog(err) 144 | } else { 145 | doc.state = "confirming"; 146 | doc.confirm_code = confirm_code; 147 | db.insert(doc, errLog); 148 | } 149 | }); 150 | }); 151 | 152 | }; 153 | 154 | 155 | function handleChannels(control, db, server) { 156 | control.safe("channel", "new", function(doc) { 157 | var db_name = "db-"+doc._id; 158 | if (doc["public"]) { 159 | errLog("PDI","please implement public databases") 160 | } else { 161 | server.db.create(db_name, function(err, resp) { 162 | if (err && err.code != 412) { 163 | // 412 means the db already exists, so we should still mark the channel ready. 164 | errLog(err, resp); 165 | } else { 166 | doc.state = "ready"; 167 | doc.syncpoint = PUBLIC_HOST_URL + db_name; 168 | db.insert(doc, errLog); 169 | } 170 | }); 171 | } 172 | }); 173 | 174 | control.safe("channel", "ready", function(doc) { 175 | var channel_db = urlDb(doc.syncpoint); 176 | channel_db.insert({ 177 | _id : 'description', 178 | name : doc.name 179 | }, errLog); 180 | }); 181 | }; 182 | 183 | exports.start = function(db_host, db_name) { 184 | var control = docstate.connect(db_host, db_name) 185 | , server = nano(db_host) 186 | , db = server.use(db_name) 187 | ; 188 | 189 | handleDevices(control, db, server); 190 | handleChannels(control, db, server); 191 | 192 | control.start(); 193 | }; 194 | 195 | // put device_creds as pending delegate on the user (w/ timestamps for expiry as these are created on the client's pace...) 196 | // (maybe create user*) 197 | // new doc can only be read by the user associated with the device creds, 198 | // so until the pending creds become active, the device can't connect. 199 | // email the user with link to confirm. 200 | // when the email goes, set device-doc.state = email-sent 201 | 202 | 203 | 204 | 205 | // let's talk about new backups 206 | 207 | 208 | 209 | 210 | --------------------------------------------------------------------------------