├── .gitignore ├── LICENSE.txt ├── README.md ├── javascript ├── README.md ├── demo.js ├── package.json └── repubsub.js ├── python ├── README.md ├── demo.py ├── repubsub.py ├── requirements.txt ├── setup.cfg └── setup.py └── ruby ├── Gemfile ├── README.md ├── demo.rb ├── repubsub.gemspec ├── repubsub.rb └── sender.rb /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | node_modules 3 | *.egg-info 4 | dist 5 | build 6 | *.gem 7 | Gemfile.lock 8 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 RethinkDB 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Publish Subscribe with RethinkDB 2 | 3 | This repo contains example libraries for doing publish-subscribe with 4 | RethinkDB [changefeeds](http://rethinkdb.com/docs/changefeeds/). 5 | 6 | There are 3 versions of the library: 7 | 8 | * [python](https://github.com/rethinkdb/example-pubsub/tree/master/python) 9 | * [ruby](https://github.com/rethinkdb/example-pubsub/tree/master/ruby) 10 | * [javascript](https://github.com/rethinkdb/example-pubsub/tree/master/javascript) 11 | -------------------------------------------------------------------------------- /javascript/README.md: -------------------------------------------------------------------------------- 1 | # repubsub.js # 2 | 3 | Repubsub is a publish-subscribe library built on top of 4 | [RethinkDB](http://rethinkdb.com). This is the javascript version of 5 | the library. There is a 6 | [full article](http://rethinkdb.com/docs/publish-subscribe/javascript/) 7 | describing this library in depth. 8 | 9 | ## Installation ## 10 | 11 | You'll need to install RethinkDB first. You can find instructions for 12 | that [on this page](http://rethinkdb.com/docs/install). 13 | 14 | To install the library, go into the source directory containing 15 | `package.json` and run: 16 | 17 | ```bash 18 | $ npm install 19 | ``` 20 | 21 | ## Usage ## 22 | 23 | To connect to an exchange, create a topic and publish to it: 24 | 25 | ```javascript 26 | var repubsub = require('repubsub'); 27 | 28 | var exchange = new repubsub.Exchange('exchangeName', {db: 'databaseName'}); 29 | 30 | var topic = exchange.topic('hi.there'); 31 | 32 | topic.publish("All subscribers to 'hi.there' will pick this up"); 33 | ``` 34 | 35 | To create a queue for listening for messages, the process is similar 36 | except you'll need to create a 37 | [ReQL](http://rethinkdb.com/docs/introduction-to-reql/) filter 38 | function: 39 | 40 | ```javascript 41 | function topicFilter(topic){ 42 | return topic.match('hi.*'); 43 | } 44 | var queue = exchange.queue(topicFilter); 45 | 46 | queue.subscribe(function(topic, message){ 47 | console.log('Received the message:', message); 48 | console.log('on the topic:', topic); 49 | }); 50 | ``` 51 | 52 | In addition, examples of usage can be found in the 53 | [demo.js](https://github.com/rethinkdb/example-pubsub/blob/master/javascript/demo.js) 54 | file. There is also an extensive description of how the library works 55 | and how to use it 56 | [here](http://rethinkdb.com/docs/publish-subscribe/javascript). 57 | 58 | ## Bugs ## 59 | 60 | Please report any bugs at our 61 | [github issue tracker](https://github.com/rethinkdb/example-pubsub/issues) 62 | -------------------------------------------------------------------------------- /javascript/demo.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var _ = require('underscore')._; 4 | var repubsub = require('./repubsub.js'); 5 | 6 | module.exports = { 7 | regexPublish: regexPublish, 8 | regexSubscribe: regexSubscribe, 9 | tagsPublish: tagsPublish, 10 | tagsSubscribe: tagsSubscribe, 11 | hierarchyPublish: hierarchyPublish, 12 | hierarchySubscribe: hierarchySubscribe, 13 | randomHierarchy: randomHierarchy, 14 | } 15 | 16 | 17 | // Publishes messages to a simple string topic 18 | function regexPublish(){ 19 | var exchange = new repubsub.Exchange('regex_demo', {db: 'repubsub'}); 20 | 21 | setInterval(function(){ 22 | var rnd = randomTopic(); 23 | var topicKey = rnd.category + '.' + rnd.chartype + '.' + rnd.character; 24 | var payload = _.sample(CATEGORIES[rnd.category]); 25 | 26 | console.log('Publishing on topic ' + topicKey + ': ' + payload); 27 | 28 | exchange.topic(topicKey).publish(payload); 29 | }, 500); 30 | } 31 | 32 | // Subscribes to messages on a topic that match a regex 33 | function regexSubscribe() { 34 | var exchange = new repubsub.Exchange('regex_demo', {db: 'repubsub'}); 35 | var rnd = randomTopic(); 36 | 37 | var maybeCharacter = _.sample([rnd.character, '(.+)']); 38 | var maybeChartype = _.sample([ 39 | rnd.chartype + '\\.' + maybeCharacter, 40 | '(.+)', 41 | ]); 42 | var topicRegex = '^' + rnd.category + '\\.' + maybeChartype + '$'; 43 | var queue = exchange.queue(function(topic){ 44 | return topic.match(topicRegex); 45 | }); 46 | 47 | var subMessage = "Subscribed to: " + topicRegex; 48 | printSubscription(subMessage); 49 | 50 | var i = 0; 51 | queue.subscribe(function(topic, payload){ 52 | if(i % 20 === 19){ 53 | // Reminder what we're subscribed to 54 | printSubscription(subMessage); 55 | } 56 | console.log("Received on " + topic + ": " + payload); 57 | i++; 58 | }); 59 | } 60 | 61 | // Publishes messages with an array of tags as a topic 62 | function tagsPublish(){ 63 | var exchange = new repubsub.Exchange('tags_demo', {db: 'repubsub'}); 64 | 65 | setInterval(function(){ 66 | // Get two random topics, remove duplicates, and sort them 67 | // Sorting ensures that if two topics consist of the same 68 | // tags, the same document in the database will be updated 69 | // This should result in 270 possible tag values 70 | var topicTags = _.union(_.values(randomTopic()), 71 | _.values(randomTopic())).sort(); 72 | var payload = _.sample(TEAMUPS.concat(EVENTS).concat(FIGHTS)); 73 | 74 | console.log('Publishing on tags #' + topicTags.join(' #')); 75 | console.log('\t' + payload); 76 | 77 | exchange.topic(topicTags).publish(payload); 78 | }, 500); 79 | } 80 | 81 | // Subscribes to messages that have specific tags in the topic 82 | function tagsSubscribe(){ 83 | var exchange = new repubsub.Exchange('tags_demo', {db: 'repubsub'}); 84 | 85 | var tags = _.sample(_.values(randomTopic()), 2); 86 | var queue = exchange.queue(function(topic){ 87 | return topic.contains.apply(tags); 88 | }); 89 | 90 | var subMessage = "Subscribed to messages with tags #" + tags.join(' #'); 91 | printSubscription(subMessage); 92 | 93 | var i = 0; 94 | queue.subscribe(function(topic, payload){ 95 | if(i % 10 === 9){ 96 | // Reminder what we're subscribed to 97 | printSubscription(subMessage); 98 | } 99 | console.log("Received message with tags: #" + topic.join(' #')); 100 | console.log("\t" + payload); 101 | i++; 102 | }); 103 | } 104 | 105 | // Publishes messages on a hierarchical topic 106 | function hierarchyPublish(){ 107 | var exchange = new repubsub.Exchange('hierarchy_demo', {db: 'repubsub'}); 108 | 109 | setInterval(function(){ 110 | var rnd = randomHierarchy(); 111 | var topicObj = rnd.topicObj, payload = rnd.payload; 112 | 113 | console.log('Publishing on a hierarchical topic:'); 114 | printHierarchy(rnd.topicObj); 115 | console.log(' -' + payload + '\n'); 116 | 117 | exchange.topic(rnd.topicObj).publish(rnd.payload); 118 | }, 500); 119 | } 120 | 121 | // Subscribes to messages on a hierarchical topic 122 | function hierarchySubscribe(){ 123 | var exchange = new repubsub.Exchange('hierarchy_demo', {db: 'repubsub'}); 124 | 125 | var rnd = randomTopic(); 126 | var queue = exchange.queue(function(topic){ 127 | return topic(rnd.category)(rnd.chartype).contains(rnd.character); 128 | }); 129 | var subMessage = "Subscribed to topic('" + rnd.category + "')('" + 130 | rnd.chartype + "').contains('" + rnd.character + "')"; 131 | printSubscription(subMessage); 132 | 133 | var i = 0; 134 | queue.subscribe(function(topic, payload){ 135 | if(i % 5 == 4){ 136 | printSubscription(subMessage); 137 | } 138 | console.log('Received message with topic:'); 139 | printHierarchy(topic); 140 | console.log(' -' + payload + '\n'); 141 | i++; 142 | }); 143 | } 144 | 145 | // Returns an object with the pieces of a random topic 146 | function randomTopic(){ 147 | var ret = { 148 | category: _.sample(_.keys(CATEGORIES)), 149 | chartype: _.sample(_.keys(CHARACTERS)), 150 | } 151 | ret.character = _.sample(CHARACTERS[ret.chartype]); 152 | 153 | return ret; 154 | } 155 | 156 | // Returns a random hierarchical topic 157 | function randomHierarchy(){ 158 | var topic = {}; 159 | var categories = []; 160 | _.chain(CATEGORIES).keys().sample(_.random(1,2)).each(function(category){ 161 | Array.prototype.push.apply(categories, CATEGORIES[category]); 162 | _.chain(CHARACTERS).keys().sample(_.random(1,2)).each(function(chartype){ 163 | _.sample(CHARACTERS[chartype], _.random(1,2)).forEach(function(character){ 164 | var cat = topic[category] || (topic[category] = {}); 165 | var ct = cat[chartype] || (cat[chartype] = []); 166 | ct.push(character); 167 | ct.sort(); 168 | }); 169 | }); 170 | }); 171 | return {topicObj: topic, payload: _.sample(categories)}; 172 | } 173 | 174 | // Prints a topic hierarchy nicely 175 | function printHierarchy(obj){ 176 | _.pairs(obj).forEach(function(ccPair){ 177 | var category = ccPair[0], chartypes = ccPair[1]; 178 | console.log(' ' + category); 179 | _.pairs(chartypes).forEach(function(ctPair){ 180 | var charType = ctPair[0], characters = ctPair[1]; 181 | console.log(" " + charType + ": " + characters.join(', ')); 182 | }); 183 | }); 184 | } 185 | 186 | // Prints a subscription reminder message 187 | function printSubscription(sub){ 188 | console.log(new Array(sub.length + 1).join('=')); 189 | console.log(sub); 190 | console.log(new Array(sub.length + 1).join('=')); 191 | console.log(); 192 | } 193 | 194 | // These are used in the demos 195 | 196 | var CHARACTERS = { 197 | superheroes: ['Batman', 'Superman', 'CaptainAmerica'], 198 | supervillains: ['Joker', 'LexLuthor', 'RedSkull'], 199 | sidekicks: ['Robin', 'JimmyOlsen', 'BuckyBarnes'], 200 | } 201 | 202 | var TEAMUPS = [ 203 | "You'll never guess who's teaming up", 204 | 'A completely one-sided fight between superheroes', 205 | 'Sidekick goes on rampage. Hundreds given parking tickets', 206 | 'Local politician warns of pairing between villains', 207 | 'Unexpected coalition teams up to take on opponents', 208 | ] 209 | 210 | var FIGHTS = [ 211 | 'A fight rages between combatants', 212 | 'Tussle between mighty foes continues', 213 | 'All out war in the streets between battling heroes', 214 | "City's greatest hero defeated!", 215 | "Villain locked in minimum security prison after defeat", 216 | ] 217 | 218 | var EVENTS = [ 219 | "Scientists accidentally thaw a T-Rex and release it", 220 | "Time vortex opens over downtown", 221 | "EMP turns out the lights. You'll never guess who turned them back on", 222 | "Inter-dimensional sludge released. Who can contain it?", 223 | "Super computer-virus disables all police cars. City helpless.", 224 | ] 225 | 226 | var CATEGORIES = { 227 | teamups: TEAMUPS, 228 | fights: FIGHTS, 229 | events: EVENTS, 230 | } 231 | 232 | function main(){ 233 | var argv = require('yargs') 234 | .usage('$0 {regex,tags,hierarchy} {publish,subscribe}\n' + 235 | '\n' + 236 | 'Demo for RethinkDB pub-sub\n' + 237 | '\n' + 238 | 'positional arguments:\n' + 239 | ' {regex,tags,hierarchy} Which demo to run\n' + 240 | ' {publish,subscribe} Whether to publish or subscribe' 241 | ) 242 | .demand(2) 243 | .check(function(argv){ 244 | var demos = ['regex', 'tags', 'hierarchy']; 245 | var pubOrSub = ['publish', 'subscribe']; 246 | if (!_.contains(demos, argv._[0])){ 247 | throw "First arg must be regex, tags or hierarchy"; 248 | } 249 | if (!_.contains(pubOrSub, argv._[1])){ 250 | throw "Second arg must publish or subscribe"; 251 | } 252 | return true; 253 | }) 254 | .argv; 255 | 256 | var demoName = argv._[0] + argv._[1].slice(0,1).toUpperCase() + argv._[1].slice(1); 257 | module.exports[demoName](); 258 | } 259 | 260 | main() 261 | -------------------------------------------------------------------------------- /javascript/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "repubsub", 3 | "version": "1.0.1", 4 | "description": "A publish-subscribe library using RethinkDB", 5 | "main": "repubsub.js", 6 | "files": ["repubsub.js", "demo.js"], 7 | "bin": {"repubsub-demo": "./demo.js"}, 8 | "readmeFilename": "README.md", 9 | "keywords": ["pubsub", "rethinkdb", "publish", "subscribe"], 10 | "homepage": "http://rethinkdb.com/docs/publish-subscribe/javascript/", 11 | "bugs": "https://github.com/rethinkdb/example-pubsub/issues", 12 | "license": "MIT", 13 | "repository": { 14 | "url": "https://github.com/rethinkdb/example-pubsub", 15 | "type": "git" 16 | }, 17 | "dependencies": { 18 | "rethinkdb": ">=1.13", 19 | "underscore": "~1.6.0", 20 | "yargs": "~1.3.1" 21 | }, 22 | "author": { 23 | "name": "Josh Kuhn", 24 | "email": "josh@rethinkdb.com" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /javascript/repubsub.js: -------------------------------------------------------------------------------- 1 | // Implementation of message queueing on top of RethinkDB changefeeds. 2 | 3 | // In this model, exchanges are databases, and documents are topics. The 4 | // current value of the topic in the database is just whatever the last 5 | // message sent happened to be. The document only exists to force 6 | // RethinkDB to generate change notifications for changefeed 7 | // subscribers. These notifications are the actual messages. 8 | 9 | // Internally, RethinkDB buffers changefeed notifications in a buffer per 10 | // client connection. These buffers are analogous to AMQP queues. This 11 | // has several benefits vs. (for example) having one document per 12 | // message: 13 | 14 | // * change notifications aren't created unless someone is subscribed 15 | // * notifications are deleted as soon as they're read from the buffer 16 | // * the notification buffers are implicitly ordered, so no sorting needs 17 | // to happen at the query level. 18 | 19 | // One large difference from existing message queues like RabbitMQ is 20 | // that there is no way to cause the change buffers to be persisted 21 | // across connections. Because of this, if the client sends a STOP 22 | // request to the changefeed, or disconnects, the queue is effectively 23 | // lost. Any messages on the queue are unrecoverable. 24 | 25 | var util = require('util'); 26 | var r = require('rethinkdb'); 27 | 28 | module.exports = { 29 | Exchange: Exchange, 30 | Topic: Topic, 31 | Queue: Queue 32 | }; 33 | 34 | // Represents a message exchange which messages can be sent to and 35 | // consumed from. Each exchange has an underlying RethinkDB table. 36 | function Exchange(name, connOpts){ 37 | this.db = connOpts.db = connOpts.db || 'test'; 38 | this.name = name; 39 | this.conn = null; 40 | this.table = r.table(name); 41 | this._asserted = false; 42 | 43 | // Bluebird's .bind ensures `this` inside our callbacks is the exchange 44 | this.promise = r.connect(connOpts).bind(this).then(function(conn){ 45 | this.conn = conn; 46 | }).catch(r.Error.RqlRuntimeError, function(err){ 47 | console.log(err.message); 48 | process.exit(1); 49 | }); 50 | } 51 | 52 | // Returns a topic in this exchange 53 | Exchange.prototype.topic = function(name){ 54 | return new Topic(this, name); 55 | }; 56 | 57 | // Returns a new queue on this exchange that will filter messages by 58 | // the given query 59 | Exchange.prototype.queue = function(filterFunc){ 60 | return new Queue(this, filterFunc); 61 | }; 62 | 63 | // The full ReQL query for a given filter function 64 | Exchange.prototype.fullQuery = function(filterFunc){ 65 | return this.table.changes()('new_val').filter(function(row){ 66 | return filterFunc(row('topic')); 67 | }); 68 | }; 69 | 70 | // Publish a message to this exchange on the given topic 71 | Exchange.prototype.publish = function(topicKey, payload){ 72 | return this.assertTable().then(function(){ 73 | var topIsObj = Object.prototype.toString.call( 74 | topicKey) === '[object Object]'; 75 | var topic = topIsObj ? r.literal(topicKey) : topicKey; 76 | return this.table.filter({ 77 | topic: topic 78 | }).update({ 79 | payload: payload, 80 | updated_on: r.now 81 | }).run(this.conn); 82 | }).then(function(updateResult){ 83 | // If the topic doesn't exist yet, insert a new document. Note: 84 | // it's possible someone else could have inserted it in the 85 | // meantime and this would create a duplicate. That's a risk we 86 | // take here. The consequence is that duplicated messages may 87 | // be sent to the consumer. 88 | if(updateResult.replaced === 0){ 89 | return this.table.insert({ 90 | topic: topicKey, 91 | payload: payload, 92 | updated_on: r.now() 93 | }).run(this.conn); 94 | }else{ 95 | return updateResult; 96 | } 97 | }); 98 | }; 99 | 100 | // Receives a callback that is called whenever a new message comes in 101 | // matching the filter function 102 | Exchange.prototype.subscribe = function(filterFunc, iterFunc){ 103 | return this.assertTable().then(function(){ 104 | return this.fullQuery(filterFunc).run(this.conn); 105 | }).then(function(cursor){ 106 | cursor.each(function(err, message){ 107 | iterFunc(message.topic, message.payload); 108 | }); 109 | }); 110 | 111 | 112 | return this.exchange.subscription(this.filterFunc).then(function(cursor){ 113 | cursor.each(iterFunc); 114 | }); 115 | }; 116 | 117 | // Ensures the table specified exists and has the correct primary_key 118 | // and durability settings 119 | Exchange.prototype.assertTable = function(){ 120 | return this.promise.then(function(){ 121 | if (this._asserted){ 122 | return; 123 | } 124 | 125 | return r.dbCreate(this.db).run(this.conn).bind(this).finally(function(){ 126 | return r.db(this.db).tableCreate(this.name).run(this.conn).bind(this); 127 | }).catch(r.Error.RqlRuntimeError, function(err){ 128 | if(err.msg.indexOf('already exists') === -1){ 129 | throw err; 130 | } 131 | }).then(function(){ 132 | this._asserted = true; 133 | }); 134 | }); 135 | }; 136 | 137 | // Represents a topic that may be published to 138 | function Topic(exchange, topicKey) { 139 | this.exchange = exchange; 140 | this.key = topicKey; 141 | } 142 | 143 | // Publish a payload to the current topic 144 | Topic.prototype.publish = function(payload){ 145 | return this.exchange.publish(this.key, payload); 146 | }; 147 | 148 | // A queue that filters messages in the exchange 149 | function Queue(exchange, filterFunc) { 150 | this.exchange = exchange; 151 | this.filterFunc = filterFunc; 152 | } 153 | 154 | // Returns the full ReQL query for this queue 155 | Queue.prototype.fullQuery = function(){ 156 | return this.exchange.fullQuery(this.filterFunc); 157 | }; 158 | 159 | // Subscribe to messages from this queue's subscriptions 160 | Queue.prototype.subscribe = function(iterFunc){ 161 | return this.exchange.subscribe(this.filterFunc, iterFunc); 162 | }; 163 | -------------------------------------------------------------------------------- /python/README.md: -------------------------------------------------------------------------------- 1 | # repubsub.py # 2 | 3 | Repubsub is a publish-subscribe library built on top of 4 | [RethinkDB](http://rethinkdb.com). This is the python version of 5 | the library. There is a 6 | [full article](http://rethinkdb.com/docs/publish-subscribe/python/) 7 | describing this library in depth. 8 | 9 | ## Installation ## 10 | 11 | You'll need to install RethinkDB first. You can find instructions for 12 | that [on this page](http://rethinkdb.com/docs/install). 13 | 14 | To install the library, go into the source directory containing 15 | `setup.py` and run: 16 | 17 | ```bash 18 | $ python setup.py install 19 | ``` 20 | 21 | ## Usage ## 22 | 23 | To connect to an exchange, create a topic and publish to it: 24 | 25 | ```python 26 | import repubsub 27 | 28 | exchange = repubsub.Exchange('exchange_name', db='database_name') 29 | 30 | topic = exchange.topic('hi.there') 31 | 32 | topic.publish("All subscribers to 'hi.there' will pick this up") 33 | ``` 34 | 35 | To create a queue for listening for messages, the process is similar 36 | except you'll need to create a 37 | [ReQL](http://rethinkdb.com/docs/introduction-to-reql/) filter 38 | function: 39 | 40 | ```python 41 | queue = exchange.queue(lambda topic: topic.match('hi.*')) 42 | 43 | for topic, message in queue.subscription(): 44 | print 'Received the message:', message 45 | print 'on the topic:', topic 46 | ``` 47 | 48 | In addition, examples of usage can be found in the [demo.py](https://github.com/rethinkdb/example-pubsub/blob/master/python/demo.py) 49 | file. There is also an extensive description of how the library works 50 | and how to use it 51 | [here](http://rethinkdb.com/docs/publish-subscribe/python). 52 | 53 | ## Bugs ## 54 | 55 | Please report any bugs at our 56 | [github issue tracker](https://github.com/rethinkdb/example-pubsub/issues) 57 | -------------------------------------------------------------------------------- /python/demo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | import sys 4 | import logging 5 | import argparse 6 | import time 7 | import random 8 | from random import randint 9 | 10 | import repubsub 11 | import rethinkdb as r 12 | 13 | 14 | def main(): 15 | '''Parse command line args and use them to run the corresponding 16 | function''' 17 | parser = argparse.ArgumentParser( 18 | description='Demo for RethinkDB pub-sub') 19 | parser.add_argument( 20 | 'demo', 21 | type=str, 22 | help='Which demo to run', 23 | choices=['regex', 'tags', 'hierarchy'], 24 | ) 25 | parser.add_argument( 26 | 'pub_or_sub', 27 | type=str, 28 | help="Whether to publish or subscribe", 29 | choices=['publish', 'subscribe'], 30 | ) 31 | args = parser.parse_args() 32 | 33 | globals()['{0.demo}_{0.pub_or_sub}'.format(args)]() 34 | 35 | 36 | def regex_publish(): 37 | '''Publishes messages to a simple string topic''' 38 | 39 | exchange = repubsub.Exchange('regex_demo', db='repubsub') 40 | 41 | while True: 42 | category, chartype, character = random_topic() 43 | topic_key = '{category}.{chartype}.{character}'.format( 44 | category=category, chartype=chartype, character=character) 45 | payload = random.choice(CATEGORIES[category]) 46 | 47 | print 'Publishing on topic', topic_key, ':', payload 48 | 49 | exchange.topic(topic_key).publish(payload) 50 | time.sleep(0.5) 51 | 52 | 53 | def regex_subscribe(): 54 | '''Subscribes to messages on a topic that match a regex''' 55 | 56 | exchange = repubsub.Exchange('regex_demo', db='repubsub') 57 | 58 | category, chartype, character = random_topic() 59 | topic_regex = r'^{category}\.{chartype_character}$'.format( 60 | # This avoids regexes like 'fights\.(.+)\.Batman' where the 61 | # chartype can only be one thing. 62 | chartype_character = random.choice([ 63 | chartype + '\.' + random.choice([character, '(.+)']), 64 | '(.+)', 65 | ]), 66 | category = random.choice([category, '(.+)']), 67 | ) 68 | reql_filter = lambda topic: topic.match(topic_regex) 69 | queue = exchange.queue(reql_filter) 70 | 71 | sub_message = 'Subscribed to: %s' % topic_regex 72 | print_subscription(sub_message) 73 | 74 | for i, (topic, payload) in enumerate(queue.subscription()): 75 | if i % 20 == 19: 76 | # Reminder what we're subscribed to 77 | print_subscription(sub_message) 78 | 79 | print 'Received on', topic, ':', payload 80 | 81 | 82 | def tags_publish(): 83 | '''Publishes messages with an array of tags as a topic''' 84 | 85 | exchange = repubsub.Exchange('tags_demo', db='repubsub') 86 | 87 | while True: 88 | # Get two random topics, remove duplicates, and sort them 89 | # Sorting ensures that if two topics consist of the same 90 | # tags, the same document in the database will be updated 91 | # This should result in 270 possible tag values 92 | topic_tags = sorted(set(random_topic() + random_topic())) 93 | payload = random.choice(TEAMUPS + EVENTS + FIGHTS) 94 | 95 | print 'Publishing on tags #{}'.format(' #'.join(topic_tags)) 96 | print '\t', payload 97 | 98 | exchange.topic(topic_tags).publish(payload) 99 | time.sleep(0.5) 100 | 101 | 102 | def tags_subscribe(): 103 | '''Subscribes to messages that have specific tags in the topic''' 104 | 105 | exchange = repubsub.Exchange('tags_demo', db='repubsub') 106 | 107 | tags = random.sample(random_topic(), 2) 108 | reql_filter = lambda topic: topic.contains(*tags) 109 | queue = exchange.queue(reql_filter) 110 | 111 | sub_message = 'Subscribed to messages with tags: #%s' % ' #'.join(tags) 112 | print_subscription(sub_message) 113 | 114 | for i, (topic_tags, payload) in enumerate(queue.subscription()): 115 | if i % 10 == 9: 116 | # Reminder what we're subscribed to 117 | print_subscription(sub_message) 118 | 119 | print 'Received message with tags: #{}'.format(' #'.join(topic_tags)) 120 | print '\t', payload 121 | print 122 | 123 | 124 | def hierarchy_publish(): 125 | '''Publishes messages on a hierarchical topic''' 126 | 127 | exchange = repubsub.Exchange('hierarchy_demo', db='repubsub') 128 | 129 | while True: 130 | topic_key, payload = random_hierarchy() 131 | 132 | print 'Publishing on hierarchical topic:' 133 | print_hierarchy(topic_key) 134 | print ' -', payload 135 | print 136 | 137 | exchange.topic(topic_key).publish(payload) 138 | time.sleep(0.5) 139 | 140 | 141 | def hierarchy_subscribe(): 142 | '''Subscribes to messages on a hierarchical topic''' 143 | 144 | exchange = repubsub.Exchange('hierarchy_demo', db='repubsub') 145 | 146 | category, chartype, character = random_topic() 147 | reql_filter = lambda topic: topic[category][chartype].contains(character) 148 | queue = exchange.queue(reql_filter) 149 | 150 | sub_message = 'Subscribed to topic: [%r][%r].contains(%r)' % ( 151 | category, chartype, character) 152 | print_subscription(sub_message) 153 | 154 | for i, (topic, payload) in enumerate(queue.subscription()): 155 | if i % 5 == 4: 156 | # Reminder what we're subscribed to 157 | print_subscription(sub_message) 158 | 159 | print 'Received message with topic:' 160 | print_hierarchy(topic) 161 | print ' -', payload, '\n' 162 | 163 | 164 | def random_topic(): 165 | '''Returns the pieces of a random topic''' 166 | category = random.choice(CATEGORIES.keys()) 167 | chartype = random.choice(CHARACTERS.keys()) 168 | character = random.choice(CHARACTERS[chartype]) 169 | return category, chartype, character 170 | 171 | 172 | def random_hierarchy(): 173 | '''Returns a random hierarchical topic''' 174 | topic = {} 175 | categories = [] 176 | for category in random.sample(CATEGORIES.keys(), randint(1, 2)): 177 | categories.extend(CATEGORIES[category]) 178 | for chartype in random.sample(CHARACTERS.keys(), randint(1, 2)): 179 | for character in random.sample(CHARACTERS[chartype], randint(1, 2)): 180 | characters = topic.setdefault( 181 | category, {}).setdefault(chartype, []) 182 | characters.append(character) 183 | characters.sort() 184 | return topic, random.choice(categories) 185 | 186 | 187 | def print_hierarchy(h): 188 | '''Prints a topic hierarchy nicely''' 189 | for category, chartypes in h.iteritems(): 190 | print ' ', category, ':' 191 | for chartype, characters in chartypes.iteritems(): 192 | print ' ', chartype, ':', ', '.join(characters) 193 | 194 | def print_subscription(sub): 195 | '''Prints a subscription reminder message''' 196 | print '=' * len(sub) 197 | print sub 198 | print '=' * len(sub) 199 | print 200 | 201 | 202 | # These are used in the demos 203 | CHARACTERS = { 204 | 'superheroes': ['Batman', 'Superman', 'CaptainAmerica'], 205 | 'supervillains': ['Joker', 'Lex Luthor', 'RedSkull'], 206 | 'sidekicks': ['Robin', 'JimmyOlsen', 'BuckyBarnes'], 207 | } 208 | 209 | TEAMUPS = [ 210 | "You'll never guess who's teaming up", 211 | 'A completely one-sided fight between superheroes', 212 | 'Sidekick goes on rampage. Hundreds given parking tickets', 213 | 'Local politician warns of pairing between villains', 214 | 'Unexpected coalition teams up to take on opponents', 215 | ] 216 | 217 | FIGHTS = [ 218 | 'A fight rages between combatants', 219 | 'Tussle between mighty foes continues', 220 | 'All out war in the streets between battling heroes', 221 | "City's greatest hero defeated!", 222 | "Villain locked in minimum security prison after defeat", 223 | ] 224 | 225 | EVENTS = [ 226 | "Scientists accidentally thaw a T-Rex and release it", 227 | "Time vortex opens over downtown", 228 | "EMP turns out the lights. You'll never guess who turned them back on", 229 | "Inter-dimensional sludge released. Who can contain it?", 230 | "Super computer-virus disables all police cars. City helpless.", 231 | ] 232 | 233 | CATEGORIES = { 234 | 'teamups': TEAMUPS, 235 | 'fights': FIGHTS, 236 | 'events': EVENTS, 237 | } 238 | 239 | if __name__ == '__main__': 240 | main() 241 | -------------------------------------------------------------------------------- /python/repubsub.py: -------------------------------------------------------------------------------- 1 | '''Implementation of message queueing on top of RethinkDB changefeeds. 2 | 3 | In this model, exchanges are databases, and documents are topics. The 4 | current value of the topic in the database is just whatever the last 5 | message sent happened to be. The document only exists to force 6 | RethinkDB to generate change notifications for changefeed 7 | subscribers. These notifications are the actual messages. 8 | 9 | Internally, RethinkDB buffers changefeed notifications in a buffer per 10 | client connection. These buffers are analogous to AMQP queues. This 11 | has several benefits vs. (for example) having one document per 12 | message: 13 | 14 | * change notifications aren't created unless someone is subscribed 15 | * notifications are deleted as soon as they're read from the buffer 16 | * the notification buffers are implicitly ordered, so no sorting needs 17 | to happen at the query level. 18 | 19 | One large difference from existing message queues like RabbitMQ is 20 | that there is no way to cause the change buffers to be persisted 21 | across connections. Because of this, if the client sends a STOP 22 | request to the changefeed, or disconnects, the queue is effectively 23 | lost. Any messages on the queue are unrecoverable. 24 | ''' 25 | 26 | import random 27 | 28 | from rethinkdb import connect 29 | import rethinkdb as r 30 | 31 | 32 | __all__ = ['connect', 'Exchange', 'Topic', 'Queue'] 33 | 34 | 35 | class Exchange(object): 36 | '''Represents a message exchange which messages can be sent to and 37 | consumed from. Each exchange has an underlying RethinkDB table.''' 38 | 39 | def __init__(self, name, **kwargs): 40 | self.db = kwargs.setdefault('db', 'test') 41 | self.name = name 42 | self.conn = r.connect(**kwargs) 43 | self.table = r.table(name) 44 | self._asserted = False 45 | 46 | def __repr__(self): 47 | return 'Exchange({.name})'.format(self) 48 | 49 | def topic(self, topic_key): 50 | '''Returns a topic in this exchange with the given key''' 51 | return Topic(self, topic_key) 52 | 53 | def queue(self, binding_query=None): 54 | '''Returns a new queue on this exchange that will filter messages by 55 | the given query.''' 56 | return Queue(self, binding_query) 57 | 58 | def full_query(self, filter_func): 59 | '''Returns the full ReQL query for a given filter function''' 60 | return self.table.changes()['new_val'].filter( 61 | lambda row: filter_func(row['topic'])) 62 | 63 | def publish(self, topic_key, payload): 64 | '''Publish a message to this exchange on the given topic''' 65 | self.assert_table() 66 | 67 | # first try to just update an existing document 68 | result = self.table.filter({ 69 | 'topic': topic_key 70 | if not isinstance(topic_key, dict) 71 | else r.literal(topic_key), 72 | }).update({ 73 | 'payload': payload, 74 | 'updated_on': r.now(), 75 | }).run(self.conn) 76 | 77 | # If the topic doesn't exist yet, insert a new document. Note: 78 | # it's possible someone else could have inserted it in the 79 | # meantime and this would create a duplicate. That's a risk we 80 | # take here. The consequence is that duplicated messages may 81 | # be sent to the consumer. 82 | if not result['replaced']: 83 | result = self.table.insert({ 84 | 'topic': topic_key, 85 | 'payload': payload, 86 | 'updated_on': r.now(), 87 | }).run(self.conn) 88 | 89 | def subscription(self, filter_func): 90 | '''Generator of messages from the exchange with topics matching the 91 | given filter function 92 | ''' 93 | self.assert_table() 94 | 95 | for message in self.full_query(filter_func).run(self.conn): 96 | yield message['topic'], message['payload'] 97 | 98 | def assert_table(self): 99 | '''Ensures the table specified exists and has the correct 100 | primary_key and durability settings''' 101 | if self._asserted: 102 | return 103 | try: 104 | # Assert the db into existence 105 | r.db_create(self.conn.db).run(self.conn) 106 | except r.RqlRuntimeError as rre: 107 | if 'already exists' not in rre.message: 108 | raise 109 | try: 110 | # We set durability to soft because we don't actually care 111 | # if the write is confirmed on disk, we just want the 112 | # change notification (i.e. the message on the queue) to 113 | # be generated. 114 | r.table_create( 115 | self.name, 116 | durability='soft', 117 | ).run(self.conn) 118 | except r.RqlRuntimeError as rre: 119 | if 'already exists' not in rre.message: 120 | raise 121 | self._asserted = True 122 | 123 | 124 | class Topic(object): 125 | '''Represents a topic that may be published to. 126 | ''' 127 | 128 | def __init__(self, exchange, topic_key): 129 | self.key = topic_key 130 | self.exchange = exchange 131 | 132 | def publish(self, payload): 133 | '''Publish a payload to the current topic''' 134 | self.exchange.publish(self.key, payload) 135 | 136 | def __repr__(self): 137 | return 'Topic({})'.format(self.key) 138 | 139 | 140 | class Queue(object): 141 | '''A queue that filters messages in the exchange''' 142 | 143 | def __init__(self, exchange, filter_func): 144 | self.exchange = exchange 145 | self.filter_func = filter_func 146 | 147 | def subscription(self): 148 | '''Returns a generator that returns messages from this queue's 149 | subscriptions''' 150 | return self.exchange.subscription(self.filter_func) 151 | 152 | def full_query(self): 153 | '''Returns the full ReQL query for this queue''' 154 | return self.exchange.full_query(self.filter_func) 155 | 156 | def __repr__(self): 157 | return 'Queue({})'.format(r.expr(self.filter_func)) 158 | -------------------------------------------------------------------------------- /python/requirements.txt: -------------------------------------------------------------------------------- 1 | rethinkdb>=1.13,<1.14 2 | -------------------------------------------------------------------------------- /python/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md -------------------------------------------------------------------------------- /python/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | setup( 5 | name='repubsub', 6 | version='1.0.1', 7 | packages=['.'], 8 | description='A publish-subscribe library using RethinkDB', 9 | license='MIT', 10 | author='Josh Kuhn', 11 | author_email='josh@rethinkdb.com', 12 | url='http://rethinkdb.com/docs/publish-subscribe/python/', 13 | keywords=['pubsub', 'rethinkdb', 'publish', 'subscribe'], 14 | install_requires=['rethinkdb>=1.13'], 15 | classifiers=[ 16 | 'Development Status :: 2 - Beta', 17 | 'Intended Audience :: Developers', 18 | 'License :: OSI Approved :: MIT License', 19 | 'Programming Language :: Python :: 2.7', 20 | ], 21 | ) 22 | -------------------------------------------------------------------------------- /ruby/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gem 'rethinkdb', '>=1.13' -------------------------------------------------------------------------------- /ruby/README.md: -------------------------------------------------------------------------------- 1 | # repubsub.rb # 2 | 3 | Repubsub is a publish-subscribe library built on top of 4 | [RethinkDB](http://rethinkdb.com). This is the ruby version of 5 | the library. There is a 6 | [full article](http://rethinkdb.com/docs/publish-subscribe/ruby/) 7 | describing this library in depth. 8 | 9 | ## Installation ## 10 | 11 | You'll need to install RethinkDB first. You can find instructions for 12 | that [on this page](http://rethinkdb.com/docs/install). 13 | 14 | To install the library, go into the source directory containing 15 | `repubsub.gemspec` and run: 16 | 17 | ```bash 18 | $ bundle install 19 | ``` 20 | 21 | ## Usage ## 22 | 23 | To connect to an exchange, create a topic and publish to it: 24 | 25 | ```ruby 26 | require 'repubsub' 27 | 28 | exchange = Repubsub::Exchange.new(:exchange_name, :db => :database_name) 29 | 30 | topic = exchange.topic('hi.there') 31 | 32 | topic.publish("All subscribers to 'hi.there' will pick this up") 33 | ``` 34 | 35 | To create a queue for listening for messages, the process is similar 36 | except you'll need to provide a 37 | [ReQL](http://rethinkdb.com/docs/introduction-to-reql/) filter 38 | block: 39 | 40 | ```ruby 41 | queue = exchange.queue{|topic| topic.match('hi.*')} 42 | 43 | queue.subscription.each do |topic, message| 44 | puts "Received the message #{message}" 45 | puts "on the topic: #{topic}" 46 | end 47 | ``` 48 | 49 | In addition, examples of usage can be found in the 50 | [demo.rb](https://github.com/rethinkdb/example-pubsub/blob/master/ruby/demo.rb) 51 | file. There is also an extensive description of how the library works 52 | and how to use it [here](http://rethinkdb.com/docs/publish-subscribe/ruby). 53 | 54 | ## Bugs ## 55 | 56 | Please report any bugs at our 57 | [github issue tracker](https://github.com/rethinkdb/example-pubsub/issues) 58 | -------------------------------------------------------------------------------- /ruby/demo.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'optparse' 4 | 5 | require_relative 'repubsub' 6 | 7 | # Publishes messages to a simple string topic 8 | def regex_publish 9 | exchange = Repubsub::Exchange.new(:regex_demo, :db => :repubsub) 10 | 11 | loop do 12 | category, chartype, character = random_topic 13 | topic_key = "#{category}.#{chartype}.#{character}" 14 | payload = $CATEGORIES[category].sample 15 | 16 | puts "Publishing on topic #{topic_key}: #{payload}" 17 | 18 | exchange.topic(topic_key).publish(payload) 19 | sleep(0.5) 20 | end 21 | end 22 | 23 | # Subscribes to messages on a topic that match a regex 24 | def regex_subscribe 25 | exchange = Repubsub::Exchange.new(:regex_demo, :db => :repubsub) 26 | 27 | category, chartype, character = random_topic 28 | maybe_character = [character, '(.+)'].sample 29 | maybe_chartype = ["#{chartype}\.#{maybe_character}", '(.+)'].sample 30 | topic_regex = "^#{category}\.#{maybe_chartype}$" 31 | queue = exchange.queue{|topic| topic.match(topic_regex)} 32 | 33 | sub_message = "Subscribed to: #{topic_regex}" 34 | print_subscription(sub_message) 35 | 36 | queue.subscription.with_index do |(topic, payload), i| 37 | if i % 20 == 19 38 | # Reminder what we're subscribed to 39 | print_subscription(sub_message) 40 | end 41 | puts "Received on #{topic}: #{payload}" 42 | end 43 | end 44 | 45 | 46 | # Publishes messages with an array of tags as a topic 47 | def tags_publish 48 | exchange = Repubsub::Exchange.new(:tags_demo, :db => :repubsub) 49 | 50 | loop do 51 | # Get two random topics, remove duplicates, and sort them 52 | # Sorting ensures that if two topics consist of the same 53 | # tags, the same document in the database will be updated 54 | # This should result in 270 possible tag values 55 | topic_tags = (random_topic + random_topic).to_set.sort 56 | payload = ($TEAMUPS + $EVENTS + $FIGHTS).sample 57 | 58 | puts "Publishing on tags ##{topic_tags.join(' #')}", "\t #{payload}" 59 | 60 | exchange.topic(topic_tags).publish(payload) 61 | sleep(0.5) 62 | end 63 | end 64 | 65 | # Subscribes to messages that have specific tags in the topic 66 | def tags_subscribe 67 | exchange = Repubsub::Exchange.new(:tags_demo, :db => :repubsub) 68 | 69 | tags = random_topic.sample(2) 70 | queue = exchange.queue{|topic| topic.contains(*tags)} 71 | 72 | sub_message = "Subscribed to messages with tags ##{tags.join(' #')}" 73 | print_subscription(sub_message) 74 | 75 | queue.subscription.with_index do |(topic, payload), i| 76 | if i % 10 == 9 77 | # Reminder what we're subscribed to 78 | print_subscription(sub_message) 79 | end 80 | puts "Received message with tags: ##{topic.join(' #')}", "\t #{payload}\n" 81 | end 82 | end 83 | 84 | # Publishes messages on a hierarchical topic 85 | def hierarchy_publish 86 | exchange = Repubsub::Exchange.new(:hierarchy_demo, :db => :repubsub) 87 | 88 | loop do 89 | topic_hash, payload = random_hierarchy 90 | 91 | puts "Publishing on a hierarchical topic:" 92 | print_hierarchy(topic_hash) 93 | puts " - #{payload}", '' 94 | 95 | exchange.topic(topic_hash).publish(payload) 96 | sleep(0.5) 97 | end 98 | end 99 | 100 | # Subscribes to messages on a hierarchical topic 101 | def hierarchy_subscribe 102 | exchange = Repubsub::Exchange.new(:hierarchy_demo, :db => :repubsub) 103 | 104 | category, chartype, character = random_topic 105 | queue = exchange.queue{|topic| topic[category][chartype].contains(character)} 106 | 107 | sub_message = "Subscribed to topic: ['#{category}']['#{chartype}']\ 108 | .contains('#{character}')" 109 | print_subscription(sub_message) 110 | 111 | queue.subscription.with_index do |(topic,payload),i| 112 | if i % 5 == 4 113 | # Reminder what we're subscribed to 114 | print_subscription(sub_message) 115 | end 116 | 117 | puts 'Received message with topic:' 118 | print_hierarchy(topic) 119 | puts " - #{payload}", '' 120 | end 121 | end 122 | 123 | # Returns the pieces of a random topic 124 | def random_topic 125 | category = $CATEGORIES.keys.sample 126 | chartype = $CHARACTERS.keys.sample 127 | character = $CHARACTERS[chartype].sample 128 | return category, chartype, character 129 | end 130 | 131 | # Returns a random hierarchical topic 132 | def random_hierarchy 133 | # Default value is a Hash with a default value of an Array 134 | topic = Hash.new{|hash,key| hash[key] = Hash.new{|h,k| h[k] = []}} 135 | categories = [] 136 | $CATEGORIES.keys.sample(1 + rand(2)).each do |category| 137 | categories.concat($CATEGORIES[category]) 138 | $CHARACTERS.keys.sample(1 + rand(2)).each do |chartype| 139 | $CHARACTERS[chartype].sample(1 + rand(2)).each do |character| 140 | characters = topic[category][chartype] << character 141 | characters.sort! 142 | end 143 | end 144 | end 145 | return topic, categories.sample 146 | end 147 | 148 | # Prints a topic hierarchy nicely 149 | def print_hierarchy(hash) 150 | hash.each_pair do |category,chartypes| 151 | puts " #{category}:" 152 | chartypes.each_pair do |chartype,characters| 153 | puts " #{chartype}: #{characters.join(', ')}" 154 | end 155 | end 156 | end 157 | 158 | # Prints a subscription reminder message 159 | def print_subscription(sub) 160 | puts '=' * sub.length 161 | puts sub 162 | puts '=' * sub.length 163 | puts 164 | end 165 | 166 | # These are used in the demos 167 | $CHARACTERS = { 168 | :superheroes => [:Batman, :Superman, :CaptainAmerica], 169 | :supervillains => [:Joker, :LexLuthor, :RedSkull], 170 | :sidekicks => [:Robin, :JimmyOlsen, :BuckyBarnes], 171 | } 172 | 173 | $TEAMUPS = [ 174 | "You'll never guess who's teaming up", 175 | 'A completely one-sided fight between superheroes', 176 | 'Sidekick goes on rampage. Hundreds given parking tickets', 177 | 'Local politician warns of pairing between villains', 178 | 'Unexpected coalition teams up to take on opponents', 179 | ] 180 | 181 | $FIGHTS = [ 182 | 'A fight rages between combatants', 183 | 'Tussle between mighty foes continues', 184 | 'All out war in the streets between battling heroes', 185 | "City's greatest hero defeated!", 186 | "Villain locked in minimum security prison after defeat", 187 | ] 188 | 189 | $EVENTS = [ 190 | "Scientists accidentally thaw a T-Rex and release it", 191 | "Time vortex opens over downtown", 192 | "EMP turns out the lights. You'll never guess who turned them back on", 193 | "Inter-dimensional sludge released. Who can contain it?", 194 | "Super computer-virus disables all police cars. City helpless.", 195 | ] 196 | 197 | $CATEGORIES = { 198 | :teamups => $TEAMUPS, 199 | :fights => $FIGHTS, 200 | :events => $EVENTS, 201 | } 202 | 203 | OptionParser.new do |opts| 204 | opts.banner = <<-BANNER 205 | usage: #{$PROGRAM_NAME} [-h] {regex,tags,hierarchy} {publish,subscribe} 206 | 207 | Demo for RethinkDB pub-sub 208 | 209 | positional arguments: 210 | {regex,tags,hierarchy} 211 | Which demo to run 212 | {publish,subscribe} Whether to publish or subscribe 213 | 214 | optional arguments: 215 | -h, --help show this help message and exit 216 | BANNER 217 | 218 | opts.on('-h') {|b| puts opts.banner; exit } 219 | opts.parse! 220 | demo = ARGV.shift 221 | demos = ['regex', 'tags', 'hierarchy'] 222 | pub_or_sub = ARGV.shift 223 | pubsubs = ['publish', 'subscribe'] 224 | if not demos.include?(demo) 225 | puts("invalid choice: '#{demo}' (choose from regex, tags, hierarchy)") 226 | puts opts.banner 227 | exit 228 | end 229 | if not pubsubs.include?(pub_or_sub) 230 | puts("invalid choice: '#{pub_or_sub}' (choose from publish, subscribe)") 231 | puts opts.banner 232 | exit 233 | end 234 | 235 | send("#{demo}_#{pub_or_sub}") 236 | end 237 | -------------------------------------------------------------------------------- /ruby/repubsub.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | s.name = 'repubsub' 3 | s.version = '1.0.1' 4 | s.summary = "pubsub on RethinkDB" 5 | s.summary = "A publish-subscribe library using RethinkDB" 6 | s.homepage = "http://rethinkdb.com/docs/publish-subscribe/ruby/" 7 | s.files = ["repubsub.rb"] 8 | s.authors = ["Josh Kuhn"] 9 | s.email = "josh@rethinkdb.com" 10 | s.license = "MIT" 11 | s.add_runtime_dependency 'rethinkdb', '>= 1.13' 12 | end 13 | -------------------------------------------------------------------------------- /ruby/repubsub.rb: -------------------------------------------------------------------------------- 1 | # Implementation of message queueing on top of RethinkDB changefeeds. 2 | 3 | # In this model, exchanges are databases, and documents are topics. The 4 | # current value of the topic in the database is just whatever the last 5 | # message sent happened to be. The document only exists to force 6 | # RethinkDB to generate change notifications for changefeed 7 | # subscribers. These notifications are the actual messages. 8 | 9 | # Internally, RethinkDB buffers changefeed notifications in a buffer per 10 | # client connection. These buffers are analogous to AMQP queues. This 11 | # has several benefits vs. (for example) having one document per 12 | # message: 13 | 14 | # * change notifications aren't created unless someone is subscribed 15 | # * notifications are deleted as soon as they're read from the buffer 16 | # * the notification buffers are implicitly ordered, so no sorting needs 17 | # to happen at the query level. 18 | 19 | # One large difference from existing message queues like RabbitMQ is 20 | # that there is no way to cause the change buffers to be persisted 21 | # across connections. Because of this, if the client sends a STOP 22 | # request to the changefeed, or disconnects, the queue is effectively 23 | # lost. Any messages on the queue are unrecoverable. 24 | 25 | require 'set' 26 | 27 | require 'rethinkdb' 28 | include RethinkDB::Shortcuts 29 | 30 | module Repubsub 31 | 32 | # Represents a message exchange which messages can be sent to and 33 | # consumed from. Each exchange has an underlying RethinkDB table. 34 | class Exchange 35 | def initialize(name, opts={}) 36 | @db = opts[:db] = opts.fetch(:db, :test) 37 | @name = name.to_s 38 | @conn = r.connect(opts) 39 | @table = r.table(name) 40 | @asserted = false 41 | end 42 | 43 | # Returns a topic in this exchange 44 | def topic(name) 45 | Topic.new(self, name) 46 | end 47 | 48 | # Returns a new queue on this exchange that will filter messages 49 | # by the given query 50 | def queue(&filter_func) 51 | Queue.new(self, &filter_func) 52 | end 53 | 54 | # The full ReQL query for a given filter function 55 | def full_query(filter_func) 56 | @table.changes[:new_val].filter{|row| filter_func.call(row[:topic])} 57 | end 58 | 59 | # Publish a message to this exchange on the given topic 60 | def publish(topic_key, payload) 61 | assert_table 62 | 63 | result = @table.filter( 64 | :topic => topic_key.class == Hash ? r.literal(topic_key) : topic_key 65 | ).update( 66 | :payload => payload, 67 | :updated_on => r.now, 68 | ).run(@conn) 69 | 70 | # If the topic doesn't exist yet, insert a new document. Note: 71 | # it's possible someone else could have inserted it in the 72 | # meantime and this would create a duplicate. That's a risk we 73 | # take here. The consequence is that duplicated messages may 74 | # be sent to the consumer. 75 | if result['replaced'].zero? 76 | @table.insert( 77 | :topic => topic_key, 78 | :payload => payload, 79 | :updated_on => r.now, 80 | ).run(@conn) 81 | end 82 | end 83 | 84 | # Lazy Enumerator of messages from the exchange with topics 85 | # matching the given filter 86 | def subscription(filter_func) 87 | assert_table 88 | full_query(filter_func).run(@conn).lazy.map do |message| 89 | [message['topic'], message['payload']] 90 | end 91 | end 92 | 93 | # Ensures the table specified exists and has the correct primary_key 94 | # and durability settings 95 | def assert_table 96 | if @asserted 97 | return 98 | end 99 | begin 100 | # Assert the db into existence 101 | r.db_create(@db).run(@conn) 102 | rescue RethinkDB::RqlRuntimeError => rre 103 | unless rre.to_s.include?('already exists') 104 | raise 105 | end 106 | end 107 | begin 108 | # We set durability to soft because we don't actually care if 109 | # the write is confirmed on disk, we just want the change 110 | # notification (i.e. the message on the queue) to be generated. 111 | r.table_create(@name, :durability => :soft).run(@conn) 112 | rescue RethinkDB::RqlRuntimeError => rre 113 | unless rre.to_s.include?('already exists') 114 | raise 115 | end 116 | end 117 | @asserted = true 118 | end 119 | end 120 | 121 | 122 | # Represents a topic that may be published to. 123 | 124 | class Topic 125 | def initialize(exchange, topic_key) 126 | @key = topic_key 127 | @exchange = exchange 128 | end 129 | 130 | # Publish a payload to the current topic 131 | def publish(payload) 132 | @exchange.publish(@key, payload) 133 | end 134 | end 135 | 136 | # A queue that filters messages in the exchange 137 | class Queue 138 | def initialize(exchange, &filter_func) 139 | @exchange = exchange 140 | @filter_func = filter_func 141 | end 142 | 143 | # Returns the full ReQL query for this queue 144 | def full_query 145 | @exchange.full_query(@filter_func) 146 | end 147 | 148 | # An Enumerator of messages from this queue's subscriptions 149 | def subscription 150 | @exchange.subscription(@filter_func) 151 | end 152 | end 153 | 154 | end 155 | -------------------------------------------------------------------------------- /ruby/sender.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # This is a simple message sender application that uses the rethinkmq 3 | # library. 4 | 5 | # It receives a message at the command line and sends it to the 6 | # specified topic 7 | 8 | require 'logger' 9 | require_relative 'rethinkmq' 10 | 11 | $USAGE = < 13 | 14 | Examples: 15 | 16 | Broadcast the temperature in Mountain View 17 | #{$PROGRAM_NAME} weather.us.ca.mountainview 74 18 | Broadcast the temperature in Mountain View 19 | #{$PROGRAM_NAME} weather.uk.conditions "A bit rainy" 20 | 21 | USAGE 22 | 23 | def main 24 | if ARGV[0] == '-h' 25 | puts $USAGE 26 | return 27 | elsif ARGV[0] == '-v' 28 | ARGV.shift 29 | else 30 | $RMQLogger.level = Logger::ERROR 31 | end 32 | 33 | MQConnection.new('MQ').exchange('messages').topic(ARGV[0]).publish(ARGV[1]) 34 | end 35 | 36 | main 37 | --------------------------------------------------------------------------------