├── python ├── requirements.txt ├── rabbit_listener.py ├── rabbit_feeder.py └── change_generator.py ├── .gitignore ├── ruby ├── Gemfile ├── rabbit_listener.rb └── rabbit_feeder.rb ├── javascript ├── package.json ├── rabbit_listener.js └── rabbit_feeder.js └── README.md /python/requirements.txt: -------------------------------------------------------------------------------- 1 | rethinkdb>=1.13 2 | pika>=0.9.13 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | *.pyc 3 | *.log 4 | *.lock 5 | .#* 6 | -------------------------------------------------------------------------------- /ruby/Gemfile: -------------------------------------------------------------------------------- 1 | # A sample Gemfile 2 | source "https://rubygems.org" 3 | 4 | gem "rethinkdb", ">=1.13" 5 | gem "bunny", ">=1.3.1" 6 | -------------------------------------------------------------------------------- /javascript/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "rethinkdb-examples-rabbitmq", 3 | "version": "1.0.0", 4 | "description": "Example integration between RethinkDB and RabbitMQ", 5 | "bugs": "https://github.com/rethinkdb/example-rabbitmq/issues", 6 | "bin": { 7 | "rabbit_feeder.js": "./rabbit_feeder.js", 8 | "rabbit_listener.js": "./rabbit_listener.js" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "https://github.com/rethinkdb/example-rabbitmq" 13 | }, 14 | "dependencies": { 15 | "rethinkdb": ">=1.13", 16 | "amqplib": ">=0.2.0" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /ruby/rabbit_listener.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'bunny' 4 | require 'json' 5 | 6 | 7 | # Setup the rabbit connection and queue 8 | rabbit_conn = Bunny.new(:host => 'localhost', :port => 5672).start 9 | channel = rabbit_conn.create_channel 10 | exchange = channel.topic("rethinkdb", :durable => false) 11 | queue = channel.queue('', :exclusive => true) 12 | 13 | # Bind to all changes on the 'mytable' topic 14 | queue.bind(exchange, :routing_key => 'mytable.*') 15 | 16 | # Listen for changes and print them out 17 | puts 'Started listening...' 18 | 19 | queue.subscribe(:block => true) do |delivery_info, metadata, payload| 20 | change = JSON.parse(payload) 21 | tablename = delivery_info.routing_key.split('.')[0] 22 | puts "#{tablename} -> RabbitMQ -( #{delivery_info.routing_key} )-> Listener" 23 | puts JSON.pretty_generate(change) 24 | puts "="*80, "\n" 25 | end 26 | -------------------------------------------------------------------------------- /python/rabbit_listener.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | import pika 4 | import json 5 | 6 | # Setup the rabbit connection and queue 7 | rabbit_conn = pika.BlockingConnection( 8 | pika.ConnectionParameters(host='localhost', port=5672), 9 | ) 10 | exchange = 'rethinkdb' 11 | channel = rabbit_conn.channel() 12 | channel.exchange_declare(exchange, exchange_type='topic', durable=False) 13 | queue = channel.queue_declare(exclusive=True).method.queue 14 | 15 | # Bind to all changes on the 'mytable' topic 16 | channel.queue_bind(queue, exchange, routing_key='mytable.*') 17 | 18 | # Listen for changes and print them out 19 | print 'Started listening...' 20 | 21 | for method, properties, payload in channel.consume(queue): 22 | change = json.loads(payload) 23 | tablename = method.routing_key.split('.')[0] 24 | print tablename, ' -> RabbitMQ -(', method.routing_key, ')-> Listener' 25 | print json.dumps(change, indent=True, sort_keys=True) 26 | print '='*80, '\n' 27 | -------------------------------------------------------------------------------- /javascript/rabbit_listener.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var amqp = require('amqplib'); 4 | 5 | // Setup the rabbit connection 6 | 7 | var rabbitConn; 8 | var channel; 9 | var exchange = 'rethinkdb'; 10 | var queue; 11 | 12 | // Create the rabbit connection 13 | amqp.connect('amqp://localhost:5672').then(function(conn){ 14 | rabbitConn = conn; 15 | // Create the rabbit channel 16 | return rabbitConn.createChannel(); 17 | }).then(function(ch){ 18 | channel = ch; 19 | // Create the exchange (or do nothing if it exists) 20 | return channel.assertExchange(exchange, 'topic', {durable: false}); 21 | }).then(function(){ 22 | // Create the queue 23 | return channel.assertQueue('', {exclusive: true}); 24 | }).then(function(q){ 25 | queue = q.queue; 26 | // Bind the queue to all topics about 'mytable' 27 | return channel.bindQueue(queue, exchange, 'mytable.*'); 28 | }).then(function(){ 29 | console.log('Started listening...'); 30 | channel.consume(queue, function(msg){ 31 | // Handle each message as it comes in from RabbitMQ 32 | var change = JSON.parse(msg.content); 33 | var tablename = msg.fields.routingKey.split('.'); 34 | console.log(tablename, '-> RabbitMQ -(', 35 | msg.fields.routingKey, ')-> Listener'); 36 | console.log(JSON.stringify(change, undefined, 2)); 37 | console.log(new Array(81).join('=') + '\n') 38 | }) 39 | }) 40 | -------------------------------------------------------------------------------- /ruby/rabbit_feeder.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # Listens for changes on a rethinkdb table and sends them over a 4 | # RabbitMQ topic exchange. 5 | # 6 | # The topic will be . where 7 | # type_of_change is create, delete or update. 8 | 9 | require 'bunny' 10 | require 'rethinkdb' 11 | include RethinkDB::Shortcuts 12 | require 'json' 13 | 14 | # Setup RethinkDB connection 15 | rethink_conn = r.connect(:host => 'localhost', :port => 28015) 16 | 17 | # Ensure correct database and table exist for this example 18 | begin 19 | r.db_create("change_example").run(rethink_conn) 20 | rescue RethinkDB::RqlRuntimeError 21 | end 22 | 23 | begin 24 | r.db("change_example").table_create("mytable").run(rethink_conn) 25 | rescue RethinkDB::RqlRuntimeError 26 | end 27 | 28 | # Setup rabbit connection and exchange 29 | rabbit_conn = Bunny.new(:host => 'localhost', :port => 5672).start 30 | channel = rabbit_conn.create_channel 31 | exchange = channel.topic("rethinkdb", :durable => false) 32 | 33 | # Determines whether the change is a create, delete or update 34 | def type_of_change(change) 35 | if change['old_val'].nil? 36 | :create 37 | elsif change['new_val'].nil? 38 | :delete 39 | else 40 | :update 41 | end 42 | end 43 | 44 | # Start feeding... 45 | table_changes = r.db('change_example').table('mytable').changes() 46 | 47 | begin 48 | table_changes.run(rethink_conn).each do |change| 49 | routing_key = "mytable.#{type_of_change change}" 50 | puts "RethinkDB -( #{routing_key} )-> RabbitMQ" 51 | exchange.publish(change.to_json, :routing_key => routing_key) 52 | end 53 | rescue RethinkDB::RqlRuntimeError => e 54 | # Table may have been dropped, connection failed etc 55 | puts e.message 56 | end 57 | -------------------------------------------------------------------------------- /python/rabbit_feeder.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | '''Listens for changes on a rethinkdb table and sends them over a 3 | RabbitMQ topic exchange. 4 | 5 | The topic will be . where type_of_change is 6 | create, delete or update.''' 7 | 8 | import pika 9 | import rethinkdb as r 10 | import json 11 | 12 | # Setup RethinkDB connection 13 | rethink_conn = r.connect(host='localhost', port=28015) 14 | 15 | # Ensure correct database and table exist for this example 16 | try: 17 | r.db_create('change_example').run(rethink_conn) 18 | except r.RqlRuntimeError: 19 | pass 20 | 21 | try: 22 | r.db('change_example').table_create('mytable').run(rethink_conn) 23 | except r.RqlRuntimeError: 24 | pass 25 | 26 | # Setup rabbit connection and exchange 27 | rabbit_conn = pika.BlockingConnection( 28 | pika.ConnectionParameters(host='localhost', port=5672), 29 | ) 30 | exchange = 'rethinkdb' 31 | channel = rabbit_conn.channel() 32 | channel.exchange_declare(exchange, exchange_type='topic', durable=False) 33 | 34 | 35 | def type_of_change(change): 36 | '''Determines whether the change is a create, delete or update''' 37 | if change['old_val'] is None: 38 | return 'create' 39 | elif change['new_val'] is None: 40 | return 'delete' 41 | else: 42 | return 'update' 43 | 44 | # Start feeding... 45 | table_changes = r.db('change_example').table('mytable').changes() 46 | try: 47 | for change in table_changes.run(rethink_conn): 48 | routing_key = 'mytable.' + type_of_change(change) 49 | print 'RethinkDB -(', routing_key, ')-> RabbitMQ' 50 | channel.basic_publish(exchange, routing_key, json.dumps(change)) 51 | except r.RqlRuntimeError as e: 52 | # Table may have been dropped, connection failed etc 53 | print e.message 54 | -------------------------------------------------------------------------------- /python/change_generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | '''This script generates random changes on the table. It's not 3 | necessary to understand to use the other scripts, but running this is 4 | a convenient way to make sure your change feed is constantly cranking 5 | things out. 6 | ''' 7 | 8 | import rethinkdb as r 9 | import random 10 | import time 11 | import string 12 | 13 | class ChangeGenerator(object): 14 | 15 | def __init__(self): 16 | self.conn = r.connect() 17 | self.absorb(r.db_create('change_example')) 18 | self.absorb(r.db('change_example').table_create('mytable')) 19 | self.table = r.db('change_example').table('mytable') 20 | 21 | def absorb(self, query): 22 | '''Ignore already created errors''' 23 | try: 24 | return query.run(self.conn) 25 | except r.RqlRuntimeError: 26 | pass 27 | 28 | @staticmethod 29 | def random_doc(): 30 | return { 31 | random.choice(string.ascii_lowercase): 32 | random.choice(string.ascii_uppercase) 33 | for _ in xrange(random.randint(0, 3)) 34 | } 35 | 36 | def create(self): 37 | print 'Creating a random doc...' 38 | self.table.insert(self.random_doc()).run(self.conn) 39 | 40 | def delete(self): 41 | print 'Deleting a random doc...' 42 | self.table.sample(1).delete().run(self.conn) 43 | 44 | def update(self): 45 | print 'Updating a random doc...' 46 | self.table.sample(1).update(self.random_doc()).run(self.conn) 47 | 48 | def create_changes(self): 49 | while True: 50 | getattr(self, random.choice(['create', 'update', 'delete']))() 51 | time.sleep(random.randint(2, 4)) 52 | 53 | 54 | if __name__ == '__main__': 55 | ChangeGenerator().create_changes() 56 | -------------------------------------------------------------------------------- /javascript/rabbit_feeder.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var r = require('rethinkdb'); 4 | var amqp = require('amqplib'); 5 | 6 | var rethinkConn; 7 | var rabbitConn; 8 | var channel; 9 | var exchange = 'rethinkdb'; 10 | 11 | 12 | r.connect({host: 'localhost', port: 28015}).then(function(conn) { 13 | // Setup RethinkDB connection 14 | rethinkConn = conn; 15 | }).catch(r.Error.RqlDriverError, function(err){ 16 | console.log(err.message); 17 | process.exit(1); 18 | }).then(function createDB(){ 19 | // Ensure database exists for this example 20 | return r.dbCreate('change_example').run(rethinkConn); 21 | }).finally(function createTable(){ 22 | // Ensure table exists for this example 23 | return r.db('change_example').tableCreate('mytable').run(rethinkConn); 24 | }).catch(r.Error.RqlRuntimeError, function(){ 25 | // We ignore db/table exists errors here because amqplib uses a 26 | // different promise library from rethinkdb, and the error doesn't 27 | // propagate correctly from one to the other. 28 | }).then(function(){ 29 | // Setup rabbit connection 30 | return amqp.connect('amqp://localhost:5672'); 31 | }).then(function(conn){ 32 | rabbitConn = conn; 33 | // Setup rabbit channel 34 | return rabbitConn.createChannel(); 35 | }).then(function(ch){ 36 | channel = ch; 37 | // Setup rabbit exchange 38 | return channel.assertExchange(exchange, 'topic', {durable: false}); 39 | }).then(function(){ 40 | // Listen for changes on our table 41 | return r.db('change_example').table('mytable').changes().run(rethinkConn); 42 | }).then(function(changeCursor){ 43 | // Feed changes into rabbit 44 | changeCursor.each(function(err, change){ 45 | if(err){ 46 | // The table may have been deleted, or possibly connection issues 47 | console.log('A problem reading from RethinkDB cursor:'); 48 | console.log(err.msg); 49 | process.exit(1); 50 | } 51 | var routingKey = 'mytable.' + typeOfChange(change); 52 | console.log('RethinkDB -(', routingKey, ')-> RabbitMQ') 53 | channel.publish( 54 | exchange, routingKey, new Buffer(JSON.stringify(change))); 55 | }); 56 | }).catch(function(err){ 57 | console.log(err.message); 58 | process.exit(1); 59 | }); 60 | 61 | function typeOfChange(change) { 62 | // Determines whether the change is a create, delete or update 63 | if(change.old_val === null){ 64 | return 'create'; 65 | } else if(change.new_val === null){ 66 | return 'delete'; 67 | } else { 68 | return 'update'; 69 | } 70 | return 'something' 71 | } 72 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Integrating RethinkDB with RabbitMQ # 2 | 3 | Example code for connecting RethinkDB change feeds into RabbitMQ topic 4 | exchanges. 5 | 6 | This repo contains an example integration for Python, Ruby, and 7 | JavaScript (NodeJS). 8 | 9 | You can find a tutorial on the techniques behind these examples in 10 | [javascript](http://rethinkdb.com/docs/rabbitmq/javascript/), 11 | [python](http://rethinkdb.com/docs/rabbitmq/python/) and 12 | [ruby](http://rethinkdb.com/docs/rabbitmq/ruby/) 13 | 14 | ## Prerequisites 15 | 16 | First, [install RethinkDB](http://rethinkdb.com/docs/install/). If you 17 | haven't already, you may want to have a look at the 18 | [quickstart guide](http://rethinkdb.com/docs/quickstart) as well. 19 | 20 | You'll also want to 21 | [install RabbitMQ](https://www.rabbitmq.com/download.html) for your 22 | platform. 23 | 24 | Finally, we recommend trying these out inside a 25 | [virtualenv](virtualenv.readthedocs.com) if you're using Python, and 26 | an [rvm gemset](https://rvm.io/) if you're using Ruby. 27 | 28 | ## Dependencies 29 | 30 | Each script has two dependencies, the RethinkDB client and a library 31 | for interacting with RabbitMQ in that language. For Python that's 32 | [pika](http://pika.readthedocs.org), for Ruby it's 33 | [Bunny](http://rubybunny.info), and for JavaScript it's 34 | [amqplib](http://www.squaremobius.net/amqp.node) (also known as 35 | amqp.node). 36 | 37 | ```bash 38 | $ git clone http://github.com/rethinkdb/example-rabbitmq 39 | $ cd example-rabbitmq/python 40 | $ pip install -r requirements.txt 41 | # or 42 | $ cd example-rabbitmq/ruby 43 | $ bundler install 44 | # or 45 | $ cd example-rabbitmq/javascript 46 | $ npm install . 47 | ``` 48 | 49 | ## Rabbit feeder 50 | 51 | The feeder script listens to changes to a RethinkDB table and inserts 52 | them into a RabbitMQ exchange as it's notified about them. Due to 53 | RethinkDB's [changefeeds](http://rethinkdb.com/docs/changefeeds), 54 | there's no need to poll for table changes, the entire system is 55 | push/pull. 56 | 57 | ```bash 58 | $ ./rabbit_feeder.py 59 | # or 60 | $ ./rabbit_feeder.rb 61 | # or 62 | $ ./rabbit_feeder.js 63 | ``` 64 | 65 | ## Rabbit listener 66 | 67 | The listener script simply connects to the RabbitMQ instance and binds 68 | a queue to the `rethinkdb` exchange (created by the feeder script). It 69 | then prints out the changes coming through and what topic it received 70 | the message on. You'll want to run the listener in another window from 71 | the feeder: 72 | 73 | ```bash 74 | $ ./rabbit_listener.py 75 | # or 76 | $ ./rabbit_listener.rb 77 | # or 78 | $ ./rabbit_listener.js 79 | ``` 80 | 81 | ## Change generator 82 | 83 | This repo also contains a python script for generating random changes 84 | in the table the feeder scripts are subscribed to. This script isn't a 85 | part of the integration itself, but is useful for seeing changes 86 | happen in real-time. Again, you'll want to run the generator in a 87 | different window from the feeder and the listener. 88 | 89 | ```bash 90 | $ ./change_generator.py 91 | ``` 92 | --------------------------------------------------------------------------------