├── .gitignore ├── LICENSE.md ├── README.md ├── rethinkdb_fdw ├── __init__.py ├── operatorFunctions.py └── rethinkdb_fdw.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.py[cod] 3 | *.log 4 | *.out 5 | *.swp 6 | .*.swp 7 | .noseids 8 | build 9 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Wilson RMS 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ###rethinkdb-multicorn-postgresql-fdw 2 | 3 | Multicorn based PostgreSQL Foreign Data Wrapper for RethinkDB 4 | 5 | This was set up for Python 2.7, Multicorn 1.1, PostgreSQL 9.3, and RethinkDB 1.15. If you are using something else, your mileage may vary. ... Heck, your mileage may vary anyhow. We think this works for us, we make no guarantee or promise it will work for you too ... 6 | 7 |
First install RethinkDB's Python libraries and Multicorn on your PostgreSQL database server.
8 |
    9 |
    $ sudo pip install rethinkdb
    10 |
    $ sudo pgxn install multicorn
    11 |
12 | 13 |
Then install this package on your PostgreSQL database server:
14 |
    15 |
    $ git clone https://github.com/rotten/rethinkdb-multicorn-postgresql-fdw
    16 |
    $ cd rethinkdb-multicorn-postgresql-fdw
    17 |
    $ sudo python setup.py install
    18 |
19 | 20 |
Then create a table like this:
21 |
    22 |
    mydb=# create extension multicorn;
    23 |
    mydb-# create server myrethinkdb foreign data wrapper multicorn options (wrapper 'rethinkdb_fdw.rethinkdb_fdw.RethinkDBFDW', host 'myhost', port '28015', database 'somerethinkdb');
    24 |
    mydb-#
    create foreign table mytable (
25 |     id uuid,
26 |     somekey varchar,
27 |     someotherkey varchar,
28 |     sometimestamp timestamp (6) with time zone,
29 |     bigintegerkey bigint,
30 |     nestedjsonkey json,
31 |     yetanotherkey varchar
32 |     ) server myrethinkdb options (table_name 'rethinkdb_table');
33 |
    34 |
35 | 36 | When foreign table performance is an issue, you may want to put a materialized view in front of your foreign table. ** Remember to refresh the materialized view when you need to see the latest stuff from your RethinkDB. (PostgreSQL does not yet have auto-refreshing materialized views.) 37 | 38 |
39 | 40 | #####Some Notes on development/troubleshooting this FDW: 41 | 42 | 1. You can set: `log_min_messages = debug1` in your postgresql.conf to see the log_to_postgres() DEBUG messages. 43 | 2. You will probably need to exit psql and re-enter it to pick up changes to the python libraries when you push an update. (You do not necessarily have to drop your server and table definitions if you are only changing the querying logic.) 44 | 3. Send us a pull request if you have bug fixes or enhancements or good ideas to make it better. 45 | 46 |
47 | 48 | 49 | Here is a noteworthy blog post on the RethinkDB site about this project: http://rethinkdb.com/blog/postgres-foreign-data-wrapper/ 50 | 51 | 52 | -------------------------------------------------------------------------------- /rethinkdb_fdw/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rotten/rethinkdb-multicorn-postgresql-fdw/7498c728b1d095c4313565e9bd58fcfe4ec27c98/rethinkdb_fdw/__init__.py -------------------------------------------------------------------------------- /rethinkdb_fdw/operatorFunctions.py: -------------------------------------------------------------------------------- 1 | ## This is by our Multicorn ForeignDataWrapper to convert strings with PostgreSQL operators in them to functions. 2 | ## R.Otten - 2014 3 | 4 | import operator 5 | import re 6 | 7 | ## We throw this if we encounter an operator that we don't know what to do with: 8 | class unknownOperatorException(Exception): 9 | 10 | def __init__(self, msg): 11 | self.msg = msg 12 | 13 | def __str__(self): 14 | return repr(self.msg) 15 | 16 | 17 | ################################################################################ 18 | ### Custom functions to implement some of the operators: 19 | def reverseContains(a, b): 20 | return operator.contains(b, a) 21 | 22 | def strictlyLeft(a, b): 23 | return max(a) < min(b) 24 | 25 | def strictlyRight(a, b): 26 | return min(a) > max(b) 27 | 28 | def rightBounded(a, b): 29 | return max(a) <= max(b) 30 | 31 | def leftBounded(a, b): 32 | return min(a) >= min(b) 33 | 34 | def overlap(a, b): 35 | if (min(a) >= min(b)) and (min(a) <= max(b)): 36 | return True 37 | if (max(a) <= max(b)) and (max(a) >= min(b)): 38 | return True 39 | return False 40 | 41 | def regexSearch(a, b): 42 | if re.search(b, a): 43 | return True 44 | else: 45 | return False 46 | 47 | def regexSearch_i(a, b): 48 | if re.search(b, a, re.I): 49 | return True 50 | else: 51 | return False 52 | 53 | def notRegexSearch(a, b): 54 | return not regexSearch(a, b) 55 | 56 | def notRegexSearch_i(a, b): 57 | return not regexSearch_i(a, b) 58 | 59 | def likeSearch(a, b): 60 | b.replace('%%', '.*') 61 | b.replace('_', '.') 62 | return regexSearch(a, b) 63 | 64 | def likeSearch_i(a, b): 65 | b.replace('%%', '.*') 66 | b.replace('_', '.') 67 | return regexSearch_i(a, b) 68 | 69 | def notLikeSearch(a, b): 70 | b.replace('%%', '.*') 71 | b.replace('_', '.') 72 | return not regexSearch(a, b) 73 | 74 | def notLikeSearch_i(a, b): 75 | b.replace('%%', '.*') 76 | b.replace('_', '.') 77 | return not regexSearch_i(a, b) 78 | 79 | 80 | ################################################################################ 81 | ### The main function we use external to this file: 82 | ## Translate a string with an operator in it (eg. ">=") into a function. 83 | ## 84 | ## Not supported (yet -- feel free to add more support!): 85 | ## "between" -- it isn't clear if we'll get those. 86 | ## "OR" -- it isnt' clear if we'll get those. 87 | ## Geometric Operators 88 | ## Text Search Operators 89 | ## Network Address Operators 90 | ## JSON Operators 91 | ## The Array operators when used on Ranges 92 | ## 93 | def getOperatorFunction(opr): 94 | 95 | operatorFunctionMap = { 96 | '<': operator.lt, 97 | '>': operator.gt, 98 | '<=': operator.le, 99 | '>=': operator.ge, 100 | '=': operator.eq, 101 | '<>': operator.ne, 102 | '!=': operator.ne, 103 | '@>': operator.contains, 104 | '<@': reverseContains, 105 | '<<': strictlyLeft, 106 | '>>': strictlyRight, 107 | '&<': rightBounded, 108 | '>&': leftBounded, 109 | '&&': overlap, 110 | 'is': operator.eq, # this one won't work in every sql context, but should for some cases 111 | '~': regexSearch, 112 | '~*': regexSearch_i, 113 | '!~': notRegexSearch, 114 | '!~*': notRegexSearch_i, 115 | '~~': likeSearch, 116 | '!~~': notLikeSearch, 117 | 'like': likeSearch, 118 | 'not like': notLikeSearch, 119 | '~~*': likeSearch_i, 120 | '!~~*': notLikeSearch_i, 121 | 'ilike': likeSearch_i, 122 | 'not ilike': likeSearch_i, 123 | 'similar to': regexSearch, 124 | 'not similar to': notRegexSearch 125 | } 126 | 127 | if not operatorFunctionMap.has_key(opr): 128 | raise unknownOperatorException("'%s' is not a supported operator." % opr) 129 | 130 | return operatorFunctionMap[opr] 131 | 132 | 133 | 134 | -------------------------------------------------------------------------------- /rethinkdb_fdw/rethinkdb_fdw.py: -------------------------------------------------------------------------------- 1 | ## This is the implementation of the Multicorn ForeignDataWrapper class that does all of the work in RethinkDB 2 | ## R.Otten - 2014 3 | 4 | from collections import OrderedDict 5 | import json 6 | 7 | from multicorn import ForeignDataWrapper 8 | from multicorn.utils import log_to_postgres, ERROR, WARNING, DEBUG 9 | 10 | import rethinkdb as r 11 | 12 | from operatorFunctions import unknownOperatorException, getOperatorFunction 13 | 14 | 15 | ## The Foreign Data Wrapper Class: 16 | class RethinkDBFDW(ForeignDataWrapper): 17 | 18 | """ 19 | RethinkDB FDW for PostgreSQL 20 | """ 21 | 22 | def __init__(self, options, columns): 23 | 24 | super(RethinkDBFDW, self).__init__(options, columns) 25 | 26 | log_to_postgres('options: %s' % options, DEBUG) 27 | log_to_postgres('columns: %s' % columns, DEBUG) 28 | 29 | if options.has_key('host'): 30 | self.host = options['host'] 31 | else: 32 | self.host = 'localhost' 33 | log_to_postgres('Using Default host: localhost.', WARNING) 34 | 35 | if options.has_key('port'): 36 | self.port = options['port'] 37 | else: 38 | self.port = '28015' 39 | log_to_postgres('Using Default port: 28015.', WARNING) 40 | 41 | if options.has_key('database'): 42 | self.database = options['database'] 43 | else: 44 | log_to_postgres('database parameter is required.', ERROR) 45 | 46 | if options.has_key('table_name'): 47 | self.table = options['table_name'] 48 | else: 49 | log_to_postgres('table_name parameter is required.', ERROR) 50 | 51 | if options.has_key('auth_key'): 52 | self.auth_key = options['auth_key'] 53 | else: 54 | self.auth_key = '' 55 | 56 | self.columns = columns 57 | 58 | 59 | # actually do the work: 60 | def _run_rethinkdb_action(self, action): 61 | 62 | # try to connect 63 | try: 64 | 65 | conn = r.connect(host=self.host, port=self.port, db=self.database, auth_key=self.auth_key) 66 | 67 | except Exception, e: 68 | 69 | log_to_postgres('Connection Falure: %s' % e, ERROR) 70 | 71 | 72 | # Now try to run the action: 73 | try: 74 | 75 | log_to_postgres('RethinkDB Action: %s' % action, DEBUG) 76 | result = action.run(conn) 77 | 78 | except Exception, e: 79 | 80 | conn.close() 81 | log_to_postgres('RethinkDB error: %s' %e, ERROR) 82 | 83 | 84 | return result 85 | 86 | 87 | # SQL SELECT: 88 | def execute(self, quals, columns): 89 | 90 | log_to_postgres('Query Columns: %s' % columns, DEBUG) 91 | log_to_postgres('Query Filters: %s' % quals, DEBUG) 92 | 93 | myQuery = r.table(self.table)\ 94 | .pluck(self.columns.keys()) 95 | 96 | for qual in quals: 97 | 98 | try: 99 | operatorFunction = getOperatorFunction(qual.operator) 100 | except unknownOperatorException, e: 101 | log_to_postgres(e, ERROR) 102 | 103 | myQuery = myQuery.filter(operatorFunction(r.row[qual.field_name], qual.value)) 104 | 105 | rethinkResults = self._run_rethinkdb_action(action=myQuery) 106 | 107 | # By default, Multicorn seralizes dictionary types into something for hstore column types. 108 | # That looks something like this: "key => value" 109 | # What we really want is this: "{key:value}" 110 | # so we serialize it here. (This is git issue #1 for this repo, and issue #86 in the Multicorn repo.) 111 | 112 | for resultRow in rethinkResults: 113 | 114 | # I don't think we can mutate the row in the rethinkResults cursor directly. 115 | # It needs to be copied out of the cursor to be reliably mutable. 116 | row = OrderedDict() 117 | for resultColumn in resultRow.keys(): 118 | 119 | if type(resultRow[resultColumn]) is dict: 120 | 121 | row[resultColumn] = json.dumps(resultRow[resultColumn]) 122 | 123 | elif type(resultRow[resultColumn]) is list: 124 | 125 | row[resultColumn] = json.dumps(resultRow[resultColumn]) 126 | 127 | else: 128 | 129 | row[resultColumn] = resultRow[resultColumn] 130 | 131 | yield row 132 | 133 | 134 | # SQL INSERT: 135 | def insert(self, new_values): 136 | 137 | log_to_postgres('Insert Request - new values: %s' % new_values, DEBUG) 138 | 139 | return self._run_rethinkdb_action(action=r.table(self.table)\ 140 | .insert(new_values)) 141 | 142 | # SQL UPDATE: 143 | def update(self, rowid, new_values): 144 | 145 | log_to_postgres('Update Request - new values: %s' % new_values, DEBUG) 146 | 147 | if not rowid: 148 | 149 | log_to_postgres('Update request requires rowid (PK).', ERROR) 150 | 151 | return self._run_rethinkdb_action(action=r.table(self.table)\ 152 | .get(rowid)\ 153 | .update(new_values)) 154 | 155 | # SQL DELETE 156 | def delete(self, rowid): 157 | 158 | log_to_postgres('Delete Request - rowid: %s' % rowid, DEBUG) 159 | 160 | if not rowid: 161 | 162 | log_to_postgres('Update request requires rowid (PK).', ERROR) 163 | 164 | return self._run_rethinkdb_action(action=r.table(self.table)\ 165 | .get(rowid)\ 166 | .delete()) 167 | 168 | 169 | def rowid_column(self, rowid): 170 | 171 | log_to_postgres('rowid requested', DEBUG) 172 | 173 | return 'id' 174 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | ## This is a very simple setup for the module that allows us to create a foreign data wrapper in PostgreSQL to RethinkDB. 2 | ## It is based on the Multicorn README and the Hive FDW example. 3 | ## 4 | from distutils.core import setup 5 | 6 | setup( 7 | name='rethinkdb_fdw', 8 | version='0.1', 9 | author='Rick Otten', 10 | author_email='rotten@windfish.net', 11 | license='Postgresql', 12 | packages=['rethinkdb_fdw'], 13 | url='https://github.com/rotten/rethinkdb-multicorn-postgresql-fdw' 14 | ) 15 | 16 | --------------------------------------------------------------------------------