├── .gitignore ├── INSTALL.md ├── MANIFEST.in ├── Makefile ├── README.md ├── examples ├── connect.py └── drop_slot.py ├── lib └── __init__.py ├── logicaldecoding ├── connection.c ├── connection.h ├── logicaldecodingmodule.c ├── pylogicaldecoding.h ├── python.h └── reader_type.c ├── setup.cfg ├── setup.py └── tests └── __init__.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # PyInstaller 26 | # Usually these files are written by a python script from a template 27 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 28 | *.manifest 29 | *.spec 30 | 31 | # Installer logs 32 | pip-log.txt 33 | pip-delete-this-directory.txt 34 | 35 | # Unit test / coverage reports 36 | htmlcov/ 37 | .tox/ 38 | .coverage 39 | .coverage.* 40 | .cache 41 | nosetests.xml 42 | coverage.xml 43 | *,cover 44 | 45 | # Translations 46 | *.mo 47 | *.pot 48 | 49 | # Django stuff: 50 | *.log 51 | 52 | # Sphinx documentation 53 | doc/_build/ 54 | 55 | # PyBuilder 56 | target/ 57 | -------------------------------------------------------------------------------- /INSTALL.md: -------------------------------------------------------------------------------- 1 | Sorry, no pypi package at the moment... 2 | 3 | Tested only on Debian Jessie and Ubuntu Trusty (should work on any version 4 | of these distribs, and many more UNIXes-like systems). 5 | 6 | # packages 7 | 8 | - postgresql >= 9.4 (on the origin DB) 9 | - libpq5-dev (on the comsumer) 10 | 11 | Please use pg provided packages (TODO: URL) 12 | 13 | # PGHacks 14 | 15 | Install PGHacks following [INSTALL instructions](https://github.com/lisael/PGHacks/blob/master/INSTALL.md#from-source-tarball-recommended) 16 | 17 | # pylogicaldecoding 18 | 19 | Clone the repository and create and new Python2.7 virtualenv. 20 | 21 | ```sh 22 | cd pylogicaldecoding 23 | make install 24 | ``` 25 | 26 | # prepare postgres 27 | 28 | ## PG configuration 29 | 30 | ```sh 31 | sudo vi /etc/postgresql/9.4/main/postgresql.conf 32 | # change max_replication_slots to > 0 33 | # change wal_level to logical 34 | # change max_wal_senders to > 0 35 | 36 | sudo vi /etc/postgresql/9.4/main/pg_hba.conf 37 | # uncomment or create the line 38 | # local replication postgres peer 39 | 40 | sudo /etc/init.d/postgresql restart 41 | ``` 42 | 43 | ## create a replication slot 44 | 45 | Of course you can write your own decoder... however, for testing purpose, you 46 | can use the one provided by postgresql. Unfortunatly it may not be packaged yet, 47 | you may have to compile and install it by hand. 48 | 49 | If you're lucky this may be enough. 50 | 51 | ```sql 52 | sudo -u postgres psql 53 | postgres=# SELECT * FROM pg_create_logical_replication_slot('test_slot', 'test_decoding'); 54 | slot_name | xlog_position 55 | -----------+--------------- 56 | test_slot | 0/16ACC80 57 | (1 row) 58 | ``` 59 | 60 | If you get something like : 61 | 62 | `ERROR: could not access file "test_decoding": No such file or directory` 63 | 64 | Your distrib did not compile the test decoder. 65 | 66 | Otherwise, the installation is completed :) You can start playing with 67 | pylogicaldecoding (Check `README.md` and `/examples` ) 68 | 69 | # install postgres’ `test_decoding.so` contrib 70 | 71 | ## lib requirements 72 | 73 | - libreadline-dev 74 | - postgresql-server-dev-9.4 75 | 76 | ## Download the source 77 | 78 | TODO: URL 79 | 80 | Be careful to download exactly the same version as your postgresql package. 81 | 82 | Untar, Unzip... 83 | 84 | ## Compile 85 | 86 | ``` 87 | cd postgresql-9.4.X/ 88 | ./configure --with-includes=`pg_config --includedir-server` 89 | cd contrig/test_decoding/ 90 | make all 91 | sudo cp test_decoding.so `pg_config --pkglibdir` 92 | ``` 93 | 94 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include logicaldecoding *.c *.h *.manifest 2 | recursive-include lib *.py 3 | recursive-include tests *.py 4 | recursive-include examples *.py 5 | recursive-include doc 6 | recursive-include doc Makefile requirements.txt 7 | recursive-include doc/src *.rst *.py *.css Makefile 8 | include AUTHORS README.rst INSTALL LICENSE NEWS 9 | include PKG-INFO MANIFEST.in MANIFEST setup.py setup.cfg Makefile 10 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PYTHON := python$(PYTHON_VERSION) 2 | PYTHON_VERSION ?= $(shell $(PYTHON) -c 'import sys; print ("%d.%d" % sys.version_info[:2])') 3 | BUILD_DIR = $(shell pwd)/build/lib.$(PYTHON_VERSION) 4 | 5 | SOURCE_C := $(wildcard logicladecoding/*.c logicaldecoding/*.h) 6 | SOURCE_PY := $(wildcard lib/*.py) 7 | SOURCE_TESTS := $(wildcard tests/*.py) 8 | SOURCE_DOC := $(wildcard doc/src/*.rst) 9 | SOURCE := $(SOURCE_C) $(SOURCE_PY) $(SOURCE_TESTS) $(SOURCE_DOC) 10 | 11 | PACKAGE := $(BUILD_DIR)/logicaldecoding 12 | PLATLIB := $(PACKAGE)/_logicaldecoding.so 13 | PURELIB := $(patsubst lib/%,$(PACKAGE)/%,$(SOURCE_PY)) \ 14 | $(patsubst tests/%,$(PACKAGE)/tests/%,$(SOURCE_TESTS)) 15 | 16 | BUILD_OPT := --build-lib=$(BUILD_DIR) 17 | BUILD_EXT_OPT := --build-lib=$(BUILD_DIR) 18 | SDIST_OPT := --formats=gztar 19 | 20 | ifdef PG_CONFIG 21 | BUILD_EXT_OPT += --pg-config=$(PG_CONFIG) 22 | endif 23 | 24 | VERSION := $(shell grep PYLD_VERSION setup.py | head -1 | sed -e "s/.*'\(.*\)'/\1/") 25 | SDIST := dist/pylogicaldecoding-$(VERSION).tar.gz 26 | 27 | .PHONY: env check clean 28 | 29 | default: package 30 | 31 | install: 32 | $(PYTHON) setup.py install 33 | 34 | all: package sdist 35 | 36 | package: $(PLATLIB) $(PURELIB) 37 | 38 | package: $(PLATLIB) $(PURELIB) 39 | 40 | docs: docs-html docs-txt 41 | 42 | docs-html: doc/html/genindex.html 43 | 44 | docs-txt: doc/psycopg2.txt 45 | 46 | # for PyPI documentation 47 | docs-zip: doc/docs.zip 48 | 49 | sdist: $(SDIST) 50 | 51 | env: 52 | $(MAKE) -C doc $@ 53 | 54 | check: 55 | PYTHONPATH=$(BUILD_DIR):$(PYTHONPATH) $(PYTHON) -c "from psycopg2 import tests; tests.unittest.main(defaultTest='tests.test_suite')" --verbose 56 | 57 | testdb: 58 | @echo "* Creating $(TESTDB)" 59 | @if psql -l | grep -q " $(TESTDB) "; then \ 60 | dropdb $(TESTDB) >/dev/null; \ 61 | fi 62 | createdb $(TESTDB) 63 | # Note to packagers: this requires the postgres user running the test 64 | # to be a superuser. You may change this line to use the superuser only 65 | # to install the contrib. Feel free to suggest a better way to set up the 66 | # testing environment (as the current is enough for development). 67 | psql -f `pg_config --sharedir`/contrib/hstore.sql $(TESTDB) 68 | 69 | 70 | $(PLATLIB): $(SOURCE_C) 71 | $(PYTHON) setup.py build_ext $(BUILD_EXT_OPT) 72 | 73 | $(PACKAGE)/%.py: lib/%.py 74 | $(PYTHON) setup.py build_py $(BUILD_OPT) 75 | touch $@ 76 | 77 | $(PACKAGE)/tests/%.py: tests/%.py 78 | $(PYTHON) setup.py build_py $(BUILD_OPT) 79 | touch $@ 80 | 81 | $(SDIST): MANIFEST $(SOURCE) 82 | $(PYTHON) setup.py sdist $(SDIST_OPT) 83 | 84 | MANIFEST: MANIFEST.in $(SOURCE) 85 | # Run twice as MANIFEST.in includes MANIFEST 86 | $(PYTHON) setup.py sdist --manifest-only 87 | $(PYTHON) setup.py sdist --manifest-only 88 | 89 | # docs depend on the build as it partly use introspection. 90 | doc/html/genindex.html: $(PLATLIB) $(PURELIB) $(SOURCE_DOC) 91 | $(MAKE) -C doc html 92 | 93 | doc/psycopg2.txt: $(PLATLIB) $(PURELIB) $(SOURCE_DOC) 94 | $(MAKE) -C doc text 95 | 96 | doc/docs.zip: doc/html/genindex.html 97 | (cd doc/html && zip -r ../docs.zip *) 98 | 99 | clean: 100 | rm -rf build MANIFEST 101 | $(MAKE) -C doc clean 102 | 103 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pylogicaldecoding 2 | Python interface to PostgreSQL [logical decoding]( 3 | http://www.postgresql.org/docs/9.4/static/logicaldecoding.html) 4 | 5 | *Status : Very early proof of concept. Beware.* 6 | 7 | # Quick start 8 | 9 | I suppose you managed to install build dependences, PGHacks, and 10 | pylogicaldecoding (read INSTALL.md). 11 | 12 | Open a postgres shell 13 | 14 | ``` 15 | postgres=# CREATE TABLE data(id serial primary key, data text); 16 | CREATE TABLE 17 | ``` 18 | 19 | Run the script into your virtualenv: 20 | 21 | ``` 22 | sudo -u postgres `which python` examples/connect.py 23 | ``` 24 | 25 | From now on , every change to your database will be reflected here 26 | 27 | ``` 28 | postgres=# INSERT INTO data(data) VALUES('1'); 29 | INSERT 0 1 30 | ``` 31 | 32 | In pylogicaldecoding console, you should see: 33 | 34 | ``` 35 | got event: 36 | BEGIN 724 37 | 38 | got event: 39 | table public.data: INSERT: id[integer]:1 data[text]:'1' 40 | 41 | got event: 42 | COMMIT 724 43 | ``` 44 | 45 | Let's play a little, now: 46 | 47 | ``` 48 | postgres=# BEGIN; 49 | BEGIN 50 | postgres=# UPDATE data SET data='2' WHERE id=1; 51 | UPDATE 1 52 | postgres=# DELETE FROM data WHERE id=1; 53 | DELETE 1 54 | postgres=# COMMIT; 55 | COMMIT 56 | postgres=# BEGIN; 57 | BEGIN 58 | postgres=# INSERT INTO data(data) VALUES('1'); 59 | INSERT 0 1 60 | postgres=# ROLLBACK; 61 | ROLLBACK 62 | ``` 63 | 64 | ``` 65 | got event: 66 | BEGIN 728 67 | 68 | got event: 69 | table public.data: UPDATE: id[integer]:1 data[text]:'2' 70 | 71 | got event: 72 | table public.data: DELETE: id[integer]:1 73 | 74 | got event: 75 | COMMIT 728 76 | 77 | ``` 78 | 79 | Note that the rollbacked transaction has not been recieved. It's a feature 80 | of test_decoding (it can be changed passing options when the slot is created 81 | and soon in pylogicaldecoding itself) 82 | 83 | You may also need to know the values of updated and deleted fields. Postgres 84 | does that, at the expense of larger WAL files and a little overhead. However 85 | these issues are mitigated by postgres using per table settings: 86 | 87 | ``` 88 | postgres=# ALTER TABLE data REPLICA IDENTITY FULL; 89 | ALTER TABLE 90 | ``` 91 | 92 | Check http://www.postgresql.org/docs/9.4/static/sql-altertable.html for more 93 | info. 94 | 95 | ``` 96 | postgres=# BEGIN; 97 | BEGIN 98 | postgres=# INSERT INTO data(data) VALUES('1'); 99 | INSERT 0 1 100 | postgres=# UPDATE data SET data='2' WHERE id=2; 101 | UPDATE 1 102 | postgres=# DELETE FROM data WHERE id=2; 103 | DELETE 1 104 | postgres=# COMMIT; 105 | COMMIT 106 | ``` 107 | 108 | ``` 109 | got event: 110 | BEGIN 732 111 | 112 | got event: 113 | table public.data: INSERT: id[integer]:5 data[text]:'1' 114 | 115 | got event: 116 | table public.data: UPDATE: old-key: id[integer]:2 data[text]:'1' new-tuple: id[integer]:4 data[text]:'2' 117 | 118 | got event: 119 | table public.data: DELETE: id[integer]:2 data[text]:'2' 120 | 121 | got event: 122 | COMMIT 732 123 | ``` 124 | 125 | Yay! 126 | 127 | # logical decoding vs. NOTIFY 128 | 129 | Logical decoding uses streaming replication protocol. The origin 130 | DB server is aware of the WAL entries effectively read and 131 | treated by the consumer. The guaranties that the comsumer 132 | will never miss an event. In `example/connection.py` you can read 133 | 134 | ```python 135 | if value.startswith("COMMIT"): 136 | self.commits += 1 137 | if self.commits % 5 == 0: 138 | self.ack() 139 | ``` 140 | 141 | every 5 DB COMMIT we call `Reader.ack()` which acknowledges the 142 | DB. The database is now free to drop the acknowledged log line. If we 143 | had not call `ack()` (ctl-C before 5 postgres commands), the server 144 | would keep these logs and re-send them when the consumer restart: 145 | 146 | terminal 1: 147 | 148 | ``` 149 | (pyld)lisael@laptop:pylogicaldecoding$ sudo -u postgres `which python` examples/connect.py 150 | ``` 151 | 152 | terminal 2: 153 | 154 | ``` 155 | postgres=# INSERT INTO data(data) VALUES('1'); 156 | INSERT 0 1 157 | postgres=# INSERT INTO data(data) VALUES('1'); 158 | INSERT 0 1 159 | ``` 160 | 161 | terminal 1: 162 | ``` 163 | got event: 164 | BEGIN 734 165 | 166 | got event: 167 | table public.data: INSERT: id[integer]:7 data[text]:'1' 168 | 169 | got event: 170 | COMMIT 734 171 | 172 | got event: 173 | BEGIN 735 174 | 175 | got event: 176 | table public.data: INSERT: id[integer]:8 data[text]:'1' 177 | 178 | got event: 179 | COMMIT 735 180 | 181 | ^CTraceback (most recent call last): 182 | File "examples/connect.py", line 21, in 183 | r.start() 184 | ValueError: unexpected termination of replication stream: 185 | (pyld)lisael@laptop:pylogicaldecoding$ sudo -u postgres `which python` examples/connect.py 186 | got event: 187 | BEGIN 734 188 | 189 | got event: 190 | table public.data: INSERT: id[integer]:7 data[text]:'1' 191 | 192 | got event: 193 | COMMIT 734 194 | 195 | got event: 196 | BEGIN 735 197 | 198 | got event: 199 | table public.data: INSERT: id[integer]:8 data[text]:'1' 200 | 201 | got event: 202 | COMMIT 735 203 | ``` 204 | 205 | The server resent transactions 734 and 735 without any logic on consumer 206 | side. 207 | 208 | You can also stop the consumer, send commands in postgres, and watch them 209 | being consumed as soon as you restart the consumer. 210 | 211 | OTOH, `NOTIFY` and channels work only during the connection and there's 212 | no way to re-read missed events. 213 | 214 | Once again, be careful, it's the killer feature but it can bite you. As soon 215 | as the slot is created, the origin DB keep all unacknowledged WAL on disk. 216 | If your consumer is dead, stuck or if you forget to call `ack()` you run into 217 | big troubles. 218 | 219 | # Play, fork, hack, PR, and have fun 220 | Pylogicaldecoding is in its very early stage and there is many improvements 221 | to come. 222 | 223 | Every comment, question, critics and code is warmly welcome in github issues. 224 | 225 | The core C library is designed to be easily merged in psycopg2, please keep 226 | this fact in mind in your pull requests (however I dropped some of py3 support 227 | and all of MS windows support as long as the project is a proof of concept) 228 | 229 | 230 | -------------------------------------------------------------------------------- /examples/connect.py: -------------------------------------------------------------------------------- 1 | from logicaldecoding import Reader 2 | 3 | 4 | class MyReader(Reader): 5 | def __init__(self, *args, **kwargs): 6 | super(MyReader, self).__init__(*args, **kwargs) 7 | self.commits = 0 8 | 9 | def event(self, value): 10 | print "got event:" 11 | print value 12 | print 13 | if value.startswith("COMMIT"): 14 | self.commits += 1 15 | if self.commits % 5 == 0: 16 | self.ack() 17 | 18 | 19 | if __name__ == '__main__': 20 | r = MyReader() 21 | r.stream() 22 | -------------------------------------------------------------------------------- /examples/drop_slot.py: -------------------------------------------------------------------------------- 1 | from logicaldecoding import Reader 2 | 3 | 4 | if __name__ == '__main__': 5 | r = Reader() 6 | r.drop_slot() 7 | -------------------------------------------------------------------------------- /lib/__init__.py: -------------------------------------------------------------------------------- 1 | from logicaldecoding._logicaldecoding import Reader as _Reader 2 | 3 | 4 | class Reader(_Reader): 5 | 6 | def event(self, value): 7 | raise NotImplementedError("event must be overridden in client") 8 | -------------------------------------------------------------------------------- /logicaldecoding/connection.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lisael/pylogicaldecoding/f874ff0b25cd576cd135a9d6335d7dd3b28d766f/logicaldecoding/connection.c -------------------------------------------------------------------------------- /logicaldecoding/connection.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lisael/pylogicaldecoding/f874ff0b25cd576cd135a9d6335d7dd3b28d766f/logicaldecoding/connection.h -------------------------------------------------------------------------------- /logicaldecoding/logicaldecodingmodule.c: -------------------------------------------------------------------------------- 1 | 2 | #define PSYCOPG_MODULE 3 | #include "logicaldecoding/pylogicaldecoding.h" 4 | 5 | extern HIDDEN PyTypeObject readerType; 6 | 7 | static PyMethodDef logicaldecodingMethods[] = { 8 | {NULL,NULL,0, NULL} 9 | }; 10 | 11 | #if PY_MAJOR_VERSION > 2 12 | static struct PyModuleDef logicaldecodingmodule = { 13 | PyModuleDef_HEAD_INIT, 14 | "_psycopg", 15 | NULL, 16 | -1, 17 | logicaldecodingMethods, 18 | NULL, 19 | NULL, 20 | NULL, 21 | NULL 22 | }; 23 | #endif 24 | 25 | PyMODINIT_FUNC 26 | INIT_MODULE(_logicaldecoding)(void) 27 | { 28 | PyObject *module = NULL; 29 | 30 | /* initialize the module */ 31 | #if PY_MAJOR_VERSION < 3 32 | module = Py_InitModule("_logicaldecoding", logicaldecodingMethods); 33 | #else 34 | module = PyModule_Create(&logicaldecodingmodule); 35 | #endif 36 | if (!module) { goto exit; } 37 | 38 | Py_TYPE(&readerType) = &PyType_Type; 39 | if (PyType_Ready(&readerType) == -1) goto exit; 40 | 41 | PyModule_AddObject(module, "Reader", (PyObject*)&readerType); 42 | exit: 43 | #if PY_MAJOR_VERSION > 2 44 | return module; 45 | #else 46 | return; 47 | #endif 48 | } 49 | -------------------------------------------------------------------------------- /logicaldecoding/pylogicaldecoding.h: -------------------------------------------------------------------------------- 1 | #ifndef PYLD_H 2 | #define PYLD_H 1 3 | #include 4 | #include 5 | 6 | #if __GNUC__ >= 4 && !defined(__MINGW32__) 7 | # define HIDDEN __attribute__((visibility("hidden"))) 8 | #else 9 | # define HIDDEN 10 | #endif 11 | 12 | #include "logicaldecoding/python.h" 13 | #endif 14 | -------------------------------------------------------------------------------- /logicaldecoding/python.h: -------------------------------------------------------------------------------- 1 | /* python.h - python version compatibility stuff 2 | * 3 | * Copyright (C) 2003-2010 Federico Di Gregorio 4 | * 5 | * This file is part of psycopg. 6 | * 7 | * psycopg2 is free software: you can redistribute it and/or modify it 8 | * under the terms of the GNU Lesser General Public License as published 9 | * by the Free Software Foundation, either version 3 of the License, or 10 | * (at your option) any later version. 11 | * 12 | * In addition, as a special exception, the copyright holders give 13 | * permission to link this program with the OpenSSL library (or with 14 | * modified versions of OpenSSL that use the same license as OpenSSL), 15 | * and distribute linked combinations including the two. 16 | * 17 | * You must obey the GNU Lesser General Public License in all respects for 18 | * all of the code used other than OpenSSL. 19 | * 20 | * psycopg2 is distributed in the hope that it will be useful, but WITHOUT 21 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 22 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 23 | * License for more details. 24 | */ 25 | 26 | #ifndef PSYCOPG_PYTHON_H 27 | #define PSYCOPG_PYTHON_H 1 28 | 29 | #include 30 | #if PY_MAJOR_VERSION < 3 31 | #include 32 | #endif 33 | 34 | #if PY_VERSION_HEX < 0x02050000 35 | # error "psycopg requires Python >= 2.5" 36 | #endif 37 | 38 | /* hash() return size changed around version 3.2a4 on 64bit platforms. Before 39 | * this, the return size was always a long, regardless of arch. ~3.2 40 | * introduced the Py_hash_t & Py_uhash_t typedefs with the resulting sizes 41 | * based upon arch. */ 42 | #if PY_VERSION_HEX < 0x030200A4 43 | typedef long Py_hash_t; 44 | typedef unsigned long Py_uhash_t; 45 | #endif 46 | 47 | /* Macros defined in Python 2.6 */ 48 | #ifndef Py_REFCNT 49 | #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt) 50 | #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) 51 | #define Py_SIZE(ob) (((PyVarObject*)(ob))->ob_size) 52 | #define PyVarObject_HEAD_INIT(x,n) PyObject_HEAD_INIT(x) n, 53 | #endif 54 | 55 | /* FORMAT_CODE_PY_SSIZE_T is for Py_ssize_t: */ 56 | #define FORMAT_CODE_PY_SSIZE_T "%" PY_FORMAT_SIZE_T "d" 57 | 58 | /* FORMAT_CODE_SIZE_T is for plain size_t, not for Py_ssize_t: */ 59 | #ifdef _MSC_VER 60 | /* For MSVC: */ 61 | #define FORMAT_CODE_SIZE_T "%Iu" 62 | #else 63 | /* C99 standard format code: */ 64 | #define FORMAT_CODE_SIZE_T "%zu" 65 | #endif 66 | 67 | /* Abstract from text type. Only supported for ASCII and UTF-8 */ 68 | #if PY_MAJOR_VERSION < 3 69 | #define Text_Type PyString_Type 70 | #define Text_Check(s) PyString_Check(s) 71 | #define Text_Format(f,a) PyString_Format(f,a) 72 | #define Text_FromUTF8(s) PyString_FromString(s) 73 | #define Text_FromUTF8AndSize(s,n) PyString_FromStringAndSize(s,n) 74 | #else 75 | #define Text_Type PyUnicode_Type 76 | #define Text_Check(s) PyUnicode_Check(s) 77 | #define Text_Format(f,a) PyUnicode_Format(f,a) 78 | #define Text_FromUTF8(s) PyUnicode_FromString(s) 79 | #define Text_FromUTF8AndSize(s,n) PyUnicode_FromStringAndSize(s,n) 80 | #endif 81 | 82 | #if PY_MAJOR_VERSION > 2 83 | #define PyInt_Type PyLong_Type 84 | #define PyInt_Check PyLong_Check 85 | #define PyInt_AsLong PyLong_AsLong 86 | #define PyInt_FromLong PyLong_FromLong 87 | #define PyInt_FromSsize_t PyLong_FromSsize_t 88 | #define PyExc_StandardError PyExc_Exception 89 | #define PyString_FromFormat PyUnicode_FromFormat 90 | #define Py_TPFLAGS_HAVE_ITER 0L 91 | #define Py_TPFLAGS_HAVE_RICHCOMPARE 0L 92 | #define Py_TPFLAGS_HAVE_WEAKREFS 0L 93 | #ifndef PyNumber_Int 94 | #define PyNumber_Int PyNumber_Long 95 | #endif 96 | #endif /* PY_MAJOR_VERSION > 2 */ 97 | 98 | #if PY_MAJOR_VERSION < 3 99 | #define Bytes_Type PyString_Type 100 | #define Bytes_Check PyString_Check 101 | #define Bytes_CheckExact PyString_CheckExact 102 | #define Bytes_AS_STRING PyString_AS_STRING 103 | #define Bytes_GET_SIZE PyString_GET_SIZE 104 | #define Bytes_Size PyString_Size 105 | #define Bytes_AsString PyString_AsString 106 | #define Bytes_AsStringAndSize PyString_AsStringAndSize 107 | #define Bytes_FromString PyString_FromString 108 | #define Bytes_FromStringAndSize PyString_FromStringAndSize 109 | #define Bytes_FromFormat PyString_FromFormat 110 | #define Bytes_ConcatAndDel PyString_ConcatAndDel 111 | #define _Bytes_Resize _PyString_Resize 112 | 113 | #else 114 | 115 | #define Bytes_Type PyBytes_Type 116 | #define Bytes_Check PyBytes_Check 117 | #define Bytes_CheckExact PyBytes_CheckExact 118 | #define Bytes_AS_STRING PyBytes_AS_STRING 119 | #define Bytes_GET_SIZE PyBytes_GET_SIZE 120 | #define Bytes_Size PyBytes_Size 121 | #define Bytes_AsString PyBytes_AsString 122 | #define Bytes_AsStringAndSize PyBytes_AsStringAndSize 123 | #define Bytes_FromString PyBytes_FromString 124 | #define Bytes_FromStringAndSize PyBytes_FromStringAndSize 125 | #define Bytes_FromFormat PyBytes_FromFormat 126 | #define Bytes_ConcatAndDel PyBytes_ConcatAndDel 127 | #define _Bytes_Resize _PyBytes_Resize 128 | 129 | #endif 130 | 131 | HIDDEN PyObject *Bytes_Format(PyObject *format, PyObject *args); 132 | 133 | /* Mangle the module name into the name of the module init function */ 134 | #if PY_MAJOR_VERSION > 2 135 | #define INIT_MODULE(m) PyInit_ ## m 136 | #else 137 | #define INIT_MODULE(m) init ## m 138 | #endif 139 | 140 | #endif /* !defined(PSYCOPG_PYTHON_H) */ 141 | -------------------------------------------------------------------------------- /logicaldecoding/reader_type.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "pylogicaldecoding.h" 4 | #include 5 | #include 6 | 7 | #define reader_stream_doc \ 8 | "stream() -> start the main loop" 9 | 10 | #define reader_stop_doc \ 11 | "stop() -> stop the main loop" 12 | 13 | #define reader_acknowledge_doc \ 14 | "ack() -> send feedback message acknowledging all preceding stream\n"\ 15 | "It's user's responsibility to send regular acknowledgements. If"\ 16 | "omited, the master keeps all its WAL on disk and eventually"\ 17 | "Cthulhu eats the physical server" 18 | 19 | #define reader_drop_slot_doc \ 20 | "drop_slot() -> drop the replication slot" 21 | 22 | // 10s 23 | #define MAX_RETRY_INTERVAL 10000000 24 | 25 | extern volatile sig_atomic_t global_abort; 26 | 27 | typedef struct PyLDReader{ 28 | PyObject_HEAD 29 | pghx_ld_reader reader; 30 | } PyLDReader; 31 | 32 | int 33 | reader_stream_cb(void *self, char *data) 34 | { 35 | PyObject *pFunc = NULL, 36 | *pArgs = NULL, 37 | *pValue = NULL, 38 | *result = NULL; 39 | 40 | pFunc = PyObject_GetAttrString((PyObject *)self, "event"); 41 | if (pFunc == NULL){goto error;} 42 | pArgs = PyTuple_New(1); 43 | if (pArgs == NULL){goto error;} 44 | pValue = Text_FromUTF8(data); 45 | Py_INCREF(pValue); 46 | if (pValue == NULL){goto error;} 47 | PyTuple_SetItem(pArgs, 0, pValue); 48 | result = PyObject_CallObject(pFunc, pArgs); 49 | if (result == NULL){goto error;} 50 | Py_DECREF(pFunc); 51 | Py_DECREF(pArgs); 52 | Py_DECREF(pValue); 53 | Py_DECREF(result); 54 | return 1; 55 | 56 | error: 57 | Py_XDECREF(pFunc); 58 | Py_XDECREF(pArgs); 59 | Py_XDECREF(pValue); 60 | Py_XDECREF(result); 61 | return 0; 62 | } 63 | 64 | static void 65 | py_set_pghx_error(void){ 66 | PyObject *exception; 67 | static PyObject *error_map[PGHX_ERRORS_NUM] = { NULL }; 68 | // early return in case of OOM (it soon gonna crash anyway, 69 | // let's try to write the cause) 70 | if (pghx_error == PGHX_OUT_OF_MEMORY) 71 | { 72 | fprintf(stderr, "%s\n", pghx_error_info); 73 | fflush(stderr); 74 | PyErr_NoMemory(); 75 | return; 76 | } 77 | // this is bad, maybe we should raise something ugly or exit. 78 | if (pghx_error == PGHX_NO_ERROR) 79 | { 80 | fputs("py_set_pghx_error() called without error\n", stderr); 81 | fflush(stderr); 82 | return; 83 | } 84 | 85 | // initialize the error map 86 | // TODO: use or create sensible exceptions 87 | if (!error_map[1]) 88 | { 89 | error_map[PGHX_IO_ERROR] = PyExc_ValueError; 90 | error_map[PGHX_OUT_OF_MEMORY] = PyExc_ValueError; 91 | 92 | // generic DB errors 93 | error_map[PGHX_CONNECTION_ERROR] = PyExc_ValueError; 94 | error_map[PGHX_PASSWORD_ERROR] = PyExc_ValueError; 95 | error_map[PGHX_COMMAND_ERROR] = PyExc_ValueError; 96 | error_map[PGHX_QUERY_ERROR] = PyExc_ValueError; 97 | 98 | // logical decoding specific errors 99 | error_map[PGHX_LD_STREAM_PROTOCOL_ERROR] = PyExc_ValueError; 100 | error_map[PGHX_LD_REPLICATION_ERROR] = PyExc_ValueError; 101 | error_map[PGHX_LD_NO_SLOT] = PyExc_ValueError; 102 | error_map[PGHX_LD_BAD_PLUGIN] = PyExc_ValueError; 103 | error_map[PGHX_LD_STATUS_ERROR] = PyExc_ValueError; 104 | error_map[PGHX_LD_PARSE_ERROR] = PyExc_ValueError; 105 | } 106 | exception = error_map[pghx_error]; 107 | // probably something new in pghx, we don't handle yet 108 | if (!exception){ 109 | PyErr_Format(PyExc_Exception, "Unknown error: %s", pghx_error_info); 110 | } 111 | else 112 | { 113 | PyErr_SetString(error_map[(int)pghx_error], pghx_error_info); 114 | } 115 | } 116 | 117 | static int 118 | reader_init(PyObject *obj, PyObject *args, PyObject *kwargs) 119 | { 120 | PyLDReader *self = (PyLDReader *)obj; 121 | pghx_ld_reader *r = &(self->reader); 122 | 123 | static char *kwlist[] = { 124 | "host", "port", "username", "dbname", "password", 125 | "progname", "plugin", "slot", "create_slot", "feedback_interval", 126 | "connection_timeout", NULL}; 127 | 128 | pghx_ld_reader_init(r); 129 | 130 | if (!PyArg_ParseTupleAndKeywords( 131 | args, kwargs, "|ssssssssbil", kwlist, 132 | &(r->host), &(r->port), &(r->username), 133 | &(r->dbname), &(r->password), &(r->progname), 134 | &(r->plugin), &(r->slot), &(r->create_slot), 135 | &(r->standby_message_timeout),&(r->connection_timeout))) 136 | return -1; 137 | 138 | r->stream_cb = reader_stream_cb; 139 | r->user_data = (void *)self; 140 | // test a connection 141 | return pghx_ld_reader_connect(r, true); 142 | } 143 | 144 | static PyObject * 145 | reader_new(PyTypeObject *type, PyObject *args, PyObject *kwds) 146 | { 147 | return type->tp_alloc(type, 0); 148 | } 149 | 150 | static PyObject * 151 | reader_repr(PyLDReader *self) 152 | { 153 | pghx_ld_reader *r = &(self->reader); 154 | 155 | return PyString_FromFormat( 156 | "", self, r->slot); 157 | } 158 | 159 | static void 160 | reader_dealloc(PyObject* obj) 161 | { 162 | PyLDReader *self = (PyLDReader *)obj; 163 | pghx_ld_reader *r = &(self->reader); 164 | 165 | if (r->conn) 166 | PQfinish(r->conn); 167 | 168 | // TODO: these come from PyParseArgs. Not sure if free is 169 | // perfectly safe 170 | { 171 | free(r->host); 172 | free(r->port); 173 | free(r->username); 174 | free(r->dbname); 175 | free(r->password); 176 | free(r->progname); 177 | free(r->plugin); 178 | free(r->slot); 179 | } 180 | 181 | Py_TYPE(obj)->tp_free(obj); 182 | } 183 | 184 | static PyObject * 185 | py_reader_stream(PyLDReader *self) 186 | { 187 | if(!pghx_ld_reader_stream(&(self->reader))) 188 | { 189 | py_set_pghx_error(); 190 | return NULL; 191 | } 192 | Py_RETURN_NONE; 193 | } 194 | 195 | static PyObject * 196 | py_reader_stop(PyLDReader *self) 197 | { 198 | if(!pghx_ld_reader_stop(&(self->reader))) 199 | { 200 | py_set_pghx_error(); 201 | return NULL; 202 | } 203 | Py_RETURN_NONE; 204 | } 205 | 206 | static PyObject * 207 | py_reader_acknowledge(PyLDReader *self) 208 | { 209 | if(!pghx_ld_reader_acknowledge(&(self->reader))) 210 | { 211 | py_set_pghx_error(); 212 | return NULL; 213 | } 214 | Py_RETURN_NONE; 215 | } 216 | 217 | static PyObject * 218 | py_reader_drop_slot(PyLDReader *self) 219 | { 220 | if(!pghx_ld_reader_drop_slot(&(self->reader))) 221 | { 222 | py_set_pghx_error(); 223 | return NULL; 224 | } 225 | Py_RETURN_NONE; 226 | } 227 | 228 | static struct PyMethodDef reader_methods[] = { 229 | {"stream", (PyCFunction)py_reader_stream, METH_NOARGS, 230 | reader_stream_doc}, 231 | {"stop", (PyCFunction)py_reader_stop, METH_NOARGS, 232 | reader_stop_doc}, 233 | {"ack", (PyCFunction)py_reader_acknowledge, METH_NOARGS, 234 | reader_acknowledge_doc}, 235 | {"drop_slot", (PyCFunction)py_reader_drop_slot, METH_NOARGS, 236 | reader_drop_slot_doc}, 237 | {NULL} 238 | }; 239 | 240 | static struct PyMemberDef reader_members[] = { 241 | /*{"conn_params", T_OBJECT, offsetof(PyLDReader, conn_params), READONLY},*/ 242 | {NULL} 243 | }; 244 | 245 | #define readerType_doc \ 246 | "Reader(dsn) -> new reader object\n\n" 247 | 248 | PyTypeObject readerType = { 249 | PyVarObject_HEAD_INIT(NULL, 0) 250 | "logicaldecoding.Reader", 251 | sizeof(PyLDReader), 0, 252 | reader_dealloc, /*tp_dealloc*/ 253 | 0, /*tp_print*/ 254 | 0, /*tp_getattr*/ 255 | 0, /*tp_setattr*/ 256 | 0, /*tp_compare*/ 257 | (reprfunc)reader_repr, /*tp_repr*/ 258 | 0, /*tp_as_number*/ 259 | 0, /*tp_as_sequence*/ 260 | 0, /*tp_as_mapping*/ 261 | 0, /*tp_hash */ 262 | 0, /*tp_call*/ 263 | (reprfunc)reader_repr, /*tp_str*/ 264 | 0, /*tp_getattro*/ 265 | 0, /*tp_setattro*/ 266 | 0, /*tp_as_buffer*/ 267 | Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, 268 | /*tp_flags*/ 269 | readerType_doc, /*tp_doc*/ 270 | 271 | //(traverseproc)connection_traverse, /*tp_traverse*/ 272 | 0, 273 | 274 | //(inquiry)connection_clear, /*tp_clear*/ 275 | 0, 276 | 277 | 0, /*tp_richcompare*/ 278 | 279 | //offsetof(connectionObject, weakreflist), /* tp_weaklistoffset */ 280 | 0, 281 | 282 | 0, /*tp_iter*/ 283 | 0, /*tp_iternext*/ 284 | reader_methods, /*tp_methods*/ 285 | reader_members, /*tp_members*/ 286 | 287 | //connectionObject_getsets, /*tp_getset*/ 288 | 0, 289 | 290 | 0, /*tp_base*/ 291 | 0, /*tp_dict*/ 292 | 0, /*tp_descr_get*/ 293 | 0, /*tp_descr_set*/ 294 | 0, /*tp_dictoffset*/ 295 | reader_init, /*tp_init*/ 296 | 297 | 0, /*tp_alloc*/ 298 | reader_new, /*tp_new*/ 299 | }; 300 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [build_ext] 2 | define= 3 | 4 | # PSYCOPG_DISPLAY_SIZE enable display size calculation (a little slower) 5 | # HAVE_PQFREEMEM should be defined on PostgreSQL >= 7.4 6 | # PSYCOPG_DEBUG can be added to enable verbose debug information 7 | 8 | # "pg_config" is required to locate PostgreSQL headers and libraries needed to 9 | # build psycopg2. If pg_config is not in the path or is installed under a 10 | # different name uncomment the following option and set it to the pg_config 11 | # full path. 12 | #pg_config= 13 | 14 | # Set to 1 to use Python datetime objects for default date/time representation. 15 | use_pydatetime=1 16 | 17 | # If the build system does not find the mx.DateTime headers, try 18 | # uncommenting the following line and setting its value to the right path. 19 | #mx_include_dir= 20 | 21 | # For Windows only: 22 | # Set to 1 if the PostgreSQL library was built with OpenSSL. 23 | # Required to link in OpenSSL libraries and dependencies. 24 | have_ssl=0 25 | 26 | # Statically link against the postgresql client library. 27 | #static_libpq=1 28 | 29 | # Add here eventual extra libraries required to link the module. 30 | #libraries= 31 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # setup.py - distutils packaging 2 | # 3 | # Copyright (C) 2003-2010 Federico Di Gregorio 4 | # 5 | # psycopg2 is free software: you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published 7 | # by the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # psycopg2 is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | 15 | """Python-PostgreSQL Database Adapter 16 | 17 | psycopg2 is a PostgreSQL database adapter for the Python programming 18 | language. psycopg2 was written with the aim of being very small and fast, 19 | and stable as a rock. 20 | 21 | psycopg2 is different from the other database adapter because it was 22 | designed for heavily multi-threaded applications that create and destroy 23 | lots of cursors and make a conspicuous number of concurrent INSERTs or 24 | UPDATEs. psycopg2 also provide full asynchronous operations and support 25 | for coroutine libraries. 26 | """ 27 | 28 | # note: if you are changing the list of supported Python version please fix 29 | # the docs in install.rst and the /features/ page on the website. 30 | classifiers = """\ 31 | Development Status :: 1 - Experimental 32 | Intended Audience :: Developers 33 | License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL) 34 | Programming Language :: Python 35 | Programming Language :: Python :: 2.7 36 | Programming Language :: C 37 | Programming Language :: SQL 38 | Topic :: Database 39 | Topic :: Database :: Front-Ends 40 | Topic :: Software Development 41 | Topic :: Software Development :: Libraries :: Python Modules 42 | Operating System :: Unix 43 | """ 44 | 45 | # Note: The setup.py must be compatible with both Python 2 and 3 46 | 47 | import os 48 | import sys 49 | import re 50 | import subprocess 51 | from distutils.core import setup, Extension 52 | from distutils.command.build_ext import build_ext 53 | from distutils.sysconfig import get_python_inc 54 | from distutils.ccompiler import get_default_compiler 55 | from distutils.util import get_platform 56 | 57 | from distutils.command.build_py import build_py 58 | 59 | try: 60 | import configparser 61 | except ImportError: 62 | import ConfigParser as configparser 63 | 64 | PYLD_VERSION = '0.1.dev0' 65 | version_flags = ['dt', 'dec'] 66 | 67 | 68 | class PostgresConfig: 69 | def __init__(self, build_ext): 70 | self.build_ext = build_ext 71 | self.pg_config_exe = self.build_ext.pg_config 72 | if not self.pg_config_exe: 73 | self.pg_config_exe = self.autodetect_pg_config_path() 74 | if self.pg_config_exe is None: 75 | sys.stderr.write("""\ 76 | Error: pg_config executable not found. 77 | 78 | Please add the directory containing pg_config to the PATH 79 | or specify the full executable path with the option: 80 | 81 | python setup.py build_ext --pg-config /path/to/pg_config build ... 82 | 83 | or with the pg_config option in 'setup.cfg'. 84 | """) 85 | sys.exit(1) 86 | 87 | def query(self, attr_name): 88 | """Spawn the pg_config executable, querying for the given config 89 | name, and return the printed value, sanitized. """ 90 | try: 91 | pg_config_process = subprocess.Popen( 92 | [self.pg_config_exe, "--" + attr_name], 93 | stdin=subprocess.PIPE, 94 | stdout=subprocess.PIPE, 95 | stderr=subprocess.PIPE) 96 | except OSError: 97 | raise Warning("Unable to find 'pg_config' file in '%s'" % 98 | self.pg_config_exe) 99 | pg_config_process.stdin.close() 100 | result = pg_config_process.stdout.readline().strip() 101 | if not result: 102 | raise Warning(pg_config_process.stderr.readline()) 103 | if not isinstance(result, str): 104 | result = result.decode('ascii') 105 | return result 106 | 107 | def find_on_path(self, exename, path_directories=None): 108 | if not path_directories: 109 | path_directories = os.environ['PATH'].split(os.pathsep) 110 | for dir_name in path_directories: 111 | fullpath = os.path.join(dir_name, exename) 112 | if os.path.isfile(fullpath): 113 | return fullpath 114 | return None 115 | 116 | def autodetect_pg_config_path(self): 117 | """Find and return the path to the pg_config executable.""" 118 | return self.find_on_path('pg_config') 119 | 120 | def _get_pg_config_from_registry(self): 121 | try: 122 | import winreg 123 | except ImportError: 124 | import _winreg as winreg 125 | 126 | reg = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) 127 | try: 128 | pg_inst_list_key = winreg.OpenKey(reg, 129 | 'SOFTWARE\\PostgreSQL\\Installations') 130 | except EnvironmentError: 131 | # No PostgreSQL installation, as best as we can tell. 132 | return None 133 | 134 | try: 135 | # Determine the name of the first subkey, if any: 136 | try: 137 | first_sub_key_name = winreg.EnumKey(pg_inst_list_key, 0) 138 | except EnvironmentError: 139 | return None 140 | 141 | pg_first_inst_key = winreg.OpenKey(reg, 142 | 'SOFTWARE\\PostgreSQL\\Installations\\' 143 | + first_sub_key_name) 144 | try: 145 | pg_inst_base_dir = winreg.QueryValueEx( 146 | pg_first_inst_key, 'Base Directory')[0] 147 | finally: 148 | winreg.CloseKey(pg_first_inst_key) 149 | 150 | finally: 151 | winreg.CloseKey(pg_inst_list_key) 152 | 153 | pg_config_path = os.path.join( 154 | pg_inst_base_dir, 'bin', 'pg_config.exe') 155 | if not os.path.exists(pg_config_path): 156 | return None 157 | 158 | # Support unicode paths, if this version of Python provides the 159 | # necessary infrastructure: 160 | if sys.version_info[0] < 3 \ 161 | and hasattr(sys, 'getfilesystemencoding'): 162 | pg_config_path = pg_config_path.encode( 163 | sys.getfilesystemencoding()) 164 | 165 | return pg_config_path 166 | 167 | 168 | class pyld_build_ext(build_ext): 169 | """Conditionally complement the setup.cfg options file. 170 | 171 | This class configures the include_dirs, library_dirs, libraries 172 | options as required by the system. Most of the configuration happens 173 | in finalize_options() method. 174 | 175 | If you want to set up the build step for a peculiar platform, add a 176 | method finalize_PLAT(), where PLAT matches your sys.platform. 177 | """ 178 | user_options = build_ext.user_options[:] 179 | user_options.extend([ 180 | ('use-pydatetime', None, 181 | "Use Python datatime objects for date and time representation."), 182 | ('pg-config=', None, 183 | "The name of the pg_config binary and/or full path to find it"), 184 | ('static-libpq', None, 185 | "Statically link the PostgreSQL client library"), 186 | ]) 187 | 188 | boolean_options = build_ext.boolean_options[:] 189 | boolean_options.extend(('use-pydatetime', 'have-ssl', 'static-libpq')) 190 | 191 | def __init__(self, *args, **kwargs): 192 | build_ext.__init__(self, *args, **kwargs) 193 | 194 | def initialize_options(self): 195 | build_ext.initialize_options(self) 196 | self.use_pg_dll = 1 197 | self.pgdir = None 198 | self.mx_include_dir = None 199 | self.use_pydatetime = 1 200 | self.have_ssl = have_ssl 201 | self.static_libpq = static_libpq 202 | self.pg_config = None 203 | 204 | def compiler_is_msvc(self): 205 | return self.get_compiler_name().lower().startswith('msvc') 206 | 207 | def compiler_is_mingw(self): 208 | return self.get_compiler_name().lower().startswith('mingw') 209 | 210 | def get_compiler_name(self): 211 | """Return the name of the C compiler used to compile extensions. 212 | 213 | If a compiler was not explicitly set (on the command line, for 214 | example), fall back on the default compiler. 215 | """ 216 | if self.compiler: 217 | # distutils doesn't keep the type of self.compiler uniform; we 218 | # compensate: 219 | if isinstance(self.compiler, str): 220 | name = self.compiler 221 | else: 222 | name = self.compiler.compiler_type 223 | else: 224 | name = get_default_compiler() 225 | return name 226 | 227 | def get_export_symbols(self, extension): 228 | # Fix MSVC seeing two of the same export symbols. 229 | if self.compiler_is_msvc(): 230 | return [] 231 | else: 232 | return build_ext.get_export_symbols(self, extension) 233 | 234 | def build_extension(self, extension): 235 | build_ext.build_extension(self, extension) 236 | sysVer = sys.version_info[:2] 237 | 238 | # For Python versions that use MSVC compiler 2008, re-insert the 239 | # manifest into the resulting .pyd file. 240 | if self.compiler_is_msvc() and sysVer not in ((2, 4), (2, 5)): 241 | platform = get_platform() 242 | # Default to the x86 manifest 243 | manifest = '_psycopg.vc9.x86.manifest' 244 | if platform == 'win-amd64': 245 | manifest = '_psycopg.vc9.amd64.manifest' 246 | try: 247 | ext_path = self.get_ext_fullpath(extension.name) 248 | except AttributeError: 249 | ext_path = os.path.join(self.build_lib, 250 | 'psycopg2', '_psycopg.pyd') 251 | self.compiler.spawn( 252 | ['mt.exe', '-nologo', '-manifest', 253 | os.path.join('psycopg', manifest), 254 | '-outputresource:%s;2' % ext_path]) 255 | 256 | def finalize_win32(self): 257 | """Finalize build system configuration on win32 platform.""" 258 | sysVer = sys.version_info[:2] 259 | 260 | # Add compiler-specific arguments: 261 | extra_compiler_args = [] 262 | 263 | if self.compiler_is_mingw(): 264 | # Default MinGW compilation of Python extensions on Windows uses 265 | # only -O: 266 | extra_compiler_args.append('-O3') 267 | 268 | # GCC-compiled Python on non-Windows platforms is built with strict 269 | # aliasing disabled, but that must be done explicitly on Windows to 270 | # avoid large numbers of warnings for perfectly idiomatic Python C 271 | # API code. 272 | extra_compiler_args.append('-fno-strict-aliasing') 273 | 274 | # Force correct C runtime library linkage: 275 | if sysVer <= (2, 3): 276 | # Yes: 'msvcr60', rather than 'msvcrt', is the correct value 277 | # on the line below: 278 | self.libraries.append('msvcr60') 279 | elif sysVer in ((2, 4), (2, 5)): 280 | self.libraries.append('msvcr71') 281 | # Beyond Python 2.5, we take our chances on the default C runtime 282 | # library, because we don't know what compiler those future 283 | # versions of Python will use. 284 | 285 | for extension in ext: # ext is a global list of Extension objects 286 | extension.extra_compile_args.extend(extra_compiler_args) 287 | # End of add-compiler-specific arguments section. 288 | 289 | self.libraries.append("ws2_32") 290 | self.libraries.append("advapi32") 291 | if self.compiler_is_msvc(): 292 | # MSVC requires an explicit "libpq" 293 | self.libraries.remove("pq") 294 | self.libraries.append("secur32") 295 | self.libraries.append("libpq") 296 | self.libraries.append("shfolder") 297 | for path in self.library_dirs: 298 | if os.path.isfile(os.path.join(path, "ms", "libpq.lib")): 299 | self.library_dirs.append(os.path.join(path, "ms")) 300 | break 301 | if self.have_ssl: 302 | self.libraries.append("libeay32") 303 | self.libraries.append("ssleay32") 304 | self.libraries.append("crypt32") 305 | self.libraries.append("user32") 306 | self.libraries.append("gdi32") 307 | 308 | def finalize_darwin(self): 309 | """Finalize build system configuration on darwin platform.""" 310 | self.libraries.append('ssl') 311 | self.libraries.append('crypto') 312 | 313 | def finalize_linux(self): 314 | """Finalize build system configuration on GNU/Linux platform.""" 315 | # tell piro that GCC is fine and dandy, but not so MS compilers 316 | for extension in self.extensions: 317 | extension.extra_compile_args.append( 318 | '-Wdeclaration-after-statement') 319 | 320 | finalize_linux2 = finalize_linux 321 | finalize_linux3 = finalize_linux 322 | 323 | def finalize_options(self): 324 | """Complete the build system configuration.""" 325 | build_ext.finalize_options(self) 326 | 327 | pg_config_helper = PostgresConfig(self) 328 | 329 | self.include_dirs.append(".") 330 | if self.static_libpq: 331 | if not getattr(self, 'link_objects', None): 332 | self.link_objects = [] 333 | self.link_objects.append( 334 | os.path.join(pg_config_helper.query("libdir"), "libpq.a")) 335 | else: 336 | self.libraries.append("pq") 337 | 338 | try: 339 | self.library_dirs.append(pg_config_helper.query("libdir")) 340 | self.include_dirs.append(pg_config_helper.query("includedir")) 341 | self.include_dirs.append(pg_config_helper.query("includedir-server")) 342 | try: 343 | # Here we take a conservative approach: we suppose that 344 | # *at least* PostgreSQL 7.4 is available (this is the only 345 | # 7.x series supported by psycopg 2) 346 | pgversion = pg_config_helper.query("version").split()[1] 347 | except: 348 | pgversion = "7.4.0" 349 | 350 | verre = re.compile( 351 | r"(\d+)\.(\d+)(?:(?:\.(\d+))|(devel|(alpha|beta|rc)\d+))") 352 | m = verre.match(pgversion) 353 | if m: 354 | pgmajor, pgminor, pgpatch = m.group(1, 2, 3) 355 | if pgpatch is None or not pgpatch.isdigit(): 356 | pgpatch = 0 357 | pgmajor = int(pgmajor) 358 | pgminor = int(pgminor) 359 | pgpatch = int(pgpatch) 360 | else: 361 | sys.stderr.write( 362 | "Error: could not determine PostgreSQL version from '%s'" 363 | % pgversion) 364 | sys.exit(1) 365 | 366 | define_macros.append(("PG_VERSION_HEX", "0x%02X%02X%02X" % 367 | (pgmajor, pgminor, pgpatch))) 368 | 369 | # enable lo64 if libpq >= 9.3 and Python 64 bits 370 | if (pgmajor, pgminor) >= (9, 3) and is_py_64(): 371 | define_macros.append(("HAVE_LO64", "1")) 372 | 373 | # Inject the flag in the version string already packed up 374 | # because we didn't know the version before. 375 | # With distutils everything is complicated. 376 | for i, t in enumerate(define_macros): 377 | if t[0] == 'PSYCOPG_VERSION': 378 | n = t[1].find(')') 379 | if n > 0: 380 | define_macros[i] = ( 381 | t[0], t[1][:n] + ' lo64' + t[1][n:]) 382 | 383 | except Warning: 384 | w = sys.exc_info()[1] # work around py 2/3 different syntax 385 | sys.stderr.write("Error: %s\n" % w) 386 | sys.exit(1) 387 | 388 | if hasattr(self, "finalize_" + sys.platform): 389 | getattr(self, "finalize_" + sys.platform)() 390 | 391 | def is_py_64(): 392 | # sys.maxint not available since Py 3.1; 393 | # sys.maxsize not available before Py 2.6; 394 | # this is portable at least between Py 2.4 and 3.4. 395 | import struct 396 | return struct.calcsize("P") > 4 397 | 398 | 399 | # let's start with macro definitions (the ones not already in setup.cfg) 400 | define_macros = [] 401 | include_dirs = [] 402 | 403 | # gather information to build the extension module 404 | ext = [] 405 | data_files = [] 406 | 407 | # sources 408 | 409 | sources = [ 410 | 'logicaldecodingmodule.c', 411 | "connection.c", 412 | 'reader_type.c', 413 | ] 414 | 415 | depends = [ 416 | "python.h", 417 | "connection.h", 418 | "pylogicaldecoding.h", 419 | ] 420 | 421 | parser = configparser.ConfigParser() 422 | parser.read('setup.cfg') 423 | 424 | # Choose a datetime module 425 | have_pydatetime = True 426 | have_mxdatetime = False 427 | use_pydatetime = int(parser.get('build_ext', 'use_pydatetime')) 428 | 429 | # check for mx package 430 | if parser.has_option('build_ext', 'mx_include_dir'): 431 | mxincludedir = parser.get('build_ext', 'mx_include_dir') 432 | else: 433 | mxincludedir = os.path.join(get_python_inc(plat_specific=1), "mx") 434 | if os.path.exists(mxincludedir): 435 | # Build the support for mx: we will check at runtime if it can be imported 436 | include_dirs.append(mxincludedir) 437 | define_macros.append(('HAVE_MXDATETIME', '1')) 438 | sources.append('adapter_mxdatetime.c') 439 | depends.extend(['adapter_mxdatetime.h', 'typecast_mxdatetime.c']) 440 | have_mxdatetime = True 441 | version_flags.append('mx') 442 | 443 | # now decide which package will be the default for date/time typecasts 444 | if have_pydatetime and (use_pydatetime or not have_mxdatetime): 445 | define_macros.append(('PYLD_DEFAULT_PYDATETIME', '1')) 446 | elif have_mxdatetime: 447 | define_macros.append(('PYLD_DEFAULT_MXDATETIME', '1')) 448 | else: 449 | error_message = """\ 450 | psycopg requires a datetime module: 451 | mx.DateTime module not found 452 | python datetime module not found 453 | 454 | Note that psycopg needs the module headers and not just the module 455 | itself. If you installed Python or mx.DateTime from a binary package 456 | you probably need to install its companion -dev or -devel package.""" 457 | 458 | for line in error_message.split("\n"): 459 | sys.stderr.write("error: " + line) 460 | sys.exit(1) 461 | 462 | # generate a nice version string to avoid confusion when users report bugs 463 | version_flags.append('pq3') # no more a choice 464 | version_flags.append('ext') # no more a choice 465 | 466 | if version_flags: 467 | PYLD_VERSION_EX = PYLD_VERSION + " (%s)" % ' '.join(version_flags) 468 | else: 469 | PYLD_VERSION_EX = PYLD_VERSION 470 | 471 | #define_macros.append(('PYLD_VERSION', '\\"' + PYLD_VERSION_EX + '\\"')) 472 | 473 | if parser.has_option('build_ext', 'have_ssl'): 474 | have_ssl = int(parser.get('build_ext', 'have_ssl')) 475 | else: 476 | have_ssl = 0 477 | 478 | if parser.has_option('build_ext', 'static_libpq'): 479 | static_libpq = int(parser.get('build_ext', 'static_libpq')) 480 | else: 481 | static_libpq = 0 482 | 483 | # And now... explicitly add the defines from the .cfg files. 484 | # Looks like setuptools or some other cog doesn't add them to the command line 485 | # when called e.g. with "pip -e git+url'. This results in declarations 486 | # duplicate on the commandline, which I hope is not a problem. 487 | for define in parser.get('build_ext', 'define').split(','): 488 | if define: 489 | define_macros.append((define, '1')) 490 | 491 | # build the extension 492 | 493 | sources = [ os.path.join('logicaldecoding', x) for x in sources] 494 | depends = [ os.path.join('logicaldecoding', x) for x in depends] 495 | 496 | ext.append(Extension("logicaldecoding._logicaldecoding", sources, 497 | define_macros=define_macros, 498 | libraries=['pghx'], 499 | include_dirs=include_dirs, 500 | depends=depends, 501 | undef_macros=[])) 502 | 503 | try: 504 | f = open("README.rst") 505 | readme = f.read() 506 | f.close() 507 | except: 508 | print("failed to read readme: ignoring...") 509 | readme = __doc__ 510 | 511 | setup(name="pslogicaldecoding", 512 | version=PYLD_VERSION, 513 | maintainer="Bruno Dupuis", 514 | maintainer_email="bd@lisael.org", 515 | author="Federico Di Gregorio", 516 | author_email="fog@initd.org", 517 | url="http://initd.org/psycopg/", 518 | #download_url=download_url, 519 | license="LGPL with exceptions", 520 | platforms=["Unix"], 521 | description=readme.split("\n")[0], 522 | long_description="\n".join(readme.split("\n")[2:]).lstrip(), 523 | classifiers=[x for x in classifiers.split("\n") if x], 524 | data_files=data_files, 525 | package_dir={'logicaldecoding': 'lib', 'logicaldecoding.tests': 'tests'}, 526 | packages=['logicaldecoding', 'logicaldecoding.tests'], 527 | cmdclass={ 528 | 'build_ext': pyld_build_ext, 529 | 'build_py': build_py, }, 530 | ext_modules=ext) 531 | 532 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lisael/pylogicaldecoding/f874ff0b25cd576cd135a9d6335d7dd3b28d766f/tests/__init__.py --------------------------------------------------------------------------------