├── .gitignore ├── Makefile ├── README.md └── decoding_json.c /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.so 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | MODULES = decoding_json 2 | 3 | PG_CONFIG = pg_config 4 | PGXS := $(shell $(PG_CONFIG) --pgxs) 5 | include $(PGXS) 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | decoding_json 2 | ============= 3 | 4 | This plugin receives the changes from WAL and 5 | decodes them to JSON. 6 | 7 | Adapted from postgresql-9.4.1/contrib/test_decoding 8 | 9 | format 10 | ------ 11 | {"type":"transaction.begin","xid":"2010561","committed":"2015-04-22 19:23:35.714443+00"} 12 | {"type":"table","name":"abc","change":"INSERT","data":{"a":6,"b":7,"c":42}} 13 | {"type":"table","name":"abc","change":"UPDATE","key":{"a":6,"b":7},"data":{"a":6,"b":7,"c":13}} 14 | {"type":"table","name":"abc","change":"UPDATE","key":{"a":6,"b":7},"data":{"a":2,"b":7,"c":13}} 15 | {"type":"table","name":"abc","change":"DELETE","key":{"a":2,"b":7}} 16 | {"type":"transaction.commit","xid":"2010561","committed":"2015-04-22 19:23:35.714443+00"} 17 | 18 | install 19 | ------- 20 | sudo apt-get install postgresql-server-dev-9.4 21 | make 22 | sudo cp decoding_json.so /usr/lib/postgresql/9.4/lib/ 23 | 24 | configuration of postgresql 25 | --------------------------- 26 | see 27 | -------------------------------------------------------------------------------- /decoding_json.c: -------------------------------------------------------------------------------- 1 | #include "postgres.h" 2 | 3 | #include "access/genam.h" 4 | #include "access/sysattr.h" 5 | 6 | #include "catalog/pg_class.h" 7 | #include "catalog/pg_type.h" 8 | 9 | #include "nodes/parsenodes.h" 10 | 11 | #include "replication/output_plugin.h" 12 | #include "replication/logical.h" 13 | 14 | #include "utils/builtins.h" 15 | #include "utils/lsyscache.h" 16 | #include "utils/memutils.h" 17 | #include "utils/rel.h" 18 | #include "utils/relcache.h" 19 | #include "utils/syscache.h" 20 | #include "utils/typcache.h" 21 | 22 | PG_MODULE_MAGIC; 23 | 24 | extern void _PG_init(void); 25 | extern void _PG_output_plugin_init(OutputPluginCallbacks* cb); 26 | 27 | typedef struct _DecodingJsonData DecodingJsonData; 28 | struct _DecodingJsonData { 29 | MemoryContext context; 30 | bool xact_wrote_changes; 31 | }; 32 | 33 | static void pg_decode_startup(LogicalDecodingContext* ctx, OutputPluginOptions* opt, bool is_init); 34 | static void pg_decode_shutdown(LogicalDecodingContext* ctx); 35 | static void pg_decode_begin_txn(LogicalDecodingContext* ctx, ReorderBufferTXN* txn); 36 | static void pg_output_begin(LogicalDecodingContext* ctx, DecodingJsonData* data, ReorderBufferTXN* txn, bool last_write); 37 | static void pg_decode_commit_txn(LogicalDecodingContext* ctx, ReorderBufferTXN* txn, XLogRecPtr commit_lsn); 38 | static void pg_decode_change(LogicalDecodingContext* ctx, ReorderBufferTXN* txn, Relation rel, ReorderBufferChange* change); 39 | 40 | void _PG_init(void) { 41 | } 42 | 43 | void _PG_output_plugin_init(OutputPluginCallbacks *cb) { 44 | AssertVariableIsOfType(&_PG_output_plugin_init, LogicalOutputPluginInit); 45 | 46 | cb->startup_cb = pg_decode_startup; 47 | cb->begin_cb = pg_decode_begin_txn; 48 | cb->change_cb = pg_decode_change; 49 | cb->commit_cb = pg_decode_commit_txn; 50 | cb->shutdown_cb = pg_decode_shutdown; 51 | } 52 | 53 | static void pg_decode_startup(LogicalDecodingContext* ctx, OutputPluginOptions* opt, bool is_init) { 54 | DecodingJsonData* data; 55 | 56 | data = palloc0(sizeof(DecodingJsonData)); 57 | data->context = AllocSetContextCreate( 58 | ctx->context, 59 | "text conversion context", 60 | ALLOCSET_DEFAULT_MINSIZE, 61 | ALLOCSET_DEFAULT_INITSIZE, 62 | ALLOCSET_DEFAULT_MAXSIZE 63 | ); 64 | 65 | ctx->output_plugin_private = data; 66 | opt->output_type = OUTPUT_PLUGIN_TEXTUAL_OUTPUT; 67 | } 68 | 69 | static void pg_decode_shutdown(LogicalDecodingContext* ctx) { 70 | DecodingJsonData* data = ctx->output_plugin_private; 71 | MemoryContextDelete(data->context); 72 | } 73 | 74 | static void pg_decode_begin_txn(LogicalDecodingContext* ctx, ReorderBufferTXN* txn) { 75 | DecodingJsonData* data = ctx->output_plugin_private; 76 | data->xact_wrote_changes = false; 77 | pg_output_begin(ctx, data, txn, true); 78 | } 79 | 80 | static void pg_output_begin(LogicalDecodingContext* ctx, DecodingJsonData* data, ReorderBufferTXN* txn, bool last_write) { 81 | OutputPluginPrepareWrite(ctx, last_write); 82 | appendStringInfo( 83 | ctx->out, 84 | "{\"type\":\"transaction.begin\",\"xid\":\"%u\",\"committed\":\"%s\"}", 85 | txn->xid, 86 | timestamptz_to_str(txn->commit_time) 87 | ); 88 | OutputPluginWrite(ctx, last_write); 89 | } 90 | 91 | static void pg_decode_commit_txn(LogicalDecodingContext* ctx, ReorderBufferTXN* txn, XLogRecPtr commit_lsn) { 92 | OutputPluginPrepareWrite(ctx, true); 93 | appendStringInfo( 94 | ctx->out, 95 | "{\"type\":\"transaction.commit\",\"xid\":\"%u\",\"committed\":\"%s\"}", 96 | txn->xid, 97 | timestamptz_to_str(txn->commit_time) 98 | ); 99 | OutputPluginWrite(ctx, true); 100 | } 101 | 102 | static void print_literal(StringInfo s, Oid typid, char* outputstr) { 103 | const char* valptr; 104 | 105 | switch (typid) { 106 | case INT2OID: 107 | case INT4OID: 108 | case INT8OID: 109 | case OIDOID: 110 | case FLOAT4OID: 111 | case FLOAT8OID: 112 | case NUMERICOID: 113 | appendStringInfoString(s, outputstr); 114 | break; 115 | 116 | case BITOID: 117 | case VARBITOID: 118 | appendStringInfo(s, "\"B'%s'\"", outputstr); 119 | break; 120 | 121 | case BOOLOID: 122 | if (strcmp(outputstr, "t") == 0) 123 | appendStringInfoString(s, "true"); 124 | else 125 | appendStringInfoString(s, "false"); 126 | break; 127 | 128 | default: 129 | appendStringInfoChar(s, '"'); 130 | for (valptr = outputstr; *valptr; valptr++) { 131 | char ch = *valptr; 132 | if (ch == '\n') { 133 | appendStringInfoString(s, "\\n"); 134 | } else if (ch == '\r') { 135 | appendStringInfoString(s, "\\r"); 136 | } else if (ch == '\t') { 137 | appendStringInfoString(s, "\\t"); 138 | } else if (ch == '"') { 139 | appendStringInfoString(s, "\\\""); 140 | } else if (ch == '\\') { 141 | appendStringInfoString(s, "\\\\"); 142 | } else { 143 | appendStringInfoChar(s, ch); 144 | } 145 | } 146 | appendStringInfoChar(s, '"'); 147 | break; 148 | } 149 | } 150 | 151 | static void print_value(StringInfo s, TupleDesc tupdesc, HeapTuple tuple, int i) { 152 | bool typisvarlena; 153 | bool isnull; 154 | Oid typoutput; 155 | Form_pg_attribute attr = tupdesc->attrs[i]; 156 | Datum origval = fastgetattr(tuple, i + 1, tupdesc, &isnull); 157 | Oid typid = attr->atttypid; 158 | getTypeOutputInfo(typid, &typoutput, &typisvarlena); 159 | if (isnull) { 160 | appendStringInfoString(s, "null"); 161 | } else if (typisvarlena && VARATT_IS_EXTERNAL_ONDISK(origval)) { 162 | appendStringInfoString(s, "\"???unchanged-toast-datum???\""); 163 | } else if (!typisvarlena) { 164 | print_literal(s, typid, OidOutputFunctionCall(typoutput, origval)); 165 | } else { 166 | Datum val = PointerGetDatum(PG_DETOAST_DATUM(origval)); 167 | print_literal(s, typid, OidOutputFunctionCall(typoutput, val)); 168 | } 169 | } 170 | 171 | static void tuple_to_stringinfo(StringInfo s, TupleDesc tupdesc, HeapTuple tuple, bool skip_nulls) { 172 | int i; 173 | bool skip_comma = false; 174 | 175 | for (i = 0; i < tupdesc->natts; i++) { 176 | Form_pg_attribute attr = tupdesc->attrs[i]; 177 | 178 | if (attr->attisdropped || attr->attnum < 0) { 179 | skip_comma = true; 180 | continue; 181 | } 182 | 183 | if (i > 0 && !skip_comma) appendStringInfoChar(s, ','); 184 | appendStringInfo(s, "\"%s\":", NameStr(attr->attname)); 185 | print_value(s, tupdesc, tuple, i); 186 | 187 | if (skip_comma) { 188 | skip_comma = false; 189 | } 190 | } 191 | } 192 | 193 | static void pg_decode_change(LogicalDecodingContext* ctx, ReorderBufferTXN* txn, Relation relation, ReorderBufferChange* change) { 194 | DecodingJsonData* data; 195 | Form_pg_class class_form; 196 | TupleDesc tupdesc; 197 | HeapTuple tuple; 198 | MemoryContext old; 199 | char* table_name; 200 | 201 | data = ctx->output_plugin_private; 202 | 203 | data->xact_wrote_changes = true; 204 | 205 | class_form = RelationGetForm(relation); 206 | tupdesc = RelationGetDescr(relation); 207 | 208 | table_name = NameStr(class_form->relname); 209 | 210 | if (strncmp(table_name, "pg_temp_", 8) == 0) { 211 | /* ignore */ 212 | return; 213 | } 214 | 215 | old = MemoryContextSwitchTo(data->context); 216 | 217 | OutputPluginPrepareWrite(ctx, true); 218 | 219 | appendStringInfoString(ctx->out, "{\"type\":\"table\""); 220 | appendStringInfo( 221 | ctx->out, 222 | ",\"schema\":\"%s\"", 223 | get_namespace_name( 224 | get_rel_namespace( 225 | RelationGetRelid(relation) 226 | ) 227 | ) 228 | ); 229 | appendStringInfo(ctx->out, ",\"name\":\"%s\"", table_name); 230 | appendStringInfo( 231 | ctx->out, 232 | ",\"change\":\"%s\"", 233 | change->action == REORDER_BUFFER_CHANGE_INSERT 234 | ? "INSERT" 235 | : change->action == REORDER_BUFFER_CHANGE_UPDATE 236 | ? "UPDATE" 237 | : change->action == REORDER_BUFFER_CHANGE_DELETE 238 | ? "DELETE" 239 | : "FIXME" 240 | ); 241 | 242 | if (change->action == REORDER_BUFFER_CHANGE_UPDATE || change->action == REORDER_BUFFER_CHANGE_DELETE) { 243 | appendStringInfoString(ctx->out, ",\"key\":{"); 244 | RelationGetIndexList(relation); 245 | if (OidIsValid(relation->rd_replidindex)) { 246 | int i; 247 | Relation index = index_open(relation->rd_replidindex, ShareLock); 248 | tuple = 249 | change->data.tp.oldtuple 250 | ? &change->data.tp.oldtuple->tuple 251 | : &change->data.tp.newtuple->tuple; 252 | for (i = 0; i < index->rd_index->indnatts; i++) { 253 | int j = index->rd_index->indkey.values[i]; 254 | Form_pg_attribute attr = tupdesc->attrs[j - 1]; 255 | if (i > 0) appendStringInfoChar(ctx->out, ','); 256 | appendStringInfo(ctx->out, "\"%s\":", NameStr(attr->attname)); 257 | print_value(ctx->out, tupdesc, tuple, j - 1); 258 | } 259 | index_close(index, NoLock); 260 | } else { 261 | appendStringInfoString(ctx->out, "\"***FIXME***\":true"); 262 | } 263 | appendStringInfoChar(ctx->out, '}'); 264 | } 265 | 266 | if (change->action == REORDER_BUFFER_CHANGE_UPDATE || change->action == REORDER_BUFFER_CHANGE_INSERT) { 267 | appendStringInfoString(ctx->out, ",\"data\":{"); 268 | tuple_to_stringinfo(ctx->out, tupdesc, &change->data.tp.newtuple->tuple, false); 269 | appendStringInfoChar(ctx->out, '}'); 270 | } 271 | appendStringInfoChar(ctx->out, '}'); 272 | 273 | MemoryContextSwitchTo(old); 274 | MemoryContextReset(data->context); 275 | 276 | OutputPluginWrite(ctx, true); 277 | } 278 | 279 | /* adapted from test_decoding.c */ 280 | 281 | /*------------------------------------------------------------------------- 282 | * 283 | * test_decoding.c 284 | * example logical decoding output plugin 285 | * 286 | * Copyright (c) 2012-2014, PostgreSQL Global Development Group 287 | * 288 | * IDENTIFICATION 289 | * contrib/test_decoding/test_decoding.c 290 | * 291 | *------------------------------------------------------------------------- 292 | */ 293 | --------------------------------------------------------------------------------