├── LICENSE ├── README.md ├── pom.xml └── src ├── main └── java │ └── com │ └── postgresintl │ └── logicaldecoding │ ├── App.java │ ├── PgOutput.java │ ├── ProtoBuf.java │ ├── Wal2JSON.java │ └── proto │ ├── PgProto.java │ └── pg_logicaldec.proto └── test └── java └── com └── postgresintl └── logicaldecoding └── AppTest.java /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, Dave Cramer 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LogicalDecode 2 | ## Demo for PostgreSQL Logical Decoding with JDBC 3 | 4 | ### What is Logical decoding? 5 | 6 | It's useful to understand what physical replication is in order to understand logical decoding. 7 | 8 | Physical replication extends the functionality of recovery mode. Write Ahead Logs are written to disk before the actual database. These files contain enough information to recreate the transaction in the event of a catastrophic shutdown 9 | 10 | In the event of an emergency shutdown (power fail, OOM kill) when the server comes back online it will attempt to apply the outstanding WAL up to the point of the shutdown. This is referred to as recovery mode. 11 | 12 | Physical replication takes advantage of this infrastructure built into the server. The standby is started in recovery mode and WAL created by the primary are applied to the standby. How that occurs is beyond the scope but you can read about it [here](https://www.postgresql.org/docs/current/continuous-archiving.html) . 13 | 14 | The interesting bit here is that we have a mechanism by which to access the changes in the heap without connecting to the database. 15 | 16 | There are a few caveats though which is where Logical Decoding comes to the rescue. First; WAL's are binary and their format is not guaranteed to be stable (in other words they can change from version to version) and second they contain changes for every database in the server. 17 | 18 | Logical decoding changes all of that by 19 | 20 | Providing changes for only one database per slot 21 | Defining an API which facilitates writing an output plugin to output the changes in any format you define. 22 | 23 | 24 | ### Concepts of Logical Decoding 25 | 26 | Above I mentioned two new concepts slots, and plugins 27 | 28 | A slot is a stream of changes in a database. As previously mentioned logical decoding works on a single database. A slot represents a sequence of changes in that database. There can be more than one slot per database. The slot manages a set of changes sent over a particular stream such as which transaction is currently being streamed and which transaction has been acknowledged. 29 | 30 | A plugin is a library which accepts the changes and decodes the changes into a format of your choosing. Plugins need to be compiled and installed before they can be utilized by a slot. 31 | 32 | Creating a slot with JDBC: 33 | 34 | ```java 35 | public void createLogicalReplicationSlot(String slotName, String outputPlugin ) throws InterruptedException, SQLException, TimeoutException 36 | { 37 | //drop previous slot 38 | dropReplicationSlot(mgmntConnection, slotName); 39 | 40 | try (PreparedStatement preparedStatement = 41 | mgmntConnection.prepareStatement("SELECT * FROM pg_create_logical_replication_slot(?, ?)") ) 42 | { 43 | preparedStatement.setString(1, slotName); 44 | preparedStatement.setString(2, outputPlugin); 45 | try (ResultSet rs = preparedStatement.executeQuery()) 46 | { 47 | while (rs.next()) 48 | { 49 | System.out.println("Slot Name: " + rs.getString(1)); 50 | System.out.println("Xlog Position: " + rs.getString(2)); 51 | } 52 | } 53 | 54 | } 55 | } 56 | ``` 57 | 58 | The function `pg_create_logical_replication_slot(, )` 59 | returns slot_name and xlog_position 60 | 61 | From the manual: 62 | 63 | * Creates a new logical (decoding) replication slot named slot_name using the output plugin plugin. 64 | A call to this function has the same effect as the replication protocol command 65 | CREATE_REPLICATION_SLOT LOGICAL 66 | 67 | * *Note* in the above example we are dropping the replication slot if it exists, in production you 68 | would not do this. The code below first terminates any existing replication connection and then drops 69 | the slot 70 | 71 | 72 | ```java 73 | public void dropReplicationSlot(Connection connection, String slotName) 74 | throws SQLException, InterruptedException, TimeoutException 75 | { 76 | try (PreparedStatement preparedStatement = connection.prepareStatement( 77 | "select pg_terminate_backend(active_pid) from pg_replication_slots " 78 | + "where active = true and slot_name = ?")) 79 | { 80 | preparedStatement.setString(1, slotName); 81 | preparedStatement.execute(); 82 | } 83 | 84 | waitStopReplicationSlot(connection, slotName); 85 | 86 | try (PreparedStatement preparedStatement = connection.prepareStatement("select pg_drop_replication_slot(slot_name) " 87 | + "from pg_replication_slots where slot_name = ?")) { 88 | preparedStatement.setString(1, slotName); 89 | preparedStatement.execute(); 90 | } 91 | } 92 | ``` 93 | 94 | ### What have we done so far? 95 | 96 | * We have created a replication slot 97 | * We know the current xlog location. In order to read the xlog location the driver provides a helper 98 | class LogicalSequenceNumber. It is public so you can use it easily. 99 | 100 | ```java 101 | package org.postgresql.replication; 102 | /** 103 | * LSN (Log Sequence Number) data which is a pointer to a location in the XLOG 104 | */ 105 | public final class LogSequenceNumber { 106 | /** 107 | * Zero is used indicate an invalid pointer. Bootstrap skips the first possible WAL segment, 108 | * initializing the first WAL page at XLOG_SEG_SIZE, so no XLOG record can begin at zero. 109 | */ 110 | public static final LogSequenceNumber INVALID_LSN = LogSequenceNumber.valueOf(0); 111 | 112 | private final long value; 113 | 114 | private LogSequenceNumber(long value) { 115 | this.value = value; 116 | } 117 | 118 | /** 119 | * @param value numeric represent position in the write-ahead log stream 120 | * @return not null LSN instance 121 | */ 122 | public static LogSequenceNumber valueOf(long value) { 123 | return new LogSequenceNumber(value); 124 | } 125 | 126 | /** 127 | * Create LSN instance by string represent LSN 128 | * 129 | * @param strValue not null string as two hexadecimal numbers of up to 8 digits each, separated by 130 | * a slash. For example {@code 16/3002D50}, {@code 0/15D68C50} 131 | * @return not null LSN instance where if specified string represent have not valid form {@link 132 | * LogSequenceNumber#INVALID_LSN} 133 | */ 134 | public static LogSequenceNumber valueOf(String strValue) { 135 | int slashIndex = strValue.lastIndexOf('/'); 136 | 137 | if (slashIndex <= 0) { 138 | return INVALID_LSN; 139 | } 140 | 141 | String logicalXLogStr = strValue.substring(0, slashIndex); 142 | int logicalXlog = (int) Long.parseLong(logicalXLogStr, 16); 143 | String segmentStr = strValue.substring(slashIndex + 1, strValue.length()); 144 | int segment = (int) Long.parseLong(segmentStr, 16); 145 | 146 | ByteBuffer buf = ByteBuffer.allocate(8); 147 | buf.putInt(logicalXlog); 148 | buf.putInt(segment); 149 | buf.position(0); 150 | long value = buf.getLong(); 151 | 152 | return LogSequenceNumber.valueOf(value); 153 | } 154 | 155 | /** 156 | * @return Long represent position in the write-ahead log stream 157 | */ 158 | public long asLong() { 159 | return value; 160 | } 161 | 162 | /** 163 | * @return String represent position in the write-ahead log stream as two hexadecimal numbers of 164 | * up to 8 digits each, separated by a slash. For example {@code 16/3002D50}, {@code 0/15D68C50} 165 | */ 166 | public String asString() { 167 | ByteBuffer buf = ByteBuffer.allocate(8); 168 | buf.putLong(value); 169 | buf.position(0); 170 | 171 | int logicalXlog = buf.getInt(); 172 | int segment = buf.getInt(); 173 | return String.format("%X/%X", logicalXlog, segment); 174 | } 175 | 176 | @Override 177 | public boolean equals(Object o) { 178 | if (this == o) { 179 | return true; 180 | } 181 | if (o == null || getClass() != o.getClass()) { 182 | return false; 183 | } 184 | 185 | LogSequenceNumber that = (LogSequenceNumber) o; 186 | 187 | return value == that.value; 188 | 189 | } 190 | 191 | @Override 192 | public int hashCode() { 193 | return (int) (value ^ (value >>> 32)); 194 | } 195 | 196 | @Override 197 | public String toString() { 198 | return "LSN{" + asString() + '}'; 199 | } 200 | } 201 | ``` 202 | 203 | ### An example how to use this class: 204 | 205 | ```java 206 | private LogSequenceNumber getCurrentLSN() throws SQLException 207 | { 208 | try (Statement st = mgmntConnection.createStatement()) 209 | { 210 | try (ResultSet rs = st.executeQuery("select " 211 | + (((BaseConnection) mgmntConnection).haveMinimumServerVersion(ServerVersion.v10) 212 | ? "pg_current_wal_location()" : "pg_current_xlog_location()"))) { 213 | 214 | if (rs.next()) { 215 | String lsn = rs.getString(1); 216 | return LogSequenceNumber.valueOf(lsn); 217 | } else { 218 | return LogSequenceNumber.INVALID_LSN; 219 | } 220 | } 221 | } 222 | } 223 | ``` 224 | 225 | `LogSequenceNumber` knows how to parse the string returned by `pg_current_xlog_location() 226 | 227 | ### Now to actually read changes from a database 228 | 229 | * We need to create a connection capable of replication. 230 | * The replication protocol only understands the Simple Query protocol 231 | 232 | ```java 233 | private void openReplicationConnection() throws Exception { 234 | Properties properties = new Properties(); 235 | properties.setProperty("user","rep"); 236 | properties.setProperty("password","test"); 237 | PGProperty.ASSUME_MIN_SERVER_VERSION.set(properties, "9.4"); 238 | PGProperty.REPLICATION.set(properties, "database"); 239 | PGProperty.PREFER_QUERY_MODE.set(properties, "simple"); 240 | replicationConnection = DriverManager.getConnection("jdbc:postgresql://localhost:15432/test",properties); 241 | } 242 | ``` 243 | 244 | * note we ensure that we have a backend version > 9.4 as logical decoding was not supported before 245 | that 246 | * the user rep is a specific user which has the replication role. You must create this using 247 | ` create user rep role with replication` additionally you must add this user to pg_hba.conf 248 | `host replication rep 0.0.0.0/0 md5` 249 | * as mentioned above we must set PREFER_QUERY_MODE TO "simple" 250 | * REPLICATION must be set to "database": this instructs the driver to connect to the database specified 251 | in the connection URL. 252 | 253 | 254 | 255 | ### We have a replication connection now what? 256 | 257 | The following code can be used to read changes from the database 258 | 259 | ```java 260 | public void receiveChangesOccursBeforStartReplication() throws Exception { 261 | PGConnection pgConnection = (PGConnection) replicationConnection; 262 | 263 | LogSequenceNumber lsn = getCurrentLSN(); 264 | 265 | Statement st = dmlConnection.createStatement(); 266 | st.execute("insert into test_logical_table(name) values('previous value')"); 267 | st.close(); 268 | 269 | PGReplicationStream stream = 270 | pgConnection 271 | .getReplicationAPI() 272 | .replicationStream() 273 | .logical() 274 | .withSlotName(SLOT_NAME) 275 | .withStartPosition(lsn) 276 | .withSlotOption("include-xids", true) 277 | // .withSlotOption("pretty-print",true) 278 | .withSlotOption("skip-empty-xacts", true) 279 | .withStatusInterval(20, TimeUnit.SECONDS) 280 | .start(); 281 | ByteBuffer buffer; 282 | while(true) 283 | { 284 | buffer = stream.readPending(); 285 | if (buffer == null) { 286 | TimeUnit.MILLISECONDS.sleep(10L); 287 | continue; 288 | } 289 | 290 | System.out.println( toString(buffer)); 291 | //feedback 292 | stream.setAppliedLSN(stream.getLastReceiveLSN()); 293 | stream.setFlushedLSN(stream.getLastReceiveLSN()); 294 | } 295 | 296 | } 297 | ``` 298 | 299 | So lets break this down. 300 | 301 | 1) First get the current lsn *before* we modify the database 302 | 2) Modify some data 303 | 3) Get a replication stream. This uses a fluent style and has a number of steps 304 | 1) ask the connection for the replicationAPI, and a stream 305 | 2) ask for logical 306 | 3) specify the slot that we want to read changes from 307 | 4) specify where we want to start from 308 | 5) Options for the decoder which are specific to the output plugin 309 | 6) set the status update timeout interval to 20 seconds, this is how often we will 310 | send the status update message back to the server 311 | 4) read data from the stream. This uses a non-blocking call ```readPending``` 312 | 5) do something with the data. In this case we simply display it. 313 | 6) Now tell the server that we have read the changes so that it is free to release the WAL buffers 314 | 7) This will automatically be sent to the server by the driver when we send the status update message 315 | 316 | 317 | ### Notes 318 | * withStartPosition can be left out; in which case replication would start from the 319 | current position 320 | * the options for each output plugin are unique to that plugin, if using an existing plugin you will 321 | have to look at the source code for details for instance wal2json has the following 322 | ``` 323 | data->include_xids = false; 324 | data->include_timestamp = false; 325 | data->include_schemas = true; 326 | data->include_types = true; 327 | data->pretty_print = false; 328 | data->write_in_chunks = false; 329 | data->include_lsn = false; 330 | 331 | ``` 332 | # Requirements 333 | 334 | Note that while logical decoding was available in 9.4 this code really only works with 9.5 and above 335 | due to the inability to easily determine which pid has the active slot. active_pid was added in 9.5 336 | For this reason 337 | 338 | As the postgres user 339 | ```postgresql 340 | 341 | create user rep with replication; 342 | create user test with password 'test'; 343 | create database test owner test; 344 | ``` 345 | As the user test 346 | ```postgresql 347 | create table test_logical_table(id serial, name text); 348 | ``` 349 | 350 | postgresql.conf 351 | max_replication_slots > 0 352 | max_wal_senders > 0 353 | wal_level = logical 354 | 355 | 356 | host replication rep 0.0.0.0/0 md5 357 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.postgresintl.logicaldecoding 5 | LogicalDecode 6 | jar 7 | 1.0-SNAPSHOT 8 | LogicalDecode 9 | http://maven.apache.org 10 | 11 | 1.8 12 | 1.8 13 | 14 | 15 | 16 | junit 17 | junit 18 | 4.13.1 19 | test 20 | 21 | 22 | org.postgresql 23 | postgresql 24 | 42.3.0-SNAPSHOT 25 | compile 26 | 27 | 28 | com.google.protobuf 29 | protobuf-java 30 | 3.16.1 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /src/main/java/com/postgresintl/logicaldecoding/App.java: -------------------------------------------------------------------------------- 1 | package com.postgresintl.logicaldecoding; 2 | 3 | import java.nio.ByteBuffer; 4 | import java.sql.*; 5 | import java.util.Properties; 6 | import java.util.concurrent.TimeUnit; 7 | import java.util.concurrent.TimeoutException; 8 | 9 | import org.postgresql.PGConnection; 10 | import org.postgresql.PGProperty; 11 | import org.postgresql.core.BaseConnection; 12 | import org.postgresql.core.ServerVersion; 13 | import org.postgresql.replication.LogSequenceNumber; 14 | import org.postgresql.replication.PGReplicationStream; 15 | 16 | 17 | 18 | /** 19 | * Logical Decoding App 20 | * 21 | */ 22 | public class App 23 | { 24 | private final static String SLOT_NAME="sub1_slot"; 25 | private final static String HOST="localhost"; 26 | private final static String PORT="5432"; 27 | private final static String DATABASE="test"; 28 | 29 | Connection connection; 30 | Connection replicationConnection; 31 | 32 | 33 | private static String toString(ByteBuffer buffer) { 34 | int offset = buffer.arrayOffset(); 35 | byte[] source = buffer.array(); 36 | int length = source.length - offset; 37 | 38 | return new String(source, offset, length); 39 | } 40 | 41 | private String createUrl(){ 42 | return "jdbc:postgresql://"+HOST+':'+PORT+'/'+DATABASE; 43 | } 44 | public void createConnection() throws SQLException 45 | { 46 | try 47 | { 48 | connection = DriverManager.getConnection(createUrl(),"davec",""); 49 | } 50 | catch (SQLException ex) 51 | { 52 | 53 | } 54 | connection = DriverManager.getConnection(createUrl(),"davec",""); 55 | // connection.createStatement().execute("set wal_debug=true"); 56 | 57 | } 58 | 59 | public void dropPublication(String publication) throws SQLException { 60 | 61 | try (PreparedStatement preparedStatement = 62 | connection.prepareStatement("DROP PUBLICATION " + publication ) ) 63 | { 64 | preparedStatement.execute(); 65 | } 66 | } 67 | public void createPublication(String publication) throws SQLException { 68 | 69 | try (PreparedStatement preparedStatement = 70 | connection.prepareStatement("CREATE PUBLICATION " + publication + " FOR ALL TABLES") ) 71 | { 72 | preparedStatement.execute(); 73 | } 74 | } 75 | 76 | 77 | public void createLogicalReplicationSlot(String slotName, String outputPlugin ) throws InterruptedException, SQLException, TimeoutException 78 | { 79 | //drop previous slot 80 | dropReplicationSlot(connection, slotName); 81 | 82 | try (PreparedStatement preparedStatement = 83 | connection.prepareStatement("SELECT * FROM pg_create_logical_replication_slot(?, ?)") ) 84 | { 85 | preparedStatement.setString(1, slotName); 86 | preparedStatement.setString(2, outputPlugin); 87 | try (ResultSet rs = preparedStatement.executeQuery()) 88 | { 89 | while (rs.next()) 90 | { 91 | System.out.println("Slot Name: " + rs.getString(1)); 92 | System.out.println("Xlog Position: " + rs.getString(2)); 93 | } 94 | } 95 | 96 | } 97 | } 98 | 99 | public void dropReplicationSlot(Connection connection, String slotName) 100 | throws SQLException, InterruptedException, TimeoutException 101 | { 102 | try (PreparedStatement preparedStatement = connection.prepareStatement( 103 | "select pg_terminate_backend(active_pid) from pg_replication_slots " 104 | + "where active = true and slot_name = ?")) 105 | { 106 | preparedStatement.setString(1, slotName); 107 | preparedStatement.execute(); 108 | } 109 | 110 | waitStopReplicationSlot(connection, slotName); 111 | 112 | try (PreparedStatement preparedStatement = connection.prepareStatement("select pg_drop_replication_slot(slot_name) " 113 | + "from pg_replication_slots where slot_name = ?")) { 114 | preparedStatement.setString(1, slotName); 115 | preparedStatement.execute(); 116 | } 117 | } 118 | 119 | public boolean isReplicationSlotActive(Connection connection, String slotName) 120 | throws SQLException 121 | { 122 | 123 | try (PreparedStatement preparedStatement = connection.prepareStatement("select active from pg_replication_slots where slot_name = ?")){ 124 | preparedStatement.setString(1, slotName); 125 | try (ResultSet rs = preparedStatement.executeQuery()) 126 | { 127 | return rs.next() && rs.getBoolean(1); 128 | } 129 | } 130 | } 131 | 132 | private void waitStopReplicationSlot(Connection connection, String slotName) 133 | throws InterruptedException, TimeoutException, SQLException 134 | { 135 | long startWaitTime = System.currentTimeMillis(); 136 | boolean stillActive; 137 | long timeInWait = 0; 138 | 139 | do { 140 | stillActive = isReplicationSlotActive(connection, slotName); 141 | if (stillActive) { 142 | TimeUnit.MILLISECONDS.sleep(100L); 143 | timeInWait = System.currentTimeMillis() - startWaitTime; 144 | } 145 | } while (stillActive && timeInWait <= 30000); 146 | 147 | if (stillActive) { 148 | throw new TimeoutException("Wait stop replication slot " + timeInWait + " timeout occurs"); 149 | } 150 | } 151 | public void receiveChangesOccursBeforStartReplication() throws Exception { 152 | PGConnection pgConnection = (PGConnection) replicationConnection; 153 | 154 | LogSequenceNumber lsn = getCurrentLSN(); 155 | /* 156 | Statement st = connection.createStatement(); 157 | st.execute("insert into t(t) values('previous value')"); 158 | st.execute("insert into t(t) values('previous value')"); 159 | st.execute("insert into t(t) values('previous value')"); 160 | st.close(); 161 | */ 162 | PGReplicationStream stream = 163 | pgConnection 164 | .getReplicationAPI() 165 | .replicationStream() 166 | .logical() 167 | .withSlotName(SLOT_NAME) 168 | .withStartPosition(lsn) 169 | .withSlotOption("proto_version",1) 170 | .withSlotOption("publication_names", "pub1") 171 | // .withSlotOption("include-xids", true) 172 | // .withSlotOption("skip-empty-xacts", true) 173 | // .withSlotOption("proto_version",1) 174 | // .withSlotOption("publication_names", "pub1") 175 | // .withSlotOption("binary","true") 176 | // .withSlotOption("sizeof_datum", "8") 177 | // .withSlotOption("sizeof_int", "4") 178 | // .withSlotOption("sizeof_long", "8") 179 | // .withSlotOption("bigendian", "false") 180 | // .withSlotOption("float4_byval", "true") 181 | // .withSlotOption("float8_byval", "true") 182 | // .withSlotOption("integer_datetimes", "true") 183 | // .withSlotOption("include-xids", true) 184 | // .withSlotOption("skip-empty-xacts", true) 185 | .withStatusInterval(10, TimeUnit.SECONDS) 186 | .start(); 187 | ByteBuffer buffer; 188 | while(true) 189 | { 190 | buffer = stream.readPending(); 191 | if (buffer == null) { 192 | TimeUnit.MILLISECONDS.sleep(10L); 193 | continue; 194 | } 195 | 196 | System.out.println( toString(buffer)); 197 | //feedback 198 | stream.setAppliedLSN(stream.getLastReceiveLSN()); 199 | stream.setFlushedLSN(stream.getLastReceiveLSN()); 200 | } 201 | 202 | } 203 | 204 | private LogSequenceNumber getCurrentLSN() throws SQLException 205 | { 206 | try (Statement st = connection.createStatement()) 207 | { 208 | try (ResultSet rs = st.executeQuery("select " 209 | + (((BaseConnection) connection).haveMinimumServerVersion(ServerVersion.v10) 210 | ? "pg_current_wal_lsn()" : "pg_current_xlog_location()"))) { 211 | 212 | if (rs.next()) { 213 | String lsn = rs.getString(1); 214 | return LogSequenceNumber.valueOf(lsn); 215 | } else { 216 | return LogSequenceNumber.INVALID_LSN; 217 | } 218 | } 219 | } 220 | } 221 | 222 | private void openReplicationConnection() throws Exception { 223 | Properties properties = new Properties(); 224 | properties.setProperty("user","davec"); 225 | properties.setProperty("password",""); 226 | PGProperty.ASSUME_MIN_SERVER_VERSION.set(properties, "9.4"); 227 | PGProperty.REPLICATION.set(properties, "database"); 228 | PGProperty.PREFER_QUERY_MODE.set(properties, "simple"); 229 | replicationConnection = DriverManager.getConnection(createUrl(),properties); 230 | } 231 | private boolean isServerCompatible() { 232 | return ((BaseConnection)connection).haveMinimumServerVersion(ServerVersion.v9_5); 233 | } 234 | 235 | /* 236 | static boolean active = true; 237 | 238 | public class ReadChanges implements Runnable { 239 | 240 | ByteBuffer buffer; 241 | 242 | 243 | public void run() { 244 | while (true) { 245 | try { 246 | buffer = stream.readPending(); 247 | if (buffer == null) { 248 | TimeUnit.MILLISECONDS.sleep(10L); 249 | continue; 250 | } 251 | 252 | System.out.println(new PgOutput(buffer).toString()); 253 | //feedback 254 | stream.setAppliedLSN(stream.getLastReceiveLSN()); 255 | stream.setFlushedLSN(stream.getLastReceiveLSN()); 256 | } catch ( Exception ex ) { 257 | ex.printStackTrace(); 258 | } 259 | } 260 | } 261 | } 262 | 263 | */ 264 | public static void main( String[] args ) 265 | { 266 | String pluginName = "pgoutput"; 267 | 268 | App app = new App(); 269 | try { 270 | app.createConnection(); 271 | 272 | if (!app.isServerCompatible()) { 273 | System.err.println("must have server version greater than 9.4"); 274 | System.exit(-1); 275 | } 276 | }catch (SQLException ex){ 277 | ex.printStackTrace(); 278 | return; 279 | } 280 | try { 281 | app.createLogicalReplicationSlot(SLOT_NAME, pluginName ); 282 | // app.dropPublication("pub1"); 283 | // app.createPublication("pub1"); 284 | app.openReplicationConnection(); 285 | app.receiveChangesOccursBeforStartReplication(); 286 | 287 | /* 288 | new Thread(new ReadChanges()).start(); 289 | while (true) { 290 | Thread.sleep(5000); 291 | } 292 | */ 293 | } catch (InterruptedException e) { 294 | e.printStackTrace(); 295 | } catch (SQLException e) { 296 | e.printStackTrace(); 297 | } catch (TimeoutException e) { 298 | e.printStackTrace(); 299 | } catch (Exception e) { 300 | e.printStackTrace(); 301 | } 302 | } 303 | } 304 | -------------------------------------------------------------------------------- /src/main/java/com/postgresintl/logicaldecoding/PgOutput.java: -------------------------------------------------------------------------------- 1 | package com.postgresintl.logicaldecoding; 2 | 3 | import com.postgresintl.logicaldecoding.model.Attribute; 4 | import com.postgresintl.logicaldecoding.model.Relation; 5 | import org.postgresql.replication.LogSequenceNumber; 6 | 7 | import java.nio.ByteBuffer; 8 | import java.sql.Timestamp; 9 | 10 | public class PgOutput { 11 | ByteBuffer buffer; 12 | long nanoTime = 0L; 13 | public PgOutput(ByteBuffer b){ 14 | buffer = b; 15 | } 16 | public String toString() { 17 | 18 | byte cmd = buffer.get(); 19 | switch (cmd) { 20 | case 'R': 21 | Relation relation = new Relation(); 22 | relation.setOid(buffer.getInt()); 23 | 24 | relation.setSchema(getString(buffer)); 25 | relation.setName(getString(buffer)); 26 | byte replicaIdent = buffer.get(); 27 | short numAttrs = buffer.getShort(); 28 | for (int i = 0; i < numAttrs;i++){ 29 | byte replicaIdentityFull = buffer.get(); 30 | String attrName=getString(buffer); 31 | int attrId = buffer.getInt(); 32 | int attrMode = buffer.getInt(); 33 | relation.addAttribute(new Attribute(attrId, attrName, attrMode, replicaIdentityFull)); 34 | } 35 | 36 | return "SCHEMA: " + relation.toString(); 37 | 38 | case 'B': 39 | nanoTime = System.nanoTime(); 40 | System.out.println("Begin Issued: " + nanoTime); 41 | LogSequenceNumber finalLSN = LogSequenceNumber.valueOf(buffer.getLong()); 42 | Timestamp commitTime = new Timestamp(buffer.getLong()); 43 | int transactionId = buffer.getInt(); 44 | return "BEGIN final LSN: " + finalLSN.toString() + " Commit Time: " + commitTime + " XID: " + transactionId; 45 | 46 | case 'C': 47 | // COMMIT 48 | System.out.println("Commit Issued: " + (System.nanoTime() - nanoTime)); 49 | byte unusedFlag = buffer.get(); 50 | LogSequenceNumber commitLSN = LogSequenceNumber.valueOf( buffer.getLong() ); 51 | LogSequenceNumber endLSN = LogSequenceNumber.valueOf( buffer.getLong() ); 52 | commitTime = new Timestamp(buffer.getLong()); 53 | return "COMMIT commit LSN:" + commitLSN.toString() + " end LSN:" + endLSN.toString() + " commitTime: " + commitTime; 54 | 55 | case 'U': // UPDATE 56 | case 'D': // DELETE 57 | StringBuffer sb = new StringBuffer(cmd=='U'?"UPDATE: ":"DELETE: "); 58 | int oid = buffer.getInt(); 59 | /* 60 | this can be O or K if Delete or possibly N if UPDATE 61 | K means key 62 | O means old data 63 | N means new data 64 | */ 65 | char keyOrTuple = (char)buffer.get(); 66 | getTuple(buffer, sb); 67 | return sb.toString(); 68 | 69 | case 'I': 70 | sb = new StringBuffer("INSERT: "); 71 | // oid of relation that is being inserted 72 | oid = buffer.getInt(); 73 | // should be an N 74 | char isNew = (char)buffer.get(); 75 | getTuple(buffer, sb); 76 | return sb.toString(); 77 | } 78 | return ""; 79 | } 80 | 81 | private void getTuple(ByteBuffer buffer, StringBuffer sb) { 82 | short numAttrs; 83 | numAttrs = buffer.getShort(); 84 | for (int i = 0; i < numAttrs; i++) { 85 | byte c = buffer.get(); 86 | switch (c) { 87 | case 'n': // null 88 | sb.append("NULL, "); 89 | break; 90 | case 'u': // unchanged toast column 91 | break; 92 | case 't': // textual data 93 | int strLen = buffer.getInt(); 94 | byte[] bytes = new byte[strLen]; 95 | buffer.get(bytes, 0, strLen); 96 | String value = new String(bytes); 97 | sb.append(value).append(", "); 98 | break; 99 | default: 100 | sb.append("command: ").append((char) c); 101 | 102 | } 103 | } 104 | } 105 | private String getString(ByteBuffer buffer){ 106 | StringBuffer sb = new StringBuffer(); 107 | while ( true ){ 108 | byte c = buffer.get(); 109 | if ( c == 0 ) { 110 | break; 111 | } 112 | sb.append((char)c); 113 | } 114 | return sb.toString(); 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/main/java/com/postgresintl/logicaldecoding/ProtoBuf.java: -------------------------------------------------------------------------------- 1 | package com.postgresintl.logicaldecoding; 2 | 3 | import com.google.protobuf.Message; 4 | import com.postgresintl.logicaldecoding.proto.PgProto; 5 | 6 | import java.nio.ByteBuffer; 7 | import java.util.Arrays; 8 | 9 | public class ProtoBuf { 10 | ByteBuffer byteBuffer; 11 | 12 | public ProtoBuf(ByteBuffer b){ 13 | byteBuffer = b; 14 | } 15 | public String toString() { 16 | int offset = byteBuffer.arrayOffset(); 17 | byte[] source = byteBuffer.array(); 18 | int length = source.length - offset; 19 | final byte[] content = Arrays.copyOfRange(source, offset, length); 20 | try { 21 | final PgProto.RowMessage message = PgProto.RowMessage.parseFrom(content); 22 | return message.toString(); 23 | } catch (Exception ex) { 24 | ex.printStackTrace(); 25 | return null; 26 | } 27 | 28 | 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/postgresintl/logicaldecoding/Wal2JSON.java: -------------------------------------------------------------------------------- 1 | package com.postgresintl.logicaldecoding; 2 | 3 | import java.nio.ByteBuffer; 4 | 5 | public class Wal2JSON { 6 | ByteBuffer byteBuffer; 7 | public Wal2JSON(ByteBuffer b){ 8 | byteBuffer = b; 9 | } 10 | 11 | @Override 12 | public String toString() { 13 | int offset = byteBuffer.arrayOffset(); 14 | byte[] source = byteBuffer.array(); 15 | int length = source.length - offset; 16 | 17 | return new String(source, offset, length); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/com/postgresintl/logicaldecoding/proto/pg_logicaldec.proto: -------------------------------------------------------------------------------- 1 | package decoderbufs; 2 | 3 | option java_package="com.postgresintl.logicaldecoding.proto"; 4 | option java_outer_classname = "PgProto"; 5 | option optimize_for = SPEED; 6 | 7 | enum Op { 8 | INSERT = 0; 9 | UPDATE = 1; 10 | DELETE = 2; 11 | } 12 | 13 | message Point { 14 | required double x = 1; 15 | required double y = 2; 16 | } 17 | 18 | message DatumMessage { 19 | optional string column_name = 1; 20 | optional int64 column_type = 2; 21 | oneof datum { 22 | int32 datum_int32 = 3; 23 | int64 datum_int64 = 4; 24 | float datum_float = 5; 25 | double datum_double = 6; 26 | bool datum_bool = 7; 27 | string datum_string = 8; 28 | bytes datum_bytes = 9; 29 | Point datum_point = 10; 30 | bool datum_missing = 11; 31 | } 32 | } 33 | 34 | message TypeInfo { 35 | required string modifier = 1; 36 | required bool value_optional = 2; 37 | } 38 | 39 | message RowMessage { 40 | optional uint32 transaction_id = 1; 41 | optional uint64 commit_time = 2; 42 | optional string table = 3; 43 | optional Op op = 4; 44 | repeated DatumMessage new_tuple = 5; 45 | repeated DatumMessage old_tuple = 6; 46 | repeated TypeInfo new_typeinfo = 7; 47 | } -------------------------------------------------------------------------------- /src/test/java/com/postgresintl/logicaldecoding/AppTest.java: -------------------------------------------------------------------------------- 1 | package com.postgresintl.logicaldecoding; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | --------------------------------------------------------------------------------