├── testdata ├── 008.waldata ├── 002.waldata ├── 005.waldata ├── 011.waldata ├── 014.waldata ├── 000.waldata ├── 001.waldata ├── 003.waldata ├── 004.waldata ├── 006.waldata ├── 007.waldata ├── 009.waldata ├── 010.waldata ├── 012.waldata ├── 013.waldata ├── 015.waldata └── 016.waldata ├── go.mod ├── go.sum ├── LICENSE ├── examples └── replicate.go ├── README.md ├── parse_test.go ├── values.go ├── sub.go └── parse.go /testdata/008.waldata: -------------------------------------------------------------------------------- 1 | D@ Kt30n -------------------------------------------------------------------------------- /testdata/002.waldata: -------------------------------------------------------------------------------- 1 | I@ Nt40tforty -------------------------------------------------------------------------------- /testdata/005.waldata: -------------------------------------------------------------------------------- 1 | U@ Nt40tfifty -------------------------------------------------------------------------------- /testdata/011.waldata: -------------------------------------------------------------------------------- 1 | I@ Nt11televen -------------------------------------------------------------------------------- /testdata/014.waldata: -------------------------------------------------------------------------------- 1 | I@ Nt12ttwelve -------------------------------------------------------------------------------- /testdata/000.waldata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kyleconroy/pgoutput/HEAD/testdata/000.waldata -------------------------------------------------------------------------------- /testdata/001.waldata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kyleconroy/pgoutput/HEAD/testdata/001.waldata -------------------------------------------------------------------------------- /testdata/003.waldata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kyleconroy/pgoutput/HEAD/testdata/003.waldata -------------------------------------------------------------------------------- /testdata/004.waldata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kyleconroy/pgoutput/HEAD/testdata/004.waldata -------------------------------------------------------------------------------- /testdata/006.waldata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kyleconroy/pgoutput/HEAD/testdata/006.waldata -------------------------------------------------------------------------------- /testdata/007.waldata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kyleconroy/pgoutput/HEAD/testdata/007.waldata -------------------------------------------------------------------------------- /testdata/009.waldata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kyleconroy/pgoutput/HEAD/testdata/009.waldata -------------------------------------------------------------------------------- /testdata/010.waldata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kyleconroy/pgoutput/HEAD/testdata/010.waldata -------------------------------------------------------------------------------- /testdata/012.waldata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kyleconroy/pgoutput/HEAD/testdata/012.waldata -------------------------------------------------------------------------------- /testdata/013.waldata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kyleconroy/pgoutput/HEAD/testdata/013.waldata -------------------------------------------------------------------------------- /testdata/015.waldata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kyleconroy/pgoutput/HEAD/testdata/015.waldata -------------------------------------------------------------------------------- /testdata/016.waldata: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kyleconroy/pgoutput/HEAD/testdata/016.waldata -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/kyleconroy/pgoutput 2 | 3 | require ( 4 | github.com/google/go-cmp v0.2.0 5 | github.com/jackc/pgx v3.2.0+incompatible 6 | github.com/pkg/errors v0.8.0 // indirect 7 | ) 8 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= 2 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 3 | github.com/jackc/pgx v3.2.0+incompatible h1:0Vihzu20St42/UDsvZGdNE6jak7oi/UOeMzwMPHkgFY= 4 | github.com/jackc/pgx v3.2.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= 5 | github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= 6 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Kyle Conroy 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /examples/replicate.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | 8 | "github.com/jackc/pgx" 9 | "github.com/kyleconroy/pgoutput" 10 | ) 11 | 12 | func main() { 13 | ctx := context.Background() 14 | config := pgx.ConnConfig{Database: "opsdash", User: "replicant"} 15 | conn, err := pgx.ReplicationConnect(config) 16 | if err != nil { 17 | log.Fatal(err) 18 | } 19 | 20 | set := pgoutput.NewRelationSet(nil) 21 | 22 | dump := func(relation uint32, row []pgoutput.Tuple) error { 23 | values, err := set.Values(relation, row) 24 | if err != nil { 25 | return fmt.Errorf("error parsing values: %s", err) 26 | } 27 | for name, value := range values { 28 | val := value.Get() 29 | log.Printf("%s (%T): %#v", name, val, val) 30 | } 31 | return nil 32 | } 33 | 34 | handler := func(m pgoutput.Message, walPos uint64) error { 35 | switch v := m.(type) { 36 | case pgoutput.Relation: 37 | log.Printf("RELATION") 38 | set.Add(v) 39 | case pgoutput.Insert: 40 | log.Printf("INSERT") 41 | return dump(v.RelationID, v.Row) 42 | case pgoutput.Update: 43 | log.Printf("UPDATE") 44 | return dump(v.RelationID, v.Row) 45 | case pgoutput.Delete: 46 | log.Printf("DELETE") 47 | return dump(v.RelationID, v.Row) 48 | } 49 | return nil 50 | } 51 | 52 | sub := pgoutput.NewSubscription(conn, "sub1", "pub1", 0, false) 53 | if err := sub.Start(ctx, 0, handler); err != nil { 54 | log.Fatal(err) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pgoutput 2 | 3 | ```go 4 | package main 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | "log" 10 | "time" 11 | 12 | "github.com/jackc/pgx" 13 | "github.com/kyleconroy/pgoutput" 14 | ) 15 | 16 | func main() { 17 | ctx := context.Background() 18 | config := pgx.ConnConfig{Database: "opsdash", User: "replicant"} 19 | conn, err := pgx.ReplicationConnect(config) 20 | if err != nil { 21 | log.Fatal(err) 22 | } 23 | 24 | // Create a slot if it doesn't already exist 25 | // if err := conn.CreateReplicationSlot("sub2", "pgoutput"); err != nil { 26 | // log.Fatalf("Failed to create replication slot: %v", err) 27 | // } 28 | 29 | set := pgoutput.NewRelationSet() 30 | 31 | dump := func(relation uint32, row []pgoutput.Tuple) error { 32 | values, err := set.Values(relation, row) 33 | if err != nil { 34 | return fmt.Errorf("error parsing values: %s", err) 35 | } 36 | for name, value := range values { 37 | val := value.Get() 38 | log.Printf("%s (%T): %#v", name, val, val) 39 | } 40 | return nil 41 | } 42 | 43 | handler := func(m pgoutput.Message) error { 44 | switch v := m.(type) { 45 | case pgoutput.Relation: 46 | log.Printf("RELATION") 47 | set.Add(v) 48 | case pgoutput.Insert: 49 | log.Printf("INSERT") 50 | return dump(v.RelationID, v.Row) 51 | case pgoutput.Update: 52 | log.Printf("UPDATE") 53 | return dump(v.RelationID, v.Row) 54 | case pgoutput.Delete: 55 | log.Printf("DELETE") 56 | return dump(v.RelationID, v.Row) 57 | } 58 | return nil 59 | } 60 | 61 | replication := pgoutput.LogicalReplication{ 62 | Subscription: "sub2", 63 | Publication: "pub2", 64 | WaitTimeout: time.Second * 10, 65 | StatusTimeout: time.Second * 10, 66 | Handler: handler, 67 | } 68 | 69 | if err := replication.Start(ctx, conn); err != nil { 70 | log.Fatal(err) 71 | } 72 | } 73 | ``` 74 | -------------------------------------------------------------------------------- /parse_test.go: -------------------------------------------------------------------------------- 1 | package pgoutput 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "path/filepath" 9 | "testing" 10 | 11 | "github.com/google/go-cmp/cmp" 12 | "github.com/jackc/pgx" 13 | ) 14 | 15 | func GenerateLogicalReplicationFiles(t *testing.T) { 16 | config := pgx.ConnConfig{ 17 | Database: "opsdash", 18 | User: "replicant", 19 | } 20 | 21 | conn, err := pgx.ReplicationConnect(config) 22 | if err != nil { 23 | log.Fatal(err) 24 | } 25 | 26 | err = conn.StartReplication("sub1", 0, -1, `("proto_version" '1', "publication_names" 'pub1')`) 27 | if err != nil { 28 | log.Fatalf("Failed to start replication: %v", err) 29 | } 30 | 31 | ctx := context.Background() 32 | count := 0 33 | 34 | for { 35 | var message *pgx.ReplicationMessage 36 | 37 | message, err = conn.WaitForReplicationMessage(ctx) 38 | if err != nil { 39 | log.Fatalf("Replication failed: %v %s", message, err) 40 | } 41 | 42 | if message.WalMessage != nil { 43 | ioutil.WriteFile(fmt.Sprintf("%03d.waldata", count), message.WalMessage.WalData, 0644) 44 | count += 1 45 | } 46 | if message.ServerHeartbeat != nil { 47 | log.Printf("Got heartbeat: %s", message.ServerHeartbeat) 48 | } 49 | } 50 | } 51 | 52 | func TestParseWalData(t *testing.T) { 53 | files, _ := filepath.Glob("testdata/*") 54 | set := NewRelationSet(nil) 55 | 56 | expected := map[int]struct { 57 | ID int32 58 | Val string 59 | }{ 60 | 2: {40, "forty"}, 61 | 11: {11, "eleven"}, 62 | 14: {12, "twelve"}, 63 | } 64 | 65 | for i, file := range files { 66 | waldata, _ := ioutil.ReadFile(file) 67 | m, err := Parse(waldata) 68 | if err != nil { 69 | t.Errorf("error parsing %s: %s", file, err) 70 | continue 71 | } 72 | 73 | switch v := m.(type) { 74 | case Relation: 75 | set.Add(v) 76 | case Insert: 77 | t.Run(fmt.Sprintf("waldata/%d", i), func(t *testing.T) { 78 | values, err := set.Values(v.RelationID, v.Row) 79 | if err != nil { 80 | t.Error(err) 81 | } 82 | 83 | exp := expected[i] 84 | if diff := cmp.Diff(exp.ID, values["id"].Get()); diff != "" { 85 | t.Errorf("id: %s", diff) 86 | } 87 | if diff := cmp.Diff(exp.Val, values["val"].Get()); diff != "" { 88 | t.Errorf("val: %s", diff) 89 | } 90 | }) 91 | case Type: 92 | if v.ID != 35756 { 93 | t.Errorf("Type OID: %d", v.ID) 94 | } 95 | if v.Namespace != "public" { 96 | t.Errorf("Type namespace: %s", v.Namespace) 97 | } 98 | if v.Name != "ticket_state" { 99 | t.Errorf("Type name: %s", v.Name) 100 | } 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /values.go: -------------------------------------------------------------------------------- 1 | package pgoutput 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/jackc/pgx/pgtype" 7 | ) 8 | 9 | type RelationSet struct { 10 | // Mutex probably will be redundant as receiving 11 | // a replication stream is currently strictly single-threaded 12 | relations map[uint32]Relation 13 | connInfo *pgtype.ConnInfo 14 | } 15 | 16 | // NewRelationSet creates a new relation set. 17 | // Optionally ConnInfo can be provided, however currently we need some changes to pgx to get it out 18 | // from ReplicationConn. 19 | func NewRelationSet(ci *pgtype.ConnInfo) *RelationSet { 20 | return &RelationSet{map[uint32]Relation{}, ci} 21 | } 22 | 23 | func (rs *RelationSet) Add(r Relation) { 24 | rs.relations[r.ID] = r 25 | } 26 | 27 | func (rs *RelationSet) Get(ID uint32) (r Relation, ok bool) { 28 | r, ok = rs.relations[ID] 29 | return 30 | } 31 | 32 | func (rs *RelationSet) Values(id uint32, row []Tuple) (map[string]pgtype.Value, error) { 33 | values := map[string]pgtype.Value{} 34 | rel, ok := rs.Get(id) 35 | if !ok { 36 | return values, fmt.Errorf("no relation for %d", id) 37 | } 38 | 39 | // assert same number of row and columns 40 | for i, tuple := range row { 41 | col := rel.Columns[i] 42 | decoder := col.Decoder() 43 | 44 | if err := decoder.DecodeText(rs.connInfo, tuple.Value); err != nil { 45 | return nil, fmt.Errorf("error decoding tuple %d: %s", i, err) 46 | } 47 | 48 | values[col.Name] = decoder 49 | } 50 | 51 | return values, nil 52 | } 53 | 54 | func (c Column) Decoder() DecoderValue { 55 | switch c.Type { 56 | case pgtype.ACLItemArrayOID: 57 | return &pgtype.ACLItemArray{} 58 | case pgtype.ACLItemOID: 59 | return &pgtype.ACLItem{} 60 | case pgtype.BoolArrayOID: 61 | return &pgtype.BoolArray{} 62 | case pgtype.BoolOID: 63 | return &pgtype.Bool{} 64 | case pgtype.ByteaArrayOID: 65 | return &pgtype.BoolArray{} 66 | case pgtype.ByteaOID: 67 | return &pgtype.Bytea{} 68 | case pgtype.CIDOID: 69 | return &pgtype.CID{} 70 | case pgtype.CIDRArrayOID: 71 | return &pgtype.CIDRArray{} 72 | case pgtype.CIDROID: 73 | return &pgtype.CIDR{} 74 | case pgtype.CharOID: 75 | // Not all possible values of QChar are representable in the text format 76 | return &pgtype.Unknown{} 77 | case pgtype.DateArrayOID: 78 | return &pgtype.DateArray{} 79 | case pgtype.DateOID: 80 | return &pgtype.Date{} 81 | case pgtype.Float4ArrayOID: 82 | return &pgtype.Float4Array{} 83 | case pgtype.Float4OID: 84 | return &pgtype.Float4{} 85 | case pgtype.Float8ArrayOID: 86 | return &pgtype.Float8Array{} 87 | case pgtype.Float8OID: 88 | return &pgtype.Float8{} 89 | case pgtype.InetArrayOID: 90 | return &pgtype.InetArray{} 91 | case pgtype.InetOID: 92 | return &pgtype.Inet{} 93 | case pgtype.Int2ArrayOID: 94 | return &pgtype.Int2Array{} 95 | case pgtype.Int2OID: 96 | return &pgtype.Int2{} 97 | case pgtype.Int4ArrayOID: 98 | return &pgtype.Int4Array{} 99 | case pgtype.Int4OID: 100 | return &pgtype.Int4{} 101 | case pgtype.Int8ArrayOID: 102 | return &pgtype.Int8Array{} 103 | case pgtype.Int8OID: 104 | return &pgtype.Int8{} 105 | case pgtype.JSONBOID: 106 | return &pgtype.JSONB{} 107 | case pgtype.JSONOID: 108 | return &pgtype.JSON{} 109 | case pgtype.NameOID: 110 | return &pgtype.Name{} 111 | case pgtype.OIDOID: 112 | // pgtype.OID does not implement the value interface 113 | return &pgtype.Unknown{} 114 | case pgtype.RecordOID: 115 | // The text format output format for Records does not include type 116 | // information and is therefore impossible to decode 117 | return &pgtype.Unknown{} 118 | case pgtype.TIDOID: 119 | return &pgtype.TID{} 120 | case pgtype.TextArrayOID: 121 | return &pgtype.TextArray{} 122 | case pgtype.TextOID: 123 | return &pgtype.Text{} 124 | case pgtype.TimestampArrayOID: 125 | return &pgtype.TimestampArray{} 126 | case pgtype.TimestampOID: 127 | return &pgtype.Timestamp{} 128 | case pgtype.TimestamptzArrayOID: 129 | return &pgtype.TimestamptzArray{} 130 | case pgtype.TimestamptzOID: 131 | return &pgtype.Timestamptz{} 132 | case pgtype.UUIDOID: 133 | return &pgtype.UUID{} 134 | case pgtype.UnknownOID: 135 | return &pgtype.Unknown{} 136 | case pgtype.VarcharArrayOID: 137 | return &pgtype.VarcharArray{} 138 | case pgtype.VarcharOID: 139 | return &pgtype.Varchar{} 140 | case pgtype.XIDOID: 141 | return &pgtype.XID{} 142 | default: 143 | // panic(fmt.Sprintf("unknown OID type %d", c.Type)) 144 | return &pgtype.Unknown{} 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /sub.go: -------------------------------------------------------------------------------- 1 | package pgoutput 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "sync/atomic" 8 | "time" 9 | 10 | "github.com/jackc/pgx" 11 | ) 12 | 13 | type Subscription struct { 14 | Name string 15 | Publication string 16 | WaitTimeout time.Duration 17 | StatusTimeout time.Duration 18 | 19 | conn *pgx.ReplicationConn 20 | maxWal uint64 21 | walRetain uint64 22 | walFlushed uint64 23 | 24 | failOnHandler bool 25 | 26 | // Mutex is used to prevent reading and writing to a connection at the same time 27 | sync.Mutex 28 | } 29 | 30 | type Handler func(Message, uint64) error 31 | 32 | func NewSubscription(conn *pgx.ReplicationConn, name, publication string, walRetain uint64, failOnHandler bool) *Subscription { 33 | return &Subscription{ 34 | Name: name, 35 | Publication: publication, 36 | WaitTimeout: 1 * time.Second, 37 | StatusTimeout: 10 * time.Second, 38 | 39 | conn: conn, 40 | walRetain: walRetain, 41 | failOnHandler: failOnHandler, 42 | } 43 | } 44 | 45 | func pluginArgs(version, publication string) string { 46 | return fmt.Sprintf(`"proto_version" '%s', "publication_names" '%s'`, version, publication) 47 | } 48 | 49 | // CreateSlot creates a replication slot if it doesn't exist 50 | func (s *Subscription) CreateSlot() (err error) { 51 | // If creating the replication slot fails with code 42710, this means 52 | // the replication slot already exists. 53 | if err = s.conn.CreateReplicationSlot(s.Name, "pgoutput"); err != nil { 54 | pgerr, ok := err.(pgx.PgError) 55 | if !ok || pgerr.Code != "42710" { 56 | return 57 | } 58 | 59 | err = nil 60 | } 61 | 62 | return 63 | } 64 | 65 | func (s *Subscription) sendStatus(walWrite, walFlush uint64) error { 66 | if walFlush > walWrite { 67 | return fmt.Errorf("walWrite should be >= walFlush") 68 | } 69 | 70 | s.Lock() 71 | defer s.Unlock() 72 | 73 | k, err := pgx.NewStandbyStatus(walFlush, walFlush, walWrite) 74 | if err != nil { 75 | return fmt.Errorf("error creating status: %s", err) 76 | } 77 | 78 | if err = s.conn.SendStandbyStatus(k); err != nil { 79 | return err 80 | } 81 | 82 | return nil 83 | } 84 | 85 | // Flush sends the status message to server indicating that we've fully applied all of the events until maxWal. 86 | // This allows PostgreSQL to purge it's WAL logs 87 | func (s *Subscription) Flush() error { 88 | wp := atomic.LoadUint64(&s.maxWal) 89 | err := s.sendStatus(wp, wp) 90 | if err == nil { 91 | atomic.StoreUint64(&s.walFlushed, wp) 92 | } 93 | 94 | return err 95 | } 96 | 97 | // Start replication and block until error or ctx is canceled 98 | func (s *Subscription) Start(ctx context.Context, startLSN uint64, h Handler) (err error) { 99 | err = s.conn.StartReplication(s.Name, startLSN, -1, pluginArgs("1", s.Publication)) 100 | if err != nil { 101 | return fmt.Errorf("failed to start replication: %s", err) 102 | } 103 | 104 | s.maxWal = startLSN 105 | 106 | sendStatus := func() error { 107 | walPos := atomic.LoadUint64(&s.maxWal) 108 | walLastFlushed := atomic.LoadUint64(&s.walFlushed) 109 | 110 | // Confirm only walRetain bytes in past 111 | // If walRetain is zero - will confirm current walPos as flushed 112 | walFlush := walPos - s.walRetain 113 | 114 | if walLastFlushed > walFlush { 115 | // If there was a manual flush - report it's position until we're past it 116 | walFlush = walLastFlushed 117 | } else if walFlush < 0 { 118 | // If we have less than walRetain bytes - just report zero 119 | walFlush = 0 120 | } 121 | 122 | return s.sendStatus(walPos, walFlush) 123 | } 124 | 125 | go func() { 126 | tick := time.NewTicker(s.StatusTimeout) 127 | defer tick.Stop() 128 | 129 | for { 130 | select { 131 | case <-tick.C: 132 | if err = sendStatus(); err != nil { 133 | return 134 | } 135 | 136 | case <-ctx.Done(): 137 | return 138 | } 139 | } 140 | }() 141 | 142 | for { 143 | select { 144 | case <-ctx.Done(): 145 | // Send final status and exit 146 | if err = sendStatus(); err != nil { 147 | return fmt.Errorf("Unable to send final status: %s", err) 148 | } 149 | 150 | return 151 | 152 | default: 153 | var message *pgx.ReplicationMessage 154 | wctx, cancel := context.WithTimeout(ctx, s.WaitTimeout) 155 | s.Lock() 156 | message, err = s.conn.WaitForReplicationMessage(wctx) 157 | s.Unlock() 158 | cancel() 159 | 160 | if err == context.DeadlineExceeded { 161 | continue 162 | } else if err == context.Canceled { 163 | return 164 | } else if err != nil { 165 | return fmt.Errorf("replication failed: %s", err) 166 | } 167 | 168 | if message == nil { 169 | return fmt.Errorf("replication failed: nil message received, should not happen") 170 | } 171 | 172 | if message.WalMessage != nil { 173 | var logmsg Message 174 | walStart := message.WalMessage.WalStart 175 | 176 | // Skip stuff that's in the past 177 | if walStart > 0 && walStart <= startLSN { 178 | continue 179 | } 180 | 181 | if walStart > atomic.LoadUint64(&s.maxWal) { 182 | atomic.StoreUint64(&s.maxWal, walStart) 183 | } 184 | 185 | logmsg, err = Parse(message.WalMessage.WalData) 186 | if err != nil { 187 | return fmt.Errorf("invalid pgoutput message: %s", err) 188 | } 189 | 190 | // Ignore the error from handler for now 191 | if err = h(logmsg, walStart); err != nil && s.failOnHandler { 192 | return 193 | } 194 | } else if message.ServerHeartbeat != nil { 195 | if message.ServerHeartbeat.ReplyRequested == 1 { 196 | if err = sendStatus(); err != nil { 197 | return 198 | } 199 | } 200 | } else { 201 | return fmt.Errorf("No WalMessage/ServerHeartbeat defined in packet, should not happen") 202 | } 203 | } 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /parse.go: -------------------------------------------------------------------------------- 1 | package pgoutput 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/jackc/pgx/pgtype" 10 | ) 11 | 12 | type decoder struct { 13 | order binary.ByteOrder 14 | buf *bytes.Buffer 15 | } 16 | 17 | func (d *decoder) bool() bool { 18 | x := d.buf.Next(1)[0] 19 | return x != 0 20 | 21 | } 22 | 23 | func (d *decoder) uint8() uint8 { 24 | x := d.buf.Next(1)[0] 25 | return x 26 | 27 | } 28 | 29 | func (d *decoder) uint16() uint16 { 30 | x := d.order.Uint16(d.buf.Next(2)) 31 | return x 32 | } 33 | 34 | func (d *decoder) string() string { 35 | s, err := d.buf.ReadBytes(0) 36 | if err != nil { 37 | // TODO: Return an error 38 | panic(err) 39 | } 40 | return string(s[:len(s)-1]) 41 | } 42 | 43 | func (d *decoder) uint32() uint32 { 44 | x := d.order.Uint32(d.buf.Next(4)) 45 | return x 46 | 47 | } 48 | 49 | func (d *decoder) uint64() uint64 { 50 | x := d.order.Uint64(d.buf.Next(8)) 51 | return x 52 | } 53 | 54 | func (d *decoder) int8() int8 { return int8(d.uint8()) } 55 | func (d *decoder) int16() int16 { return int16(d.uint16()) } 56 | func (d *decoder) int32() int32 { return int32(d.uint32()) } 57 | func (d *decoder) int64() int64 { return int64(d.uint64()) } 58 | 59 | func (d *decoder) timestamp() time.Time { 60 | micro := int(d.uint64()) 61 | ts := time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC) 62 | return ts.Add(time.Duration(micro) * time.Microsecond) 63 | } 64 | 65 | func (d *decoder) rowinfo(char byte) bool { 66 | if d.buf.Next(1)[0] == char { 67 | return true 68 | } else { 69 | d.buf.UnreadByte() 70 | return false 71 | } 72 | } 73 | 74 | func (d *decoder) tupledata() []Tuple { 75 | size := int(d.uint16()) 76 | data := make([]Tuple, size) 77 | for i := 0; i < size; i++ { 78 | switch d.buf.Next(1)[0] { 79 | case 'n': 80 | case 'u': 81 | case 't': 82 | vsize := int(d.order.Uint32(d.buf.Next(4))) 83 | data[i] = Tuple{Flag: 't', Value: d.buf.Next(vsize)} 84 | } 85 | } 86 | return data 87 | } 88 | 89 | func (d *decoder) columns() []Column { 90 | size := int(d.uint16()) 91 | data := make([]Column, size) 92 | for i := 0; i < size; i++ { 93 | data[i] = Column{ 94 | Key: d.bool(), 95 | Name: d.string(), 96 | Type: d.uint32(), 97 | Mode: d.uint32(), 98 | } 99 | } 100 | return data 101 | } 102 | 103 | type Begin struct { 104 | // The final LSN of the transaction. 105 | LSN uint64 106 | // Commit timestamp of the transaction. The value is in number of 107 | // microseconds since PostgreSQL epoch (2000-01-01). 108 | Timestamp time.Time 109 | // Xid of the transaction. 110 | XID int32 111 | } 112 | 113 | type Commit struct { 114 | Flags uint8 115 | // The final LSN of the transaction. 116 | LSN uint64 117 | // The final LSN of the transaction. 118 | TransactionLSN uint64 119 | Timestamp time.Time 120 | } 121 | 122 | type Relation struct { 123 | // ID of the relation. 124 | ID uint32 125 | // Namespace (empty string for pg_catalog). 126 | Namespace string 127 | Name string 128 | Replica uint8 129 | Columns []Column 130 | } 131 | 132 | func (r Relation) IsEmpty() bool { 133 | return r.ID == 0 && r.Name == "" && r.Replica == 0 && len(r.Columns) == 0 134 | } 135 | 136 | type Type struct { 137 | // ID of the data type 138 | ID uint32 139 | Namespace string 140 | Name string 141 | } 142 | 143 | type Insert struct { 144 | /// ID of the relation corresponding to the ID in the relation message. 145 | RelationID uint32 146 | // Identifies the following TupleData message as a new tuple. 147 | New bool 148 | Row []Tuple 149 | } 150 | 151 | type Update struct { 152 | /// ID of the relation corresponding to the ID in the relation message. 153 | RelationID uint32 154 | // Identifies the following TupleData message as a new tuple. 155 | Old bool 156 | Key bool 157 | New bool 158 | OldRow []Tuple 159 | Row []Tuple 160 | } 161 | 162 | type Delete struct { 163 | /// ID of the relation corresponding to the ID in the relation message. 164 | RelationID uint32 165 | // Identifies the following TupleData message as a new tuple. 166 | Key bool // TODO 167 | Old bool // TODO 168 | Row []Tuple 169 | } 170 | 171 | type Origin struct { 172 | LSN uint64 173 | Name string 174 | } 175 | 176 | type DecoderValue interface { 177 | pgtype.TextDecoder 178 | pgtype.Value 179 | } 180 | 181 | type Column struct { 182 | Key bool 183 | Name string 184 | Type uint32 185 | Mode uint32 186 | } 187 | type Tuple struct { 188 | Flag int8 189 | Value []byte 190 | } 191 | 192 | type Message interface { 193 | msg() 194 | } 195 | 196 | func (Begin) msg() {} 197 | func (Relation) msg() {} 198 | func (Update) msg() {} 199 | func (Insert) msg() {} 200 | func (Delete) msg() {} 201 | func (Commit) msg() {} 202 | func (Origin) msg() {} 203 | func (Type) msg() {} 204 | 205 | // Parse a logical replication message. 206 | // See https://www.postgresql.org/docs/current/static/protocol-logicalrep-message-formats.html 207 | func Parse(src []byte) (Message, error) { 208 | msgType := src[0] 209 | d := &decoder{order: binary.BigEndian, buf: bytes.NewBuffer(src[1:])} 210 | switch msgType { 211 | case 'B': 212 | b := Begin{} 213 | b.LSN = d.uint64() 214 | b.Timestamp = d.timestamp() 215 | b.XID = d.int32() 216 | return b, nil 217 | case 'C': 218 | c := Commit{} 219 | c.Flags = d.uint8() 220 | c.LSN = d.uint64() 221 | c.TransactionLSN = d.uint64() 222 | c.Timestamp = d.timestamp() 223 | return c, nil 224 | case 'O': 225 | o := Origin{} 226 | o.LSN = d.uint64() 227 | o.Name = d.string() 228 | return o, nil 229 | case 'R': 230 | r := Relation{} 231 | r.ID = d.uint32() 232 | r.Namespace = d.string() 233 | r.Name = d.string() 234 | r.Replica = d.uint8() 235 | r.Columns = d.columns() 236 | return r, nil 237 | case 'Y': 238 | t := Type{} 239 | t.ID = d.uint32() 240 | t.Namespace = d.string() 241 | t.Name = d.string() 242 | return t, nil 243 | case 'I': 244 | i := Insert{} 245 | i.RelationID = d.uint32() 246 | i.New = d.uint8() > 0 247 | i.Row = d.tupledata() 248 | return i, nil 249 | case 'U': 250 | u := Update{} 251 | u.RelationID = d.uint32() 252 | u.Key = d.rowinfo('K') 253 | u.Old = d.rowinfo('O') 254 | if u.Key || u.Old { 255 | u.OldRow = d.tupledata() 256 | } 257 | u.New = d.uint8() > 0 258 | u.Row = d.tupledata() 259 | return u, nil 260 | case 'D': 261 | dl := Delete{} 262 | dl.RelationID = d.uint32() 263 | dl.Key = d.rowinfo('K') 264 | dl.Old = d.rowinfo('O') 265 | dl.Row = d.tupledata() 266 | return dl, nil 267 | default: 268 | return nil, fmt.Errorf("Unknown message type for %s (%d)", []byte{msgType}, msgType) 269 | } 270 | } 271 | --------------------------------------------------------------------------------