├── etc └── sqledge.png ├── .gitignore ├── pkg ├── tables │ ├── buf.go │ ├── testdata │ │ └── init-db.sql │ ├── copy.go │ ├── def.go │ ├── copy_test.go │ └── decode.go ├── sqlgen │ ├── parser_test.go │ ├── driver.go │ ├── parser.go │ └── sqlite.go ├── queryproxy │ └── sqledge.go ├── config │ └── config.go ├── replicate │ ├── run.go │ └── replicate.go └── pgwire │ └── postgres.go ├── cmd └── sqledge │ └── main.go ├── go.mod ├── README.md ├── test └── integration │ └── integration_test.go └── go.sum /etc/sqledge.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zknill/sqledge/HEAD/etc/sqledge.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/*.swp 2 | **/*.swo 3 | **/*.db 4 | replicator/tmp 5 | replicator/tmp/** 6 | tmp/** 7 | -------------------------------------------------------------------------------- /pkg/tables/buf.go: -------------------------------------------------------------------------------- 1 | package tables 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | ) 7 | 8 | type buf []byte 9 | 10 | func (b *buf) popInt32() int { 11 | n := int(int32(binary.BigEndian.Uint32((*b)[:4]))) 12 | *b = (*b)[4:] 13 | return n 14 | } 15 | 16 | func (b *buf) popInt16() (n int) { 17 | n = int(binary.BigEndian.Uint16((*b)[:2])) 18 | *b = (*b)[2:] 19 | return 20 | } 21 | 22 | func (b *buf) popStringN(high int) string { 23 | s := (*b)[:high] 24 | *b = (*b)[high:] 25 | return string(s) 26 | } 27 | 28 | func (b *buf) peekNextBytes(want []byte) bool { 29 | high := len(want) 30 | return bytes.Equal((*b)[:high], want) 31 | } 32 | 33 | func (b *buf) peekNextByte(next byte, n int) bool { 34 | high := n 35 | return bytes.Equal((*b)[:high], bytes.Repeat([]byte{next}, n)) 36 | } 37 | 38 | func (b *buf) popBytes(high int) []byte { 39 | s := (*b)[:high] 40 | *b = (*b)[high:] 41 | return s 42 | } 43 | -------------------------------------------------------------------------------- /cmd/sqledge/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "os" 7 | 8 | _ "github.com/jackc/pgx/v5/stdlib" 9 | _ "github.com/mattn/go-sqlite3" 10 | "github.com/rs/zerolog" 11 | "github.com/rs/zerolog/log" 12 | "github.com/zknill/sqledge/pkg/config" 13 | "github.com/zknill/sqledge/pkg/queryproxy" 14 | "github.com/zknill/sqledge/pkg/replicate" 15 | ) 16 | 17 | func main() { 18 | flag.Parse() 19 | zerolog.SetGlobalLevel(zerolog.DebugLevel) 20 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 21 | ctx := context.Background() 22 | 23 | cfg, err := config.Load() 24 | if err != nil { 25 | log.Fatal().Err(err).Msg("failed to parse config") 26 | } 27 | 28 | if err := queryproxy.Run(ctx, cfg); err != nil { 29 | log.Fatal().Err(err).Msg("failed to start sqledge") 30 | } 31 | 32 | if err := replicate.Run(ctx, cfg); err != nil { 33 | log.Fatal().Err(err).Msg("failed in replicate") 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /pkg/tables/testdata/init-db.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS alltypes ( 2 | int2 int2, 3 | int4 integer, 4 | int8 int8, 5 | text text, 6 | varchar varchar[30], 7 | json json, 8 | jsonb jsonb, 9 | int2arr int2[], 10 | int4arr int4[], 11 | int8arr int8[], 12 | textarr text[], 13 | bool bool, 14 | boolarr bool[], 15 | numeric numeric, 16 | numericarr numeric[], 17 | float4 float4, 18 | float8 float8, 19 | float4arr float4[], 20 | float8arr float8[], 21 | bytes bytea, 22 | bytesarr bytea[] 23 | ); 24 | 25 | insert into alltypes values ( 26 | 1, 27 | 2, 28 | 3, 29 | 'a', 30 | '{b}', 31 | '"c"', 32 | '"d"', 33 | '{4, 5}', 34 | '{6, 7}', 35 | '{9, 9}', 36 | '{"e", "f"}', 37 | true, 38 | '{true, false, true}', 39 | 10101.919191, 40 | '{8888.111, 9999.222}', 41 | 10.1, 42 | 11.2, 43 | '{12.3, 12.4}', 44 | '{13.5, 13.6}', 45 | 'a', 46 | '{"b"}' 47 | ); 48 | -------------------------------------------------------------------------------- /pkg/sqlgen/parser_test.go: -------------------------------------------------------------------------------- 1 | package sqlgen_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "github.com/zknill/sqledge/pkg/sqlgen" 8 | ) 9 | 10 | func TestParseSql(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | sql string 14 | wantTable string 15 | wantCols []sqlgen.ColDef 16 | }{ 17 | { 18 | 19 | name: "simple table", 20 | sql: `CREATE TABLE my_table ( 21 | id TEXT PRIMARY KEY, 22 | value INTEGER 23 | );`, 24 | wantTable: "my_table", 25 | wantCols: []sqlgen.ColDef{ 26 | {Name: "id", Type: "TEXT", PrimaryKey: true}, 27 | {Name: "value", Type: "INTEGER"}, 28 | }, 29 | }, 30 | { 31 | 32 | name: "composite primary key", 33 | sql: `CREATE TABLE my_table ( 34 | id TEXT, 35 | value INTEGER, 36 | rr REAL, 37 | other BLOB, 38 | PRIMARY KEY (id, value) 39 | );`, 40 | wantTable: "my_table", 41 | wantCols: []sqlgen.ColDef{ 42 | {Name: "id", Type: "TEXT", PrimaryKey: true}, 43 | {Name: "value", Type: "INTEGER", PrimaryKey: true}, 44 | {Name: "rr", Type: "REAL", PrimaryKey: false}, 45 | {Name: "other", Type: "BLOB", PrimaryKey: false}, 46 | }, 47 | }, 48 | } 49 | 50 | for i := range tests { 51 | test := tests[i] 52 | t.Run(test.name, func(t *testing.T) { 53 | 54 | p := sqlgen.NewParser(test.sql) 55 | 56 | table, cols, err := p.Parse() 57 | if err != nil { 58 | t.Error(err) 59 | } 60 | 61 | assert.Equal(t, test.wantTable, table) 62 | assert.Equal(t, test.wantCols, cols) 63 | }) 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /pkg/tables/copy.go: -------------------------------------------------------------------------------- 1 | package tables 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "io" 9 | 10 | "github.com/jackc/pgx/v5/pgconn" 11 | "github.com/rs/zerolog/log" 12 | "github.com/zknill/sqledge/pkg/sqlgen" 13 | ) 14 | 15 | type Conn interface { 16 | CopyTo(ctx context.Context, w io.Writer, sql string) (pgconn.CommandTag, error) 17 | Exec(ctx context.Context, sql string) *pgconn.MultiResultReader 18 | } 19 | 20 | func Copy(ctx context.Context, table string, def []sqlgen.ColDef, c Conn) ([][]string, error) { 21 | var err error 22 | // no position stored 23 | // copy the entire database 24 | b := &bytes.Buffer{} 25 | 26 | query := fmt.Sprintf(`COPY %s TO STDOUT WITH BINARY;`, table) 27 | log.Debug().Msg(query) 28 | 29 | _, err = c.CopyTo(ctx, b, query) 30 | if err != nil { 31 | log.Error().Err(err).Msg("copy error") 32 | } 33 | 34 | buf := buf(b.Bytes()) 35 | buf.popBytes(11) 36 | 37 | // flags 38 | _ = buf.popInt32() 39 | 40 | // header extension 41 | _ = buf.popInt32() 42 | 43 | decs := decoders(def) 44 | if err != nil { 45 | return nil, fmt.Errorf("build decs: %w", err) 46 | } 47 | 48 | cols := [][]string{} 49 | 50 | for { 51 | if buf.peekNextByte(0xff, 2) { 52 | _ = buf.popBytes(2) 53 | break 54 | } 55 | 56 | row := []string{} 57 | 58 | nFields := buf.popInt16() 59 | 60 | if nFields != len(decs) { 61 | return nil, errors.New("wrong number of decoders for tuple fields") 62 | } 63 | 64 | for i := 0; i < nFields; i++ { 65 | if buf.peekNextByte(0xff, 4) { 66 | _ = buf.popBytes(4) 67 | 68 | row = append(row, "null") 69 | 70 | continue 71 | } 72 | 73 | fieldLen := buf.popInt32() 74 | row = append(row, decs[i].Decode(buf.popBytes(fieldLen))) 75 | } 76 | 77 | cols = append(cols, row) 78 | } 79 | 80 | return cols, nil 81 | } 82 | -------------------------------------------------------------------------------- /pkg/queryproxy/sqledge.go: -------------------------------------------------------------------------------- 1 | package queryproxy 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "fmt" 7 | "net" 8 | "time" 9 | 10 | _ "github.com/jackc/pgx/v5/stdlib" 11 | _ "github.com/mattn/go-sqlite3" 12 | "github.com/rs/zerolog/log" 13 | "github.com/zknill/sqledge/pkg/config" 14 | "github.com/zknill/sqledge/pkg/pgwire" 15 | ) 16 | 17 | func Run(ctx context.Context, cfg *config.Config) error { 18 | localDB, err := sql.Open("sqlite3", cfg.Local.Path) 19 | if err != nil { 20 | return fmt.Errorf("connect to local db: %w", err) 21 | } 22 | 23 | log.Debug().Msg("connected to local") 24 | 25 | remoteDB, err := sql.Open("pgx", cfg.PostgresConnString()) 26 | if err != nil { 27 | return fmt.Errorf("connect to upstream db: %w", err) 28 | } 29 | 30 | log.Debug().Msgf("connected to remote %q, pinging", cfg.PostgresConnString()) 31 | 32 | pingCtx, cancel := context.WithTimeout(ctx, 3*time.Second) 33 | defer cancel() 34 | 35 | if err := remoteDB.PingContext(pingCtx); err != nil { 36 | return fmt.Errorf("ping upstream db: %w", err) 37 | } 38 | 39 | log.Debug().Msgf("connected to remote db, listening on %s:%d", cfg.Proxy.Address, cfg.Proxy.Port) 40 | 41 | lis, err := net.Listen("tcp", fmt.Sprintf("%s:%d", cfg.Proxy.Address, cfg.Proxy.Port)) 42 | if err != nil { 43 | log.Fatal().Msg(err.Error()) 44 | } 45 | 46 | go func() { 47 | defer remoteDB.Close() 48 | defer localDB.Close() 49 | defer lis.Close() 50 | <-ctx.Done() 51 | }() 52 | 53 | go func() { 54 | for { 55 | select { 56 | case <-ctx.Done(): 57 | return 58 | default: 59 | } 60 | 61 | conn, err := lis.Accept() 62 | if err != nil { 63 | log.Error().Err(err).Msg("accept err") 64 | 65 | continue 66 | } 67 | 68 | pgwire.Handle(cfg.Upstream.Schema, remoteDB, localDB, conn) 69 | } 70 | }() 71 | 72 | return nil 73 | } 74 | -------------------------------------------------------------------------------- /pkg/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/joeshaw/envdecode" 7 | ) 8 | 9 | type Config struct { 10 | Upstream struct { 11 | User string `env:"SQLEDGE_UPSTREAM_USER,default=postgres"` 12 | Pass string `env:"SQLEDGE_UPSTREAM_PASSWORD"` 13 | Address string `env:"SQLEDGE_UPSTREAM_ADDRESS,default=localhost"` 14 | Port int `env:"SQLEDGE_UPSTREAM_PORT,default=5432"` 15 | DBName string `env:"SQLEDGE_UPSTREAM_NAME,default=postgres"` 16 | Schema string `env:"SQLEDGE_UPSTREAM_SCHEMA,default=public"` 17 | } 18 | 19 | Replication struct { 20 | Plugin string `env:"SQLEDGE_REPLICATION_PLUGIN,default=pgoutput"` 21 | SlotName string `env:"SQLEDGE_REPLICATION_SLOT_NAME,default=sqledge"` 22 | CreateSlotIfNoExists bool `env:"SQLEDGE_REPLICATION_CREATE_SLOT,default=true"` 23 | Temporary bool `env:"SQLEDGE_REPLICATION_TEMP_SLOT,default=true"` 24 | Publication string `env:"SQLEDGE_REPLICATION_PUBLICATION,default=sqledge"` 25 | } 26 | 27 | Local struct { 28 | Path string `env:"SQLEDGE_LOCAL_DB_PATH,default=./sqledge.db"` 29 | } 30 | 31 | Proxy struct { 32 | Address string `env:"SQLEDGE_PROXY_ADDRESS,default=localhost"` 33 | Port int `env:"SQLEDGE_PROXY_ADDRESS,default=5433"` 34 | } 35 | } 36 | 37 | func (c *Config) PostgresConnString() string { 38 | pass := "" 39 | if c.Upstream.Pass != "" { 40 | pass = ":" + c.Upstream.Pass 41 | } 42 | 43 | s := fmt.Sprintf("postgres://%s%s@%s:%d/%s?application_name=sqledge", 44 | c.Upstream.User, 45 | pass, c.Upstream.Address, 46 | c.Upstream.Port, 47 | c.Upstream.DBName, 48 | ) 49 | 50 | return s 51 | } 52 | 53 | func Load() (*Config, error) { 54 | var c Config 55 | 56 | if err := envdecode.StrictDecode(&c); err != nil { 57 | return nil, fmt.Errorf("parse config: %w", err) 58 | } 59 | 60 | return &c, nil 61 | } 62 | -------------------------------------------------------------------------------- /pkg/tables/def.go: -------------------------------------------------------------------------------- 1 | package tables 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | 7 | "github.com/zknill/sqledge/pkg/sqlgen" 8 | ) 9 | 10 | type Querier interface { 11 | Query(query string, args ...any) (*sql.Rows, error) 12 | } 13 | 14 | func ColDefs(db Querier, table string) ([]sqlgen.ColDef, error) { 15 | query := ` 16 | SELECT column_name column, udt_name as type 17 | FROM information_schema.columns 18 | WHERE table_name = $1 19 | ORDER BY ordinal_position; 20 | ` 21 | 22 | rows, err := db.Query(query, table) 23 | if err != nil { 24 | return nil, fmt.Errorf("query schema: %w", err) 25 | } 26 | 27 | var n, t string 28 | var arr bool 29 | 30 | defs := []sqlgen.ColDef{} 31 | 32 | for rows.Next() { 33 | arr = false 34 | 35 | if err := rows.Scan(&n, &t); err != nil { 36 | return nil, fmt.Errorf("scan") 37 | } 38 | 39 | if t[0] == '_' { 40 | t = t[1:] 41 | arr = true 42 | } 43 | 44 | defs = append(defs, sqlgen.ColDef{ 45 | Name: n, 46 | Type: sqlgen.ColType(t), 47 | PrimaryKey: false, 48 | Array: arr, 49 | }) 50 | } 51 | 52 | return defs, nil 53 | } 54 | 55 | func TableColDefs(db Querier, schema string, filterTables []string) (map[string][]sqlgen.ColDef, error) { 56 | tables := make([]string, len(filterTables)) 57 | copy(tables, filterTables) 58 | 59 | if len(tables) == 0 { 60 | t, err := findTables(db, schema) 61 | if err != nil { 62 | return nil, fmt.Errorf("find tables: %w", err) 63 | } 64 | 65 | tables = t 66 | } 67 | 68 | out := make(map[string][]sqlgen.ColDef) 69 | 70 | for _, t := range tables { 71 | defs, err := ColDefs(db, t) 72 | if err != nil { 73 | return nil, fmt.Errorf("col definitions for %q.%q: %w", schema, t, err) 74 | } 75 | 76 | out[t] = defs 77 | } 78 | 79 | return out, nil 80 | } 81 | 82 | func findTables(db Querier, schema string) ([]string, error) { 83 | query := `SELECT table_name 84 | FROM information_schema.tables 85 | WHERE table_schema = $1;` 86 | 87 | rows, err := db.Query(query, schema) 88 | if err != nil { 89 | return nil, fmt.Errorf("query tables: %w", err) 90 | } 91 | 92 | var t string 93 | var out []string 94 | 95 | for rows.Next() { 96 | if err := rows.Scan(&t); err != nil { 97 | return nil, fmt.Errorf("scan table name: %w", err) 98 | } 99 | 100 | out = append(out, t) 101 | } 102 | 103 | return out, nil 104 | } 105 | -------------------------------------------------------------------------------- /pkg/sqlgen/driver.go: -------------------------------------------------------------------------------- 1 | package sqlgen 2 | 3 | import ( 4 | "database/sql" 5 | "errors" 6 | "fmt" 7 | ) 8 | 9 | type SqliteDriver struct { 10 | db *sql.DB 11 | cfg SqliteConfig 12 | } 13 | 14 | func NewSqliteDriver(cfg SqliteConfig, db *sql.DB) *SqliteDriver { 15 | return &SqliteDriver{ 16 | cfg: cfg, 17 | db: db, 18 | } 19 | } 20 | 21 | func (s *SqliteDriver) Execute(query string) error { 22 | _, err := s.db.Exec(query) 23 | return err 24 | } 25 | 26 | func (s *SqliteDriver) Pos() (string, error) { 27 | query := `SELECT pos 28 | FROM postgres_pos 29 | WHERE source_db = ? 30 | AND plugin = ? 31 | AND publication = ?;` 32 | 33 | row := s.db.QueryRow(query, s.cfg.SourceDB, s.cfg.Plugin, s.cfg.Publication) 34 | 35 | var pos string 36 | 37 | if err := row.Scan(&pos); err != nil { 38 | if errors.Is(err, sql.ErrNoRows) { 39 | return "", nil 40 | } 41 | 42 | return "", fmt.Errorf("read position: %w", err) 43 | } 44 | 45 | return pos, nil 46 | } 47 | 48 | func (s *SqliteDriver) InitPositionTable() error { 49 | _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS postgres_pos ( 50 | source_db text, 51 | plugin text, 52 | publication text, 53 | pos text, 54 | PRIMARY KEY (source_db, plugin, publication) 55 | )`) 56 | if err != nil { 57 | return fmt.Errorf("create lsn table: %w", err) 58 | } 59 | 60 | return nil 61 | } 62 | 63 | func (s *SqliteDriver) CurrentSchema() (map[string]map[string]ColDef, error) { 64 | // tableName -> colName -> colDef 65 | out := make(map[string]map[string]ColDef) 66 | 67 | query := `SELECT tbl_name, sql FROM sqlite_schema WHERE type = 'table';` 68 | 69 | type tableRow struct { 70 | TableName string `db:"tbl_name"` 71 | SQL string `db:"sql"` 72 | } 73 | 74 | rows, err := s.db.Query(query) 75 | if err != nil { 76 | return nil, fmt.Errorf("query schema: %w", err) 77 | } 78 | 79 | for rows.Next() { 80 | tr := tableRow{} 81 | if err := rows.Scan(&tr.TableName, &tr.SQL); err != nil { 82 | return nil, fmt.Errorf("scan: %w", err) 83 | } 84 | 85 | tableName, cols, err := NewParser(tr.SQL).Parse() 86 | if err != nil { 87 | return nil, fmt.Errorf("parse table %q: %w", tr.TableName, err) 88 | } 89 | 90 | current := map[string]ColDef{} 91 | 92 | for i := range cols { 93 | col := cols[i] 94 | current[col.Name] = col 95 | } 96 | 97 | out[tableName] = current 98 | } 99 | 100 | return out, nil 101 | } 102 | -------------------------------------------------------------------------------- /pkg/replicate/run.go: -------------------------------------------------------------------------------- 1 | package replicate 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "fmt" 7 | 8 | _ "github.com/mattn/go-sqlite3" 9 | "github.com/rs/zerolog/log" 10 | "github.com/zknill/sqledge/pkg/config" 11 | "github.com/zknill/sqledge/pkg/sqlgen" 12 | ) 13 | 14 | func Run(ctx context.Context, cfg *config.Config) error { 15 | connStr := cfg.PostgresConnString() + "&replication=database" 16 | 17 | conn, err := replicateConnection(ctx, connStr, cfg.Replication.Publication) 18 | if err != nil { 19 | return fmt.Errorf("create replicate connection: %w", err) 20 | } 21 | defer conn.Close() 22 | 23 | // TODO: this is shared across reader and writer 24 | db, err := sql.Open("sqlite3", cfg.Local.Path) 25 | if err != nil { 26 | return fmt.Errorf("connect to local db: %w", err) 27 | } 28 | 29 | sqliteCfg := sqlgen.SqliteConfig{ 30 | SourceDB: cfg.Upstream.DBName, 31 | Plugin: cfg.Replication.Plugin, 32 | Publication: cfg.Replication.Publication, 33 | } 34 | 35 | driver := sqlgen.NewSqliteDriver(sqliteCfg, db) 36 | 37 | if err := driver.InitPositionTable(); err != nil { 38 | return fmt.Errorf("init position tracking: %w", err) 39 | } 40 | 41 | schema, err := driver.CurrentSchema() 42 | if err != nil { 43 | return fmt.Errorf("get current schema: %w", err) 44 | } 45 | 46 | sqlite := sqlgen.NewSqlite(sqliteCfg, schema) 47 | if err != nil { 48 | return fmt.Errorf("init sqlgen: %w", err) 49 | } 50 | 51 | slot := SlotConfig{ 52 | SlotName: cfg.Replication.SlotName, 53 | OutputPlugin: cfg.Replication.Plugin, 54 | CreateSlotIfNoExists: cfg.Replication.CreateSlotIfNoExists, 55 | Temporary: cfg.Replication.Temporary, 56 | Schema: cfg.Upstream.Schema, 57 | } 58 | 59 | log.Debug().Msg("starting streaming") 60 | 61 | if err := conn.Stream( 62 | ctx, 63 | slot, 64 | driver, 65 | sqlite, 66 | ); err != nil { 67 | return fmt.Errorf("streaming failed: %w", err) 68 | } 69 | 70 | return nil 71 | } 72 | 73 | func replicateConnection(ctx context.Context, connectionString, publication string) (*Conn, error) { 74 | conn, err := NewConn(ctx, connectionString, publication) 75 | if err != nil { 76 | return nil, fmt.Errorf("new conn: %w", err) 77 | } 78 | 79 | if err := conn.DropPublication(); err != nil { 80 | return nil, fmt.Errorf("drop publication: %w", err) 81 | } 82 | 83 | if err := conn.CreatePublication(); err != nil { 84 | return nil, fmt.Errorf("create publication: %w", err) 85 | } 86 | 87 | return conn, nil 88 | } 89 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/zknill/sqledge 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/jackc/pglogrepl v0.0.0-20230630212501-5fd22a600b50 7 | github.com/jackc/pgx/v5 v5.4.2 8 | github.com/joeshaw/envdecode v0.0.0-20200121155833-099f1fc765bd 9 | github.com/mattn/go-sqlite3 v1.14.17 10 | github.com/rs/zerolog v1.29.1 11 | github.com/stretchr/testify v1.8.4 12 | github.com/testcontainers/testcontainers-go v0.21.0 13 | github.com/testcontainers/testcontainers-go/modules/postgres v0.21.0 14 | ) 15 | 16 | require ( 17 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect 18 | github.com/Microsoft/go-winio v0.5.2 // indirect 19 | github.com/cenkalti/backoff/v4 v4.2.0 // indirect 20 | github.com/containerd/containerd v1.6.19 // indirect 21 | github.com/cpuguy83/dockercfg v0.3.1 // indirect 22 | github.com/davecgh/go-spew v1.1.1 // indirect 23 | github.com/docker/distribution v2.8.2+incompatible // indirect 24 | github.com/docker/docker v23.0.5+incompatible // indirect 25 | github.com/docker/go-connections v0.4.0 // indirect 26 | github.com/docker/go-units v0.5.0 // indirect 27 | github.com/gogo/protobuf v1.3.2 // indirect 28 | github.com/golang/protobuf v1.5.2 // indirect 29 | github.com/google/uuid v1.3.0 // indirect 30 | github.com/imdario/mergo v0.3.15 // indirect 31 | github.com/jackc/pgio v1.0.0 // indirect 32 | github.com/jackc/pgpassfile v1.0.0 // indirect 33 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect 34 | github.com/klauspost/compress v1.15.9 // indirect 35 | github.com/magiconair/properties v1.8.7 // indirect 36 | github.com/mattn/go-colorable v0.1.12 // indirect 37 | github.com/mattn/go-isatty v0.0.16 // indirect 38 | github.com/moby/patternmatcher v0.5.0 // indirect 39 | github.com/moby/sys/sequential v0.5.0 // indirect 40 | github.com/moby/term v0.5.0 // indirect 41 | github.com/morikuni/aec v1.0.0 // indirect 42 | github.com/opencontainers/go-digest v1.0.0 // indirect 43 | github.com/opencontainers/image-spec v1.1.0-rc2 // indirect 44 | github.com/opencontainers/runc v1.1.5 // indirect 45 | github.com/pkg/errors v0.9.1 // indirect 46 | github.com/pmezard/go-difflib v1.0.0 // indirect 47 | github.com/sirupsen/logrus v1.9.0 // indirect 48 | golang.org/x/crypto v0.11.0 // indirect 49 | golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect 50 | golang.org/x/net v0.10.0 // indirect 51 | golang.org/x/sys v0.10.0 // indirect 52 | golang.org/x/text v0.11.0 // indirect 53 | google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad // indirect 54 | google.golang.org/grpc v1.47.0 // indirect 55 | google.golang.org/protobuf v1.28.0 // indirect 56 | gopkg.in/yaml.v3 v3.0.1 // indirect 57 | ) 58 | -------------------------------------------------------------------------------- /pkg/tables/copy_test.go: -------------------------------------------------------------------------------- 1 | package tables_test 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "testing" 9 | "time" 10 | 11 | "github.com/jackc/pgx/v5/pgconn" 12 | "github.com/rs/zerolog" 13 | "github.com/rs/zerolog/log" 14 | "github.com/stretchr/testify/assert" 15 | "github.com/testcontainers/testcontainers-go" 16 | "github.com/testcontainers/testcontainers-go/modules/postgres" 17 | "github.com/testcontainers/testcontainers-go/wait" 18 | "github.com/zknill/sqledge/pkg/sqlgen" 19 | "github.com/zknill/sqledge/pkg/tables" 20 | ) 21 | 22 | func init() { 23 | zerolog.SetGlobalLevel(zerolog.DebugLevel) 24 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 25 | } 26 | 27 | func TestCopy(t *testing.T) { 28 | ctx := context.Background() 29 | 30 | pgContainer, err := postgres.RunContainer(ctx, 31 | testcontainers.WithImage("postgres:15.3-alpine"), 32 | postgres.WithInitScripts(filepath.Join("testdata", "init-db.sql")), 33 | postgres.WithDatabase("test-db"), 34 | postgres.WithUsername("postgres"), 35 | postgres.WithPassword("postgres"), 36 | testcontainers.WithWaitStrategy( 37 | wait.ForLog("database system is ready to accept connections"). 38 | WithOccurrence(2).WithStartupTimeout(10*time.Second)), 39 | ) 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | 44 | t.Cleanup(func() { 45 | if err := pgContainer.Terminate(ctx); err != nil { 46 | t.Fatalf("failed to terminate pgContainer: %s", err) 47 | } 48 | }) 49 | 50 | connStr, err := pgContainer.ConnectionString(ctx, "sslmode=disable") 51 | assert.NoError(t, err) 52 | 53 | connStr = strings.ReplaceAll(connStr, "host=localhost", "host=0.0.0.0") 54 | 55 | conn, err := pgconn.Connect(context.Background(), connStr) 56 | assert.NoError(t, err) 57 | 58 | def := []sqlgen.ColDef{ 59 | {Type: sqlgen.PgColTypeInt2}, 60 | {Type: sqlgen.PgColTypeInt4}, 61 | {Type: sqlgen.PgColTypeInt8}, 62 | {Type: sqlgen.PgColTypeText}, 63 | {Type: sqlgen.PgColTypeText, Array: true}, 64 | {Type: sqlgen.PgColTypeJson}, 65 | {Type: sqlgen.PgColTypeJsonB}, 66 | {Type: sqlgen.PgColTypeInt2, Array: true}, 67 | {Type: sqlgen.PgColTypeInt4, Array: true}, 68 | {Type: sqlgen.PgColTypeInt8, Array: true}, 69 | {Type: sqlgen.PgColTypeText, Array: true}, 70 | {Type: sqlgen.PgColTypeBool}, 71 | {Type: sqlgen.PgColTypeBool, Array: true}, 72 | {Type: sqlgen.PgColTypeNum}, 73 | {Type: sqlgen.PgColTypeNum, Array: true}, 74 | {Type: sqlgen.PgColTypeFloat4}, 75 | {Type: sqlgen.PgColTypeFloat8}, 76 | {Type: sqlgen.PgColTypeFloat4, Array: true}, 77 | {Type: sqlgen.PgColTypeFloat8, Array: true}, 78 | {Type: sqlgen.PgColTypeBytea}, 79 | {Type: sqlgen.PgColTypeBytea, Array: true}, 80 | } 81 | 82 | cols, err := tables.Copy(context.Background(), "alltypes", def, conn) 83 | assert.NoError(t, err) 84 | 85 | want := [][]string{{ 86 | "1", 87 | "2", 88 | "3", 89 | "a", 90 | `{"b"}`, 91 | `"c"`, 92 | `"d"`, 93 | "{4, 5}", 94 | "{6, 7}", 95 | "{9, 9}", 96 | `{"e", "f"}`, 97 | "true", 98 | `{"true", "false", "true"}`, 99 | "10101.919191", 100 | "{8888.111, 9999.222}", 101 | "10.1", 102 | "11.2", 103 | "{12.3, 12.4}", 104 | "{13.5, 13.6}", 105 | "a", 106 | `{"b"}`, 107 | }} 108 | 109 | assert.Equal(t, want, cols) 110 | 111 | } 112 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SQLedge 2 | 3 | [State: alpha] 4 | 5 | SQLedge uses Postgres logical replication to stream the changes in a source Postgres database to a SQLite database that can run on the edge. 6 | SQLedge serves reads from its local SQLite database, and forwards writes to the upstream Postgres server that it's replicating from. 7 | 8 | This lets you run your apps on the edge, and have local, fast, and eventually consistent access to your data. 9 | 10 | ![SQLedge](https://github.com/zknill/sqledge/blob/main/etc/sqledge.png?raw=true) 11 | 12 | ## SQL generation 13 | 14 | The `pkg/sqlgen` package has an SQL generator in it, which will generate the SQLite insert, update, delete statements based on the logical replication messages received. 15 | 16 | ## SQL parsing 17 | 18 | When the database is started, we look at which tables already exist in the sqlite copy, and make sure new tables are created automatically on the fly. 19 | 20 | ## Postgres wire proxy 21 | 22 | SQLedge contains a Postgres wire proxy, default on `localhost:5433`. This proxy uses the local SQlite database for reads, and forwards writes to the upstream Postgres server. 23 | 24 | ### Compatibility 25 | 26 | When running, the SQL statements interact with two databases; Postgres (for writes) and SQLite (for reads). 27 | 28 | The Postgres wire proxy (which forwards reads to SQLite) doesn't currently translate any of the SQL statements from the Postgres query format/functions to the SQLite format/functions. 29 | Read queries issued against the Postgres wire proxy need to be compatible with SQLite directly. 30 | This is fine for simple `SELECT` queries, but you will have trouble with Postgres-specific query functions or syntax. 31 | 32 | ## Copy on startup 33 | 34 | SQLEdge maintains a table called `postgres_pos`, this tracks the LSN (log sequence number) of the received logical replication messages so it can pick up processing where it left 35 | off. 36 | 37 | If no LSN is found, SQLedge will start a postgres `COPY` of all tables in the `public` schema. Creating the appropriate SQLite tables, and inserting data. 38 | 39 | When the replication slot is first created, it exports a transaction snapshot. This snapshot is used for the initial copy. This means that the `COPY` command will read the data from 40 | the transaction at the moment the replication slot was created. 41 | 42 | ## Trying it out 43 | 44 | 1. Create a database 45 | 46 | ``` 47 | create database myappdatabase; 48 | ``` 49 | 50 | 2. Create a user -- must be a super user because we create a publication on all tables 51 | 52 | ``` 53 | create user sqledger with login superuser password 'secret'; 54 | ``` 55 | 56 | 57 | 3. Run the example 58 | 59 | ``` 60 | SQLEDGE_UPSTREAM_USER=sqledger SQLEDGE_UPSTREAM_PASSWORD=secret SQLEDGE_UPSTREAM_NAME=myappdatabase go run ./cmd/sqledge/main.go 61 | ``` 62 | 63 | 4. Connect to the postgres wire proxy 64 | 65 | ``` 66 | psql -h localhost -p 5433 67 | 68 | $ CREATE TABLE my_table (id serial not null primary key, names text); 69 | $ INSERT INTO my_table (names) VALUES ('Jane'), ('John'); 70 | 71 | $ SELECT * FROM my_table; 72 | ``` 73 | The read will be served from the local database 74 | 75 | 5. Connect to the local sqlite db 76 | 77 | ``` 78 | sqlite3 ./sqledge.db 79 | 80 | .schema 81 | ``` 82 | 83 | ## Config 84 | 85 | All config is read from environment variables. The full list is available in the struct tags on the fields in `pkg/config/config.go` 86 | -------------------------------------------------------------------------------- /pkg/tables/decode.go: -------------------------------------------------------------------------------- 1 | package tables 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "fmt" 7 | "math" 8 | "strconv" 9 | "strings" 10 | 11 | "github.com/rs/zerolog/log" 12 | "github.com/zknill/sqledge/pkg/sqlgen" 13 | ) 14 | 15 | type FieldDecoder interface { 16 | Decode(b []byte) string 17 | numeric() bool 18 | } 19 | 20 | func decoders(def []sqlgen.ColDef) []FieldDecoder { 21 | out := make([]FieldDecoder, len(def)) 22 | for i, d := range def { 23 | d := d 24 | 25 | switch d.Type { 26 | case sqlgen.PgColTypeText: 27 | out[i] = new(str) 28 | case sqlgen.PgColTypeInt2: 29 | out[i] = new(int2) 30 | case sqlgen.PgColTypeInt4: 31 | out[i] = new(int4) 32 | case sqlgen.PgColTypeInt8: 33 | out[i] = new(int8) 34 | case sqlgen.PgColTypeNum: 35 | out[i] = new(numeric) 36 | case sqlgen.PgColTypeFloat4: 37 | out[i] = new(float4) 38 | case sqlgen.PgColTypeFloat8: 39 | out[i] = new(float8) 40 | case sqlgen.PgColTypeBytea: 41 | out[i] = new(bytea) 42 | case sqlgen.PgColTypeJson: 43 | out[i] = new(str) 44 | case sqlgen.PgColTypeJsonB: 45 | out[i] = new(jsonb) 46 | case sqlgen.PgColTypeBool: 47 | out[i] = new(boolean) 48 | default: 49 | out[i] = new(str) 50 | } 51 | 52 | if d.Array { 53 | out[i] = &arr{elem: out[i]} 54 | } 55 | } 56 | 57 | return out 58 | } 59 | 60 | type int2 struct{} 61 | 62 | func (i *int2) numeric() bool { return true } 63 | func (i *int2) Decode(b []byte) string { 64 | v := binary.BigEndian.Uint16(b) 65 | return strconv.FormatUint(uint64(v), 10) 66 | } 67 | 68 | type int4 struct{} 69 | 70 | func (i *int4) numeric() bool { return true } 71 | func (i *int4) Decode(b []byte) string { 72 | v := binary.BigEndian.Uint32(b) 73 | return strconv.FormatUint(uint64(v), 10) 74 | } 75 | 76 | type int8 struct{} 77 | 78 | func (i *int8) numeric() bool { return true } 79 | func (i *int8) Decode(b []byte) string { 80 | v := binary.BigEndian.Uint64(b) 81 | return strconv.FormatUint(uint64(v), 10) 82 | } 83 | 84 | type float4 struct{} 85 | 86 | func (f *float4) numeric() bool { return true } 87 | func (f *float4) Decode(b []byte) string { 88 | v := binary.BigEndian.Uint32(b) 89 | return strconv.FormatFloat(float64(math.Float32frombits(v)), 'f', -1, 32) 90 | } 91 | 92 | type float8 struct{} 93 | 94 | func (f *float8) numeric() bool { return true } 95 | func (f *float8) Decode(b []byte) string { 96 | v := binary.BigEndian.Uint64(b) 97 | return strconv.FormatFloat(math.Float64frombits(v), 'f', -1, 64) 98 | } 99 | 100 | type numeric struct{} 101 | 102 | func (n *numeric) numeric() bool { return true } 103 | func (n *numeric) Decode(b []byte) string { 104 | buf := buf(b) 105 | ndigits := buf.popInt16() 106 | weight := buf.popInt16() 107 | // sign 108 | _ = buf.popInt16() 109 | dscale := buf.popInt16() 110 | 111 | s := &strings.Builder{} 112 | 113 | for i := 0; i < ndigits; i++ { 114 | x := buf.popInt16() 115 | if weight > 0 { 116 | x *= int(math.Pow10(weight)) 117 | weight-- 118 | } 119 | 120 | if i == ndigits-1 { 121 | x = withoutZeros(x) 122 | } 123 | 124 | fmt.Fprintf(s, "%d", x) 125 | } 126 | 127 | v, _ := strconv.ParseInt(s.String(), 10, 64) 128 | out := float64(v) / math.Pow10(dscale) 129 | 130 | return strconv.FormatFloat(out, 'f', -1, 64) 131 | } 132 | 133 | func withoutZeros(i int) int { 134 | result := 0 135 | mul := 1 136 | for remainingDigits := i; remainingDigits > 0; remainingDigits /= 10 { 137 | lastDigit := remainingDigits % 10 138 | if lastDigit != 0 { 139 | result += lastDigit * mul 140 | mul *= 10 141 | } 142 | } 143 | return result 144 | } 145 | 146 | type str struct{} 147 | 148 | func (s *str) numeric() bool { return false } 149 | func (s *str) Decode(b []byte) string { return string(b) } 150 | 151 | type jsonb struct{} 152 | 153 | func (j *jsonb) numeric() bool { return false } 154 | func (j *jsonb) Decode(b []byte) string { return string(b[1:]) } 155 | 156 | type bytea struct{} 157 | 158 | func (b *bytea) numeric() bool { return false } 159 | func (b *bytea) Decode(v []byte) string { return string(v) } 160 | 161 | type boolean struct{} 162 | 163 | func (b *boolean) numeric() bool { return false } 164 | func (b *boolean) Decode(v []byte) string { 165 | if v[0] == 0x01 { 166 | return "true" 167 | } 168 | 169 | return "false" 170 | } 171 | 172 | type arr struct { 173 | elem FieldDecoder 174 | } 175 | 176 | func (d *arr) numeric() bool { return d.elem.numeric() } 177 | 178 | func (d *arr) Decode(b []byte) string { 179 | buf := buf(b) 180 | 181 | ndim := buf.popInt32() 182 | if ndim > 1 { 183 | panic("does not support multi dimension arrays") 184 | } 185 | 186 | hasNull := buf.popInt32() 187 | elemType := buf.popInt32() 188 | dim := buf.popInt32() 189 | lb := buf.popInt32() 190 | 191 | log.Trace().Msgf("num dimensions: %d", ndim) 192 | log.Trace().Msgf("hasNull: %d", hasNull) 193 | log.Trace().Msgf("elemType: %d", elemType) 194 | log.Trace().Msgf("dimensions: %d", dim) 195 | log.Trace().Msgf("lower bound: %d", lb) 196 | 197 | out := bytes.Buffer{} 198 | 199 | out.WriteRune('{') 200 | 201 | for i := 0; i < dim; i++ { 202 | if hasNull == 1 { 203 | if buf.peekNextByte(0xff, 4) { 204 | out.WriteString("null") 205 | 206 | if i < dim-1 { 207 | out.WriteString(", ") 208 | } 209 | 210 | _ = buf.popBytes(4) 211 | 212 | continue 213 | } 214 | 215 | } 216 | 217 | fieldLen := buf.popInt32() 218 | 219 | if !d.elem.numeric() { 220 | out.WriteString(`"`) 221 | } 222 | 223 | out.WriteString(d.elem.Decode(buf.popBytes(fieldLen))) 224 | 225 | if !d.elem.numeric() { 226 | out.WriteString(`"`) 227 | } 228 | 229 | if i < dim-1 { 230 | out.WriteString(", ") 231 | } 232 | } 233 | out.WriteRune('}') 234 | 235 | return out.String() 236 | } 237 | -------------------------------------------------------------------------------- /pkg/sqlgen/parser.go: -------------------------------------------------------------------------------- 1 | package sqlgen 2 | 3 | import ( 4 | "errors" 5 | "regexp" 6 | "strings" 7 | 8 | "github.com/rs/zerolog/log" 9 | ) 10 | 11 | type step int 12 | 13 | const ( 14 | stepType step = iota 15 | stepCreateTable 16 | stepIfNoExists 17 | stepSchemaTableName 18 | stepAs 19 | stepSelectStatement 20 | stepColumnDefsOpenBracket 21 | stepColumDefComma 22 | stepColumnDefTableConstraint 23 | stepColumnDefsCloseBracket 24 | stepColumnDefPrimaryKey 25 | stepColumnDefConstraints 26 | ) 27 | 28 | type ColDef struct { 29 | Name string 30 | Type ColType 31 | PrimaryKey bool 32 | Array bool 33 | } 34 | 35 | type ColType string 36 | 37 | const ( 38 | // TODO: this is not all the types, missing: datetime, etc 39 | PgColTypeText ColType = "text" 40 | PgColTypeInt2 ColType = "int2" 41 | PgColTypeInt4 ColType = "int4" 42 | PgColTypeInt8 ColType = "int8" 43 | PgColTypeNum ColType = "numeric" 44 | PgColTypeFloat4 ColType = "float4" 45 | PgColTypeFloat8 ColType = "float8" 46 | PgColTypeBytea ColType = "bytea" 47 | PgColTypeJson ColType = "json" 48 | PgColTypeJsonB ColType = "jsonb" 49 | PgColTypeBool ColType = "bool" 50 | 51 | SQLiteColTypeInteger ColType = "integer" 52 | SQLiteColTypeReal ColType = "real" 53 | SQLiteColTypeText ColType = "text" 54 | SQLiteColTypeBlob ColType = "blob" 55 | ) 56 | 57 | func (c ColType) PgType() (oid, size int) { 58 | switch c { 59 | case SQLiteColTypeText: 60 | return 25, -1 61 | case SQLiteColTypeInteger: 62 | return 23, 4 63 | case SQLiteColTypeReal: 64 | return 700, 4 65 | case SQLiteColTypeBlob: 66 | return 17, -1 67 | } 68 | 69 | return -1, -1 70 | } 71 | 72 | type Parser struct { 73 | sql string 74 | i int 75 | step step 76 | table string 77 | cols []ColDef 78 | } 79 | 80 | func NewParser(sql string) *Parser { 81 | sql = strings.NewReplacer("\n", " ", "\t", " ").Replace(sql) 82 | return &Parser{sql: strings.TrimSpace(sql)} 83 | } 84 | 85 | func (p *Parser) Parse() (string, []ColDef, error) { 86 | for { 87 | if p.i >= len(p.sql) { 88 | return p.table, p.cols, nil 89 | } 90 | 91 | switch p.step { 92 | case stepType: 93 | if peeked := p.peek(); peeked != "CREATE TABLE" { 94 | return p.table, p.cols, errors.New("invalid query, peeked: " + peeked) 95 | } 96 | p.pop() 97 | p.step = stepCreateTable 98 | case stepCreateTable: 99 | switch p.peek() { 100 | case "IF NOT EXISTS": 101 | p.pop() 102 | p.step = stepIfNoExists 103 | default: 104 | p.popWhitespace() 105 | p.step = stepSchemaTableName 106 | } 107 | case stepSchemaTableName: 108 | log.Trace().Msg("enter stepSchemaTableName") 109 | p.table = p.pop() 110 | peeked := strings.ToUpper(p.peek()) 111 | switch peeked { 112 | case "AS": 113 | p.step = stepAs 114 | p.pop() 115 | case "(": 116 | p.step = stepColumnDefsOpenBracket 117 | p.pop() 118 | default: 119 | return p.table, p.cols, errors.New("unknown table schema step, peeked: " + peeked) 120 | } 121 | case stepAs: 122 | return p.table, p.cols, errors.New("AS tables not supported") 123 | case stepColumnDefsOpenBracket: 124 | log.Trace().Msg("enter stepColumnDefsOpenBracket") 125 | switch p.peek() { 126 | case "PRIMARY KEY": 127 | p.pop() 128 | p.step = stepColumnDefPrimaryKey 129 | continue 130 | } 131 | colName := p.pop() 132 | log.Trace().Msgf("col name: %q\n", colName) 133 | typeName := p.pop() 134 | log.Trace().Msgf("type name: %q\n", typeName) 135 | p.cols = append(p.cols, ColDef{ 136 | Name: colName, 137 | Type: ColType(typeName), 138 | }) 139 | 140 | switch p.peek() { 141 | case ",": 142 | p.pop() 143 | p.step = stepColumnDefsOpenBracket 144 | case ")": 145 | p.pop() 146 | p.step = stepColumnDefsCloseBracket 147 | default: 148 | p.step = stepColumnDefConstraints 149 | } 150 | case stepColumnDefConstraints: 151 | log.Trace().Msg("enter stepColumnDefConstraints") 152 | switch p.peek() { 153 | case "NOT NULL": 154 | p.pop() 155 | case "PRIMARY KEY": 156 | p.pop() 157 | p.cols[len(p.cols)-1].PrimaryKey = true 158 | case ",": 159 | p.pop() 160 | p.step = stepColumnDefsOpenBracket 161 | case ")": 162 | p.pop() 163 | p.step = stepColumnDefsCloseBracket 164 | } 165 | 166 | case stepColumnDefPrimaryKey: 167 | log.Trace().Msg("enter stepColumnDefPrimaryKey") 168 | switch p.peek() { 169 | case "(": 170 | p.pop() 171 | colName := p.pop() 172 | p.makeColPK(colName) 173 | // make col PK 174 | case ",": 175 | p.pop() 176 | colName := p.pop() 177 | p.makeColPK(colName) 178 | case ")": 179 | p.step = stepColumnDefConstraints 180 | default: 181 | return p.table, p.cols, errors.New("unknown column def PK") 182 | } 183 | case stepColumnDefsCloseBracket: 184 | return p.table, p.cols, nil 185 | } 186 | } 187 | } 188 | 189 | func (p *Parser) makeColPK(name string) { 190 | for i := range p.cols { 191 | if p.cols[i].Name == name { 192 | p.cols[i].PrimaryKey = true 193 | return 194 | } 195 | } 196 | } 197 | 198 | func (p *Parser) peek() string { 199 | token, _ := p.peekWithLength() 200 | return token 201 | } 202 | 203 | func (p *Parser) seekToNext(r rune) bool { 204 | idx := strings.IndexRune(p.sql[p.i:], r) 205 | if idx == -1 { 206 | return false 207 | } 208 | 209 | p.i += idx + 1 210 | p.popWhitespace() 211 | return true 212 | } 213 | 214 | func (p *Parser) pop() string { 215 | log.Trace().Msgf(">popping (%d) %q\n", p.i, p.sql[p.i:]) 216 | token, l := p.peekWithLength() 217 | p.i = p.i + l 218 | p.popWhitespace() 219 | 220 | log.Trace().Msgf("= len(p.sql) { 232 | return "", 0 233 | } 234 | 235 | for _, token := range tokens { 236 | t := strings.ToUpper(p.sql[p.i:min(len(p.sql), p.i+len(token))]) 237 | if token == t { 238 | return t, len(t) 239 | } 240 | } 241 | 242 | return p.peekIdentifierWithLength() 243 | } 244 | 245 | var pattern = regexp.MustCompile(`[a-zA-Z0-9\._*]`) 246 | 247 | func (p *Parser) peekIdentifierWithLength() (string, int) { 248 | for i := p.i; i < len(p.sql); i++ { 249 | if !pattern.MatchString(string(p.sql[i])) { 250 | return p.sql[p.i:i], len(p.sql[p.i:i]) 251 | } 252 | } 253 | return p.sql[p.i:], len(p.sql[p.i:]) 254 | } 255 | 256 | func (p *Parser) popWhitespace() { 257 | for ; p.i < len(p.sql) && p.sql[p.i] == ' '; p.i++ { 258 | } 259 | } 260 | 261 | func min(a, b int) int { 262 | if a < b { 263 | return a 264 | } 265 | return b 266 | } 267 | -------------------------------------------------------------------------------- /test/integration/integration_test.go: -------------------------------------------------------------------------------- 1 | package integration_test 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "errors" 7 | "fmt" 8 | "math/rand" 9 | "os" 10 | "strings" 11 | "sync" 12 | "testing" 13 | "time" 14 | 15 | "github.com/jackc/pgx/v5/pgconn" 16 | _ "github.com/jackc/pgx/v5/stdlib" 17 | _ "github.com/mattn/go-sqlite3" 18 | "github.com/rs/zerolog" 19 | "github.com/rs/zerolog/log" 20 | "github.com/stretchr/testify/assert" 21 | "github.com/testcontainers/testcontainers-go" 22 | "github.com/testcontainers/testcontainers-go/modules/postgres" 23 | "github.com/testcontainers/testcontainers-go/wait" 24 | "github.com/zknill/sqledge/pkg/config" 25 | "github.com/zknill/sqledge/pkg/queryproxy" 26 | "github.com/zknill/sqledge/pkg/replicate" 27 | ) 28 | 29 | type nameRow struct { 30 | id int 31 | name string 32 | } 33 | 34 | const ( 35 | dbName = "test-db" 36 | userName = "postgres" 37 | password = "password" 38 | ) 39 | 40 | func init() { 41 | zerolog.SetGlobalLevel(zerolog.DebugLevel) 42 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 43 | } 44 | 45 | func TestWritesOnPrimary(t *testing.T) { 46 | t.Parallel() 47 | ctx := context.Background() 48 | 49 | container := newDB(ctx, t) 50 | upstream := newSQLConn(ctx, t, container) 51 | cfg := defaultConfig(ctx, t, container) 52 | local := newSQLiteConn(ctx, t, cfg) 53 | 54 | wg := sync.WaitGroup{} 55 | wg.Add(1) 56 | 57 | ctx, cancel := context.WithCancel(ctx) 58 | 59 | // start sqledge replication 60 | go func() { 61 | defer wg.Done() 62 | if err := replicate.Run(ctx, cfg); err != nil && !errors.Is(err, context.Canceled) { 63 | assert.NoError(t, err) 64 | } 65 | }() 66 | 67 | execStatements( 68 | t, 69 | upstream, 70 | "CREATE TABLE names (id serial not null primary key, name text);", 71 | "INSERT INTO names (name) VALUES ('hello'), ('world')", 72 | ) 73 | 74 | // TODO: make this backoff retry 75 | <-time.After(2 * time.Second) 76 | 77 | got := readAllNameRows(t, local) 78 | 79 | want := []nameRow{ 80 | {id: 1, name: "hello"}, 81 | {id: 2, name: "world"}, 82 | } 83 | 84 | assert.Equal(t, want, got) 85 | 86 | // cleanup 87 | cancel() 88 | wg.Wait() 89 | } 90 | 91 | func TestInitialCopy(t *testing.T) { 92 | t.Parallel() 93 | ctx := context.Background() 94 | 95 | container := newDB(ctx, t) 96 | upstream := newSQLConn(ctx, t, container) 97 | cfg := defaultConfig(ctx, t, container) 98 | local := newSQLiteConn(ctx, t, cfg) 99 | 100 | wg := sync.WaitGroup{} 101 | wg.Add(1) 102 | 103 | ctx, cancel := context.WithCancel(ctx) 104 | 105 | execStatements( 106 | t, 107 | upstream, 108 | "CREATE TABLE names (id serial not null primary key, name text);", 109 | "INSERT INTO names (name) VALUES ('hello'), ('world')", 110 | ) 111 | 112 | // start sqledge replication 113 | go func() { 114 | defer wg.Done() 115 | if err := replicate.Run(ctx, cfg); err != nil && !errors.Is(err, context.Canceled) { 116 | assert.NoError(t, err) 117 | } 118 | }() 119 | 120 | // TODO: make this backoff retry 121 | <-time.After(2 * time.Second) 122 | 123 | got := readAllNameRows(t, local) 124 | 125 | want := []nameRow{ 126 | {id: 1, name: "hello"}, 127 | {id: 2, name: "world"}, 128 | } 129 | 130 | assert.Equal(t, want, got) 131 | 132 | // cleanup 133 | cancel() 134 | wg.Wait() 135 | } 136 | 137 | func TestWriteForwarding(t *testing.T) { 138 | t.Parallel() 139 | ctx := context.Background() 140 | 141 | container := newDB(ctx, t) 142 | upstream := newSQLConn(ctx, t, container) 143 | cfg := defaultConfig(ctx, t, container) 144 | local := newSQLiteConn(ctx, t, cfg) 145 | 146 | assert.NoError(t, upstream.Ping()) 147 | 148 | wg := sync.WaitGroup{} 149 | wg.Add(1) 150 | 151 | ctx, cancel := context.WithCancel(ctx) 152 | 153 | // start sqledge replication 154 | go func() { 155 | defer wg.Done() 156 | if err := replicate.Run(ctx, cfg); err != nil && !errors.Is(err, context.Canceled) { 157 | assert.NoError(t, err) 158 | } 159 | }() 160 | 161 | if err := queryproxy.Run(ctx, cfg); err != nil && !errors.Is(err, context.Canceled) { 162 | assert.NoError(t, err) 163 | } 164 | 165 | <-time.After(1 * time.Second) 166 | t.Log("connecting to proxy") 167 | 168 | proxyConnStr := fmt.Sprintf( 169 | "user=postgres host=0.0.0.0 port=%d database=%s sslmode=disable", 170 | cfg.Proxy.Port, 171 | cfg.Upstream.DBName, 172 | ) 173 | 174 | db, err := sql.Open("pgx", proxyConnStr) 175 | assert.NoError(t, err) 176 | 177 | t.Log("connected to proxy") 178 | 179 | execStatements( 180 | t, 181 | db, 182 | "CREATE TABLE names (id serial not null primary key, name text);", 183 | "INSERT INTO names (name) VALUES ('hello'), ('world')", 184 | ) 185 | 186 | t.Log("executed statements") 187 | 188 | want := []nameRow{ 189 | {id: 1, name: "hello"}, 190 | {id: 2, name: "world"}, 191 | } 192 | 193 | // assert rows are in upstream 194 | gotUpstream := readAllNameRows(t, upstream) 195 | assert.Equal(t, want, gotUpstream) 196 | 197 | // TODO: make this backoff retry 198 | 199 | // assert rows are in local 200 | got := readAllNameRows(t, local) 201 | assert.Equal(t, want, got) 202 | 203 | // cleanup 204 | cancel() 205 | wg.Wait() 206 | } 207 | 208 | func newDB(ctx context.Context, t *testing.T) *postgres.PostgresContainer { 209 | pgContainer, err := postgres.RunContainer(ctx, 210 | testcontainers.WithImage("postgres:15.3-alpine"), 211 | postgres.WithDatabase(dbName), 212 | postgres.WithUsername(userName), 213 | postgres.WithPassword(password), 214 | postgres.WithConfigFile(defaultPostgresConfigFile(t)), 215 | testcontainers.WithWaitStrategy( 216 | wait.ForLog("database system is ready to accept connections"). 217 | WithOccurrence(2).WithStartupTimeout(5*time.Second)), 218 | ) 219 | if err != nil { 220 | t.Fatal(err) 221 | } 222 | 223 | t.Cleanup(func() { 224 | if err := pgContainer.Terminate(ctx); err != nil { 225 | t.Fatalf("failed to terminate pgContainer: %s", err) 226 | } 227 | }) 228 | 229 | return pgContainer 230 | } 231 | 232 | func newConn(ctx context.Context, t *testing.T, container *postgres.PostgresContainer) *pgconn.PgConn { 233 | connStr, err := container.ConnectionString(ctx, "sslmode=disable") 234 | assert.NoError(t, err) 235 | 236 | conn, err := pgconn.Connect(context.Background(), connStr) 237 | assert.NoError(t, err) 238 | 239 | return conn 240 | } 241 | 242 | func newSQLConn(ctx context.Context, t *testing.T, container *postgres.PostgresContainer) *sql.DB { 243 | connStr, err := container.ConnectionString(ctx, "sslmode=disable") 244 | assert.NoError(t, err) 245 | 246 | connStr = strings.ReplaceAll(connStr, "host=localhost", "host=0.0.0.0") 247 | 248 | db, err := sql.Open("pgx", connStr) 249 | assert.NoError(t, err) 250 | 251 | return db 252 | } 253 | 254 | func newSQLiteConn(ctx context.Context, t *testing.T, cfg *config.Config) *sql.DB { 255 | db, err := sql.Open("sqlite3", cfg.Local.Path) 256 | assert.NoError(t, err) 257 | 258 | return db 259 | } 260 | 261 | func defaultConfig(ctx context.Context, t *testing.T, container *postgres.PostgresContainer) *config.Config { 262 | cfg := config.Config{} 263 | 264 | port, _ := container.MappedPort(ctx, "5432/tcp") 265 | 266 | cfg.Upstream.User = userName 267 | cfg.Upstream.Pass = password 268 | cfg.Upstream.DBName = dbName 269 | cfg.Upstream.Schema = "public" 270 | cfg.Upstream.Address = "0.0.0.0" 271 | cfg.Upstream.Port = port.Int() 272 | 273 | cfg.Replication.Publication = "test_publication" 274 | cfg.Replication.Plugin = "pgoutput" 275 | cfg.Replication.SlotName = "sqledge_test_slot" 276 | cfg.Replication.CreateSlotIfNoExists = true 277 | cfg.Replication.Temporary = true 278 | 279 | f, err := os.CreateTemp(os.TempDir(), "sqledge-*.db") 280 | assert.NoError(t, err) 281 | 282 | cfg.Local.Path = f.Name() 283 | 284 | cfg.Proxy.Address = "localhost" 285 | cfg.Proxy.Port = rand.Intn(100) + 5433 286 | 287 | return &cfg 288 | } 289 | 290 | func defaultPostgresConfigFile(t *testing.T) string { 291 | content := ` 292 | listen_addresses = '*' 293 | max_connections = 100 # (change requires restart) 294 | shared_buffers = 128MB # min 128kB 295 | dynamic_shared_memory_type = posix # the default is usually the first option 296 | log_timezone = 'Etc/UTC' 297 | datestyle = 'iso, mdy' 298 | timezone = 'Etc/UTC' 299 | lc_messages = 'en_US.utf8' # locale for system error message 300 | lc_monetary = 'en_US.utf8' # locale for monetary formatting 301 | lc_numeric = 'en_US.utf8' # locale for number formatting 302 | lc_time = 'en_US.utf8' # locale for time formatting 303 | default_text_search_config = 'pg_catalog.english' 304 | 305 | wal_level = logical # minimal, replica, or logical 306 | max_wal_size = 1GB 307 | min_wal_size = 80MB 308 | max_wal_senders = 5 # max number of walsender processes 309 | max_replication_slots = 5 # max number of replication slots 310 | ` 311 | 312 | f, err := os.CreateTemp(os.TempDir(), "postgres-config-*.conf") 313 | assert.NoError(t, err) 314 | 315 | f.WriteString(content) 316 | f.Close() 317 | 318 | return f.Name() 319 | } 320 | 321 | func execStatements(t *testing.T, db *sql.DB, statements ...string) { 322 | for _, stmt := range statements { 323 | _, err := db.Exec(stmt) 324 | assert.NoError(t, err) 325 | t.Log("executed: " + stmt) 326 | } 327 | } 328 | 329 | func readAllNameRows(t *testing.T, db *sql.DB) []nameRow { 330 | rows, err := db.Query(`SELECT * FROM names;`) 331 | assert.NoError(t, err) 332 | 333 | var out []nameRow 334 | 335 | for rows.Next() { 336 | n := nameRow{} 337 | 338 | assert.NoError(t, rows.Scan(&(n.id), &(n.name))) 339 | out = append(out, n) 340 | } 341 | 342 | return out 343 | } 344 | -------------------------------------------------------------------------------- /pkg/pgwire/postgres.go: -------------------------------------------------------------------------------- 1 | package pgwire 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/binary" 6 | "fmt" 7 | "io" 8 | "net" 9 | "regexp" 10 | "strings" 11 | 12 | "github.com/jackc/pgx/v5/pgproto3" 13 | "github.com/rs/zerolog/log" 14 | "github.com/zknill/sqledge/pkg/sqlgen" 15 | ) 16 | 17 | // magic numbers come from here: 18 | // - https://www.postgresql.org/docs/15/protocol-message-formats.html 19 | const ( 20 | SSLRequest = 80877103 21 | StartupMessage = 196608 22 | 23 | AuthenticationOk = 'R' 24 | BackendKeyData = 'K' 25 | ReadyForQuery = 'Z' 26 | SimpleQuery = 'Q' 27 | Exit = 'X' 28 | ) 29 | 30 | var withStatement = regexp.MustCompile(`with .* as (.*) select`) 31 | 32 | func Handle(schema string, upstream, local *sql.DB, conn net.Conn) { 33 | if err := onStart(conn); err != nil { 34 | log.Error().Err(err).Msg("on start error") 35 | } 36 | 37 | log.Debug().Msg("completed startup") 38 | 39 | for { 40 | b := make([]byte, 5) 41 | 42 | if _, err := conn.Read(b); err != nil { 43 | log.Error().Err(err).Msg("read initial") 44 | return 45 | } 46 | 47 | switch b[0] { 48 | case SimpleQuery: 49 | case Exit: 50 | conn.Close() 51 | return 52 | default: 53 | log.Error().Msgf("unknown message type: %q", string(b[0])) 54 | return 55 | } 56 | 57 | l := binary.BigEndian.Uint32(b[1:5]) - 4 58 | 59 | body := make([]byte, l) 60 | 61 | if _, err := conn.Read(body); err != nil { 62 | log.Error().Err(err).Msg("read query body") 63 | return 64 | } 65 | 66 | query := strings.ToLower(string(body[:len(body)-1])) 67 | 68 | switch { 69 | case strings.HasPrefix(query, "select") || withStatement.MatchString(query): 70 | log.Debug().Msgf("querying: %q", string(query)) 71 | 72 | rows, err := local.Query(query) 73 | if err != nil { 74 | log.Error().Err(err).Msg("local query") 75 | 76 | errReadyForQuery(fmt.Errorf("failed to query local: %w", err), conn) 77 | 78 | continue 79 | } 80 | 81 | desc := rowDesc(rows) 82 | out := desc.Encode(nil) 83 | 84 | data := rowData(rows) 85 | for _, row := range data { 86 | out = row.Encode(out) 87 | } 88 | 89 | log.Debug().Msgf("found %d rows", len(data)) 90 | 91 | cmd := &pgproto3.CommandComplete{CommandTag: []byte("")} 92 | out = cmd.Encode(out) 93 | 94 | ready := &pgproto3.ReadyForQuery{TxStatus: 'I'} 95 | out = ready.Encode(out) 96 | 97 | if _, err := conn.Write(out); err != nil { 98 | log.Error().Err(err).Msg("write response") 99 | 100 | continue 101 | } 102 | case strings.HasPrefix(query, "update"): 103 | r, err := upstream.Exec(query) 104 | if err != nil { 105 | errReadyForQuery(fmt.Errorf("failed to query upstream: %w", err), conn) 106 | 107 | continue 108 | } 109 | 110 | updates, err := r.RowsAffected() 111 | cmd := pgproto3.CommandComplete{CommandTag: []byte("UPDATE " + fmt.Sprintf("%d", updates))} 112 | 113 | out := cmd.Encode(nil) 114 | ready := &pgproto3.ReadyForQuery{TxStatus: 'I'} 115 | out = ready.Encode(out) 116 | 117 | if _, err := conn.Write(out); err != nil { 118 | log.Error().Err(err).Msg("write response") 119 | 120 | continue 121 | } 122 | case strings.HasPrefix(query, "insert"): 123 | r, err := upstream.Exec(query) 124 | if err != nil { 125 | errReadyForQuery(fmt.Errorf("failed to query upstream: %w", err), conn) 126 | 127 | continue 128 | } 129 | 130 | updates, err := r.RowsAffected() 131 | cmd := pgproto3.CommandComplete{CommandTag: []byte("INSERT 0 " + fmt.Sprintf("%d", updates))} 132 | 133 | out := cmd.Encode(nil) 134 | ready := &pgproto3.ReadyForQuery{TxStatus: 'I'} 135 | out = ready.Encode(out) 136 | 137 | if _, err := conn.Write(out); err != nil { 138 | log.Error().Err(err).Msg("write response") 139 | 140 | continue 141 | } 142 | case strings.HasPrefix(query, "delete"): 143 | r, err := upstream.Exec(query) 144 | if err != nil { 145 | errReadyForQuery(fmt.Errorf("failed to query upstream: %w", err), conn) 146 | 147 | continue 148 | } 149 | 150 | updates, err := r.RowsAffected() 151 | cmd := pgproto3.CommandComplete{CommandTag: []byte("DELETE " + fmt.Sprintf("%d", updates))} 152 | 153 | out := cmd.Encode(nil) 154 | ready := &pgproto3.ReadyForQuery{TxStatus: 'I'} 155 | out = ready.Encode(out) 156 | 157 | if _, err := conn.Write(out); err != nil { 158 | log.Error().Err(err).Msg("write response") 159 | 160 | continue 161 | } 162 | case strings.HasPrefix(query, "create table"): 163 | log.Debug().Msgf("handle create table: %q", query) 164 | _, err := upstream.Exec(query) 165 | if err != nil { 166 | errReadyForQuery(fmt.Errorf("failed to query upstream: %w", err), conn) 167 | 168 | continue 169 | } 170 | log.Debug().Msgf("upstream for create table: %q", query) 171 | 172 | cmd := pgproto3.CommandComplete{CommandTag: []byte("CREATE TABLE")} 173 | out := cmd.Encode(nil) 174 | 175 | ready := &pgproto3.ReadyForQuery{TxStatus: 'I'} 176 | out = ready.Encode(out) 177 | 178 | if _, err := conn.Write(out); err != nil { 179 | log.Error().Err(err).Msg("write response") 180 | 181 | continue 182 | } 183 | log.Debug().Msgf("success create table: %q", query) 184 | case strings.HasPrefix(query, "delete table"): 185 | _, err := upstream.Exec(query) 186 | if err != nil { 187 | errReadyForQuery(fmt.Errorf("failed to query upstream: %w", err), conn) 188 | 189 | continue 190 | } 191 | 192 | cmd := pgproto3.CommandComplete{CommandTag: []byte("DELETE TABLE")} 193 | out := cmd.Encode(nil) 194 | 195 | ready := &pgproto3.ReadyForQuery{TxStatus: 'I'} 196 | out = ready.Encode(out) 197 | 198 | if _, err := conn.Write(out); err != nil { 199 | log.Error().Err(err).Msg("write response") 200 | 201 | continue 202 | } 203 | case strings.HasPrefix(query, "alter table"): 204 | _, err := upstream.Exec(query) 205 | if err != nil { 206 | errReadyForQuery(fmt.Errorf("failed to query upstream: %w", err), conn) 207 | 208 | continue 209 | } 210 | 211 | cmd := pgproto3.CommandComplete{CommandTag: []byte("ALTER TABLE")} 212 | out := cmd.Encode(nil) 213 | 214 | ready := &pgproto3.ReadyForQuery{TxStatus: 'I'} 215 | out = ready.Encode(out) 216 | 217 | if _, err := conn.Write(out); err != nil { 218 | log.Error().Err(err).Msg("write response") 219 | 220 | continue 221 | } 222 | default: 223 | // this covers all unknown queries 224 | errReadyForQuery(fmt.Errorf("unknown query type: %q", query), conn) 225 | 226 | continue 227 | } 228 | 229 | } 230 | 231 | } 232 | 233 | // Eventually this method should parse the connection 234 | // details and connect to the upstream database using them. 235 | func onStart(conn net.Conn) error { 236 | readBuf := make([]byte, 4) 237 | 238 | if _, err := conn.Read(readBuf); err != nil { 239 | return fmt.Errorf("read msg len: %w", err) 240 | } 241 | 242 | l := binary.BigEndian.Uint32(readBuf) - 4 243 | 244 | if l < 4 || l > 10000 { 245 | return fmt.Errorf("invalid msg len: %d", l) 246 | } 247 | 248 | b := make([]byte, l) 249 | 250 | if _, err := conn.Read(b); err != nil { 251 | return fmt.Errorf("read msg: %w", err) 252 | } 253 | 254 | log.Debug().Msgf("startup message size: %d", l) 255 | 256 | msgType := binary.BigEndian.Uint32(b) 257 | 258 | switch msgType { 259 | case SSLRequest: 260 | conn.Write([]byte{'N'}) 261 | return onStart(conn) 262 | 263 | case StartupMessage: 264 | // AuthenticationOk 265 | { 266 | success := uint32(0) 267 | l := uint32(8) 268 | d := make([]byte, l) 269 | 270 | binary.BigEndian.PutUint32(d[0:4], l) 271 | binary.BigEndian.PutUint32(d[4:], success) 272 | 273 | conn.Write([]byte{AuthenticationOk}) 274 | conn.Write(d) 275 | 276 | log.Debug().Msgf("auth ok: len: %d, %s", l, string(AuthenticationOk)) 277 | } 278 | 279 | // BackendKeyData 280 | { 281 | l := uint32(12) 282 | id := uint32(1234) 283 | secret := uint32(5678) 284 | 285 | d := make([]byte, l) 286 | binary.BigEndian.PutUint32(d[0:4], l) 287 | binary.BigEndian.PutUint32(d[4:8], id) 288 | binary.BigEndian.PutUint32(d[8:], secret) 289 | 290 | conn.Write([]byte{BackendKeyData}) 291 | conn.Write(d) 292 | 293 | log.Debug().Msgf("backend key data: len: %d, id: %d, secret: %d, %s", l, id, secret, string(BackendKeyData)) 294 | } 295 | 296 | // ReadyForQuery 297 | { 298 | l := uint32(5) 299 | d := make([]byte, l) 300 | 301 | binary.BigEndian.PutUint32(d[0:4], l) 302 | d[4] = 'I' 303 | 304 | conn.Write([]byte{ReadyForQuery}) 305 | conn.Write(d) 306 | 307 | log.Debug().Msgf("ready for query: len: %d %s", l, string(ReadyForQuery)) 308 | } 309 | } 310 | 311 | return nil 312 | } 313 | 314 | func rowData(rows *sql.Rows) []*pgproto3.DataRow { 315 | cols, err := rows.Columns() 316 | if err != nil { 317 | log.Error().Err(err).Msg("columns") 318 | 319 | return nil 320 | } 321 | 322 | data := []*pgproto3.DataRow{} 323 | 324 | for rows.Next() { 325 | row := make([][]byte, len(cols)) 326 | dsts := make([]any, len(cols)) 327 | 328 | for i := range row { 329 | dsts[i] = &row[i] 330 | } 331 | 332 | if err := rows.Scan(dsts...); err != nil { 333 | log.Error().Err(err).Msg("row scan") 334 | continue 335 | } 336 | 337 | data = append(data, &pgproto3.DataRow{Values: row}) 338 | } 339 | 340 | return data 341 | } 342 | 343 | func rowDesc(rows *sql.Rows) *pgproto3.RowDescription { 344 | // TODO error handling 345 | types, err := rows.ColumnTypes() 346 | if err != nil { 347 | log.Error().Err(err).Msg("column types") 348 | 349 | return nil 350 | } 351 | 352 | cols, err := rows.Columns() 353 | if err != nil { 354 | log.Error().Err(err).Msg("columns") 355 | 356 | return nil 357 | } 358 | 359 | rowDesc := &pgproto3.RowDescription{} 360 | 361 | for i, c := range cols { 362 | oid, size := sqlgen.ColType(strings.ToLower(types[i].DatabaseTypeName())).PgType() 363 | 364 | rowDesc.Fields = append(rowDesc.Fields, pgproto3.FieldDescription{ 365 | Name: []byte(c), 366 | DataTypeOID: uint32(oid), 367 | DataTypeSize: int16(size), 368 | TypeModifier: -1, 369 | }) 370 | } 371 | 372 | return rowDesc 373 | } 374 | 375 | func errReadyForQuery(err error, w io.Writer) { 376 | log.Error().Err(err).Msg("error in pgwire") 377 | errResponse := pgproto3.ErrorResponse{Message: err.Error()} 378 | out := errResponse.Encode(nil) 379 | 380 | ready := pgproto3.ReadyForQuery{TxStatus: 'I'} 381 | out = ready.Encode(out) 382 | 383 | w.Write(out) 384 | } 385 | -------------------------------------------------------------------------------- /pkg/sqlgen/sqlite.go: -------------------------------------------------------------------------------- 1 | package sqlgen 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "strings" 8 | 9 | "github.com/jackc/pglogrepl" 10 | "github.com/jackc/pgx/v5/pgtype" 11 | ) 12 | 13 | type SqliteConfig struct { 14 | SourceDB string 15 | Plugin string 16 | Publication string 17 | } 18 | 19 | type Sqlite struct { 20 | typeMap *pgtype.Map 21 | relations map[uint32]*pglogrepl.RelationMessageV2 22 | // map[table_name]map[column_name]column_type 23 | current map[string]map[string]ColDef 24 | 25 | cfg SqliteConfig 26 | 27 | // TODO: move these to the parent 28 | // tx bool 29 | pos pglogrepl.LSN 30 | } 31 | 32 | func NewSqlite(cfg SqliteConfig, current map[string]map[string]ColDef) *Sqlite { 33 | s := &Sqlite{ 34 | typeMap: pgtype.NewMap(), 35 | relations: make(map[uint32]*pglogrepl.RelationMessageV2), 36 | current: current, 37 | cfg: cfg, 38 | } 39 | 40 | return s 41 | } 42 | 43 | // SQLite data types: 44 | // NULL. The value is a NULL value. 45 | // INTEGER. The value is a signed integer, 46 | // stored in 0, 1, 2, 3, 4, 6, or 47 | // 8 bytes depending on the magnitude 48 | // of the value. 49 | // REAL. The value is a floating point value, 50 | // stored as an 8-byte IEEE floating 51 | // point number. 52 | // TEXT. The value is a text string, stored 53 | // using the database encoding (UTF-8, 54 | // UTF-16BE or UTF-16LE). 55 | // BLOB. The value is a blob of data, stored 56 | // exactly as it was input. 57 | 58 | // map of postgres to sqltypes 59 | var mappedSqLiteTypes = map[ColType]ColType{ 60 | PgColTypeText: SQLiteColTypeText, 61 | PgColTypeInt2: SQLiteColTypeInteger, 62 | PgColTypeInt4: SQLiteColTypeInteger, 63 | PgColTypeInt8: SQLiteColTypeInteger, 64 | PgColTypeNum: SQLiteColTypeReal, 65 | PgColTypeFloat4: SQLiteColTypeReal, 66 | PgColTypeFloat8: SQLiteColTypeReal, 67 | PgColTypeBytea: SQLiteColTypeBlob, 68 | PgColTypeJson: SQLiteColTypeText, 69 | PgColTypeJsonB: SQLiteColTypeText, 70 | PgColTypeBool: SQLiteColTypeText, 71 | } 72 | 73 | func (s *Sqlite) Relation(msg *pglogrepl.RelationMessageV2) (string, error) { 74 | s.relations[msg.RelationID] = msg 75 | 76 | ccols, exists := s.current[msg.RelationName] 77 | if !exists { 78 | // TODO (bug): tables created in the initial copy 79 | // aren't updated in this objects column definitions 80 | // CREATE TABLE 81 | // doesn't exist as current table 82 | currentCols := map[string]ColDef{} 83 | 84 | buf := &bytes.Buffer{} 85 | pk := []string{} 86 | 87 | for idx, col := range msg.Columns { 88 | dt, ok := s.typeMap.TypeForOID(col.DataType) 89 | if !ok { 90 | return "", errors.New("unknown type") 91 | } 92 | 93 | mappedType := SQLiteColTypeText 94 | 95 | if mt, ok := mappedSqLiteTypes[ColType(dt.Name)]; ok { 96 | mappedType = mt 97 | } 98 | 99 | cd := ColDef{ 100 | Type: mappedType, 101 | } 102 | 103 | if col.Flags == 1 { 104 | pk = append(pk, col.Name) 105 | cd.PrimaryKey = true 106 | } 107 | 108 | fmt.Fprintf(buf, "%s %s", col.Name, mappedType) 109 | 110 | if idx < len(msg.Columns)-1 { 111 | buf.WriteString(", ") 112 | } 113 | 114 | currentCols[col.Name] = cd 115 | } 116 | 117 | var pks string 118 | 119 | if len(pk) != 0 { 120 | pks = ", PRIMARY KEY (" + strings.Join(pk, ", ") + ") " 121 | } 122 | 123 | s.current[msg.RelationName] = currentCols 124 | 125 | return fmt.Sprintf( 126 | "CREATE TABLE IF NOT EXISTS %s (%s%s);", 127 | msg.RelationName, 128 | buf.String(), 129 | pks, 130 | ), nil 131 | } 132 | 133 | // ALTER TABLE 134 | statements := []string{} 135 | 136 | colsCovered := map[string]ColDef{} 137 | 138 | for k, v := range ccols { 139 | v := v 140 | colsCovered[k] = v 141 | } 142 | 143 | for _, col := range msg.Columns { 144 | delete(colsCovered, col.Name) 145 | 146 | dt, ok := s.typeMap.TypeForOID(col.DataType) 147 | if !ok { 148 | return "", errors.New("unknown type") 149 | } 150 | 151 | mappedType := SQLiteColTypeText 152 | 153 | if mt, ok := mappedSqLiteTypes[ColType(dt.Name)]; ok { 154 | mappedType = mt 155 | } 156 | 157 | ccol, ok := ccols[col.Name] 158 | if !ok { 159 | statements = append(statements, fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s;", msg.RelationName, col.Name, mappedType)) 160 | ccols[col.Name] = ColDef{ 161 | Name: col.Name, 162 | Type: mappedType, 163 | } 164 | continue 165 | } 166 | 167 | pk := col.Flags == 1 168 | 169 | if ccol.PrimaryKey && !pk { 170 | // TODO: DROP PK 171 | } 172 | 173 | if pk && !ccol.PrimaryKey { 174 | // TODO: ADD PK 175 | } 176 | 177 | if ccol.Type != mappedType { 178 | // TODO: Change col type, not supported in SQLite 179 | } 180 | } 181 | 182 | for k, v := range colsCovered { 183 | if v.PrimaryKey { 184 | // dropping PK cols not supported in sqlite 185 | continue 186 | } 187 | 188 | statements = append(statements, fmt.Sprintf("ALTER TABLE %s DROP COLUMN %s;", msg.RelationName, k)) 189 | } 190 | 191 | return strings.Join(statements, " "), nil 192 | } 193 | 194 | // Insert represents a single row insert. 195 | // Multiple VALUES (...) inserted at once 196 | // would be multiple calls to this Insert method. 197 | func (s *Sqlite) Insert(msg *pglogrepl.InsertMessageV2) (string, error) { 198 | rel, ok := s.relations[msg.RelationID] 199 | if !ok { 200 | return "", errors.New("unknown relation") 201 | } 202 | 203 | cols, err := s.parseColums(rel, msg.Tuple.Columns) 204 | if err != nil { 205 | return "", fmt.Errorf("insert: %w", err) 206 | } 207 | 208 | cBuf := &bytes.Buffer{} 209 | vBuf := &bytes.Buffer{} 210 | 211 | for idx, col := range cols { 212 | cBuf.WriteString(col.name) 213 | fmt.Fprintf(vBuf, "%v", col.val()) 214 | 215 | if idx < len(cols)-1 { 216 | cBuf.WriteString(", ") 217 | vBuf.WriteString(", ") 218 | } 219 | } 220 | 221 | return fmt.Sprintf( 222 | "INSERT INTO %s (%s) VALUES (%s);", 223 | rel.RelationName, 224 | cBuf.String(), 225 | vBuf.String(), 226 | ), nil 227 | } 228 | 229 | func (s *Sqlite) Update(msg *pglogrepl.UpdateMessageV2) (string, error) { 230 | rel, ok := s.relations[msg.RelationID] 231 | if !ok { 232 | return "", errors.New("unknown relation") 233 | } 234 | 235 | cols, err := s.parseColums(rel, msg.NewTuple.Columns) 236 | if err != nil { 237 | return "", fmt.Errorf("new: %w", err) 238 | } 239 | 240 | whereCols := cols 241 | 242 | if msg.OldTuple != nil { 243 | // what happens on delete col? 244 | whereCols, err = s.parseColums(rel, msg.OldTuple.Columns) 245 | if err != nil { 246 | return "", fmt.Errorf("old: %w", err) 247 | } 248 | } 249 | 250 | buf := &bytes.Buffer{} 251 | for _, col := range cols { 252 | if col.key && msg.OldTuple == nil { 253 | continue 254 | } 255 | 256 | fmt.Fprint(buf, col.kvSql(), ",") 257 | 258 | } 259 | 260 | kBuf := &bytes.Buffer{} 261 | 262 | for _, col := range whereCols { 263 | if !col.key { 264 | continue 265 | } 266 | 267 | fmt.Fprint(kBuf, col.kvSql(), " AND ") 268 | } 269 | 270 | return fmt.Sprintf( 271 | "UPDATE %s SET %s WHERE %s;", 272 | rel.RelationName, 273 | buf.String()[:len(buf.String())-1], 274 | kBuf.String()[:len(kBuf.String())-5], 275 | ), nil 276 | } 277 | 278 | func (s *Sqlite) Delete(msg *pglogrepl.DeleteMessageV2) (string, error) { 279 | rel, ok := s.relations[msg.RelationID] 280 | if !ok { 281 | return "", errors.New("unknown relation") 282 | } 283 | 284 | cols, err := s.parseColums(rel, msg.OldTuple.Columns) 285 | if err != nil { 286 | return "", fmt.Errorf("new: %w", err) 287 | } 288 | 289 | kBuf := &bytes.Buffer{} 290 | 291 | for _, col := range cols { 292 | if !col.key { 293 | continue 294 | } 295 | 296 | fmt.Fprint(kBuf, col.kvSql(), " AND ") 297 | } 298 | 299 | return fmt.Sprintf( 300 | "DELETE FROM %s WHERE %s;", 301 | rel.RelationName, 302 | kBuf.String()[:len(kBuf.String())-5], 303 | ), nil 304 | } 305 | 306 | func (s *Sqlite) Truncate(msg *pglogrepl.TruncateMessageV2) (string, error) { 307 | buf := &bytes.Buffer{} 308 | 309 | for _, id := range msg.RelationIDs { 310 | rel, ok := s.relations[id] 311 | if !ok { 312 | return "", errors.New("unknown relation") 313 | } 314 | 315 | fmt.Fprintf(buf, "DELETE FROM %s; ", rel.RelationName) 316 | } 317 | 318 | return buf.String(), nil 319 | } 320 | 321 | func (s *Sqlite) Begin(msg *pglogrepl.BeginMessage) (string, error) { 322 | s.pos = msg.FinalLSN 323 | return "BEGIN TRANSACTION;", nil 324 | } 325 | 326 | func (s *Sqlite) StreamStart(msg *pglogrepl.StreamStartMessageV2) (string, error) { 327 | return "BEGIN TRANSACTION;", nil 328 | } 329 | 330 | func (s *Sqlite) StreamStop(msg *pglogrepl.StreamStopMessageV2) (string, error) { 331 | return "COMMIT;", nil 332 | } 333 | 334 | func (s *Sqlite) StreamCommit(msg *pglogrepl.StreamCommitMessageV2) (string, error) { 335 | return "COMMIT;", nil 336 | } 337 | 338 | func (s *Sqlite) StreamAbort(msg *pglogrepl.StreamAbortMessageV2) (string, error) { 339 | return "ROLLBACK;", nil 340 | } 341 | 342 | func (s *Sqlite) Commit(_ *pglogrepl.CommitMessage) (string, error) { 343 | return fmt.Sprintf( 344 | "INSERT OR REPLACE INTO postgres_pos (source_db, plugin, publication, pos) VALUES ('%s', '%s', '%s', '%s');\n COMMIT;", 345 | s.cfg.SourceDB, s.cfg.Plugin, s.cfg.Publication, s.pos, 346 | ), nil 347 | } 348 | 349 | func (s *Sqlite) Pos(p string) string { 350 | s.pos, _ = pglogrepl.ParseLSN(p) 351 | 352 | return fmt.Sprintf( 353 | "INSERT OR REPLACE INTO postgres_pos (source_db, plugin, publication, pos) VALUES ('%s', '%s', '%s', '%s');", 354 | s.cfg.SourceDB, s.cfg.Plugin, s.cfg.Publication, s.pos, 355 | ) 356 | } 357 | 358 | func (s *Sqlite) CopyCreateTable(schema, tableName string, colDefs []ColDef) (string, error) { 359 | query := `CREATE TABLE IF NOT EXISTS ` + tableName + ` ( ` 360 | 361 | for i, col := range colDefs { 362 | 363 | mt := SQLiteColTypeText 364 | 365 | if t, ok := mappedSqLiteTypes[col.Type]; ok && !col.Array { 366 | mt = t 367 | } 368 | 369 | query += fmt.Sprintf("%s %s", col.Name, mt) 370 | if i < len(colDefs)-1 { 371 | query += ", " 372 | } 373 | } 374 | 375 | query += ");" 376 | 377 | return query, nil 378 | } 379 | 380 | func (s *Sqlite) InsertCopyRow(schema, tableName string, colDefs []ColDef, rowValues []string) (string, error) { 381 | query := `INSERT INTO %s VALUES ( %s );` 382 | 383 | var row string 384 | for i, v := range rowValues { 385 | if v == "null" { 386 | row += "null" 387 | } else { 388 | row += "'" + v + "'" 389 | } 390 | 391 | if i < len(rowValues)-1 { 392 | row += "," 393 | } 394 | } 395 | 396 | return fmt.Sprintf(query, tableName, row), nil 397 | } 398 | 399 | type column struct { 400 | name string 401 | value interface{} 402 | binary []byte 403 | key bool 404 | } 405 | 406 | func (c *column) kvSql() string { 407 | return c.name + "=" + c.val() 408 | } 409 | 410 | func (c *column) val() string { 411 | if c.value == "null" { 412 | return "null" 413 | } 414 | 415 | if c.binary != nil { 416 | return fmt.Sprintf("x'%v'", c.binary) 417 | } 418 | 419 | return fmt.Sprintf("'%v'", c.value) 420 | } 421 | 422 | func (s *Sqlite) parseColums(rel *pglogrepl.RelationMessageV2, cols []*pglogrepl.TupleDataColumn) ([]*column, error) { 423 | out := make([]*column, len(cols)) 424 | 425 | for idx, col := range cols { 426 | switch col.DataType { 427 | case 'n': 428 | out[idx] = &column{ 429 | name: rel.Columns[idx].Name, 430 | value: "null", 431 | key: rel.Columns[idx].Flags == 1, 432 | } 433 | case 'u': 434 | // unchanged 435 | case 't': 436 | data := col.Data 437 | 438 | out[idx] = &column{ 439 | name: rel.Columns[idx].Name, 440 | value: string(data), 441 | key: rel.Columns[idx].Flags == 1, 442 | } 443 | case 'b': 444 | out[idx] = &column{ 445 | name: rel.Columns[idx].Name, 446 | binary: col.Data, 447 | key: rel.Columns[idx].Flags == 1, 448 | } 449 | } 450 | } 451 | 452 | return out, nil 453 | } 454 | -------------------------------------------------------------------------------- /pkg/replicate/replicate.go: -------------------------------------------------------------------------------- 1 | package replicate 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "strings" 10 | "time" 11 | 12 | "github.com/jackc/pglogrepl" 13 | "github.com/jackc/pgx/v5/pgconn" 14 | "github.com/jackc/pgx/v5/pgproto3" 15 | _ "github.com/jackc/pgx/v5/stdlib" 16 | "github.com/rs/zerolog" 17 | "github.com/rs/zerolog/log" 18 | "github.com/zknill/sqledge/pkg/sqlgen" 19 | "github.com/zknill/sqledge/pkg/tables" 20 | ) 21 | 22 | type Conn struct { 23 | publication string 24 | conn *pgconn.PgConn 25 | connStr string 26 | 27 | pos pglogrepl.LSN 28 | } 29 | 30 | func NewConn(ctx context.Context, connString, publication string) (*Conn, error) { 31 | conn, err := pgconn.Connect(context.Background(), connString) 32 | if err != nil { 33 | return nil, fmt.Errorf("pgconnect: %w", err) 34 | } 35 | 36 | c := &Conn{ 37 | publication: publication, 38 | conn: conn, 39 | connStr: connString, 40 | } 41 | 42 | if err := c.identify(); err != nil { 43 | return nil, fmt.Errorf("new conn: %w", err) 44 | } 45 | 46 | return c, nil 47 | } 48 | 49 | func (c *Conn) Close() error { 50 | return c.conn.Close(context.Background()) 51 | } 52 | 53 | func (c *Conn) DropPublication() error { 54 | // TODO: care for injection 55 | result := c.conn.Exec(context.Background(), fmt.Sprintf("DROP PUBLICATION IF EXISTS %s;", c.publication)) 56 | 57 | _, err := result.ReadAll() 58 | if err != nil { 59 | return fmt.Errorf("drop publication: %w", err) 60 | } 61 | 62 | return nil 63 | } 64 | 65 | func (c *Conn) CreatePublication() error { 66 | result := c.conn.Exec(context.Background(), fmt.Sprintf("CREATE PUBLICATION %s FOR ALL TABLES;", c.publication)) 67 | 68 | _, err := result.ReadAll() 69 | if err != nil { 70 | return fmt.Errorf("create publication: %w", err) 71 | } 72 | 73 | return nil 74 | } 75 | 76 | type SlotConfig struct { 77 | SlotName string 78 | OutputPlugin string 79 | CreateSlotIfNoExists bool 80 | Temporary bool 81 | Schema string 82 | } 83 | 84 | type DBDriver interface { 85 | Pos() (string, error) 86 | Execute(query string) error 87 | } 88 | 89 | type SQLGen interface { 90 | Relation(*pglogrepl.RelationMessageV2) (string, error) 91 | Begin(*pglogrepl.BeginMessage) (string, error) 92 | Commit(*pglogrepl.CommitMessage) (string, error) 93 | Insert(*pglogrepl.InsertMessageV2) (string, error) 94 | Update(*pglogrepl.UpdateMessageV2) (string, error) 95 | Delete(*pglogrepl.DeleteMessageV2) (string, error) 96 | Truncate(*pglogrepl.TruncateMessageV2) (string, error) 97 | StreamStart(*pglogrepl.StreamStartMessageV2) (string, error) 98 | StreamStop(*pglogrepl.StreamStopMessageV2) (string, error) 99 | StreamCommit(*pglogrepl.StreamCommitMessageV2) (string, error) 100 | StreamAbort(*pglogrepl.StreamAbortMessageV2) (string, error) 101 | 102 | Pos(p string) string 103 | CopyCreateTable(schema, tableName string, colDefs []sqlgen.ColDef) (string, error) 104 | InsertCopyRow(schema, tableName string, colDefs []sqlgen.ColDef, rowValues []string) (string, error) 105 | } 106 | 107 | func (c *Conn) Stream(ctx context.Context, cfg SlotConfig, d DBDriver, gen SQLGen) error { 108 | pos, err := d.Pos() 109 | if err != nil { 110 | return fmt.Errorf("find starting pos: %w", err) 111 | } 112 | 113 | if pos != "" { 114 | lsn, err := pglogrepl.ParseLSN(pos) 115 | switch { 116 | case err == nil: 117 | c.pos = lsn 118 | case errors.Is(err, sql.ErrNoRows): 119 | // no op 120 | case err != nil: 121 | return fmt.Errorf("parse pos: %w", err) 122 | 123 | } 124 | } 125 | 126 | slot, err := c.slot(cfg.SlotName, cfg.OutputPlugin, cfg.CreateSlotIfNoExists, cfg.Temporary, c.pos) 127 | if err != nil { 128 | return fmt.Errorf("build slot: %w", err) 129 | } 130 | 131 | if pos == "" { 132 | log.Debug().Msg("starting copy") 133 | 134 | if err := c.initialCopy(ctx, cfg.Schema, slot.startSnapshot, d, gen); err != nil { 135 | return fmt.Errorf("copy: %w", err) 136 | } 137 | 138 | log.Debug().Msg("finished copy") 139 | 140 | if err := d.Execute(gen.Pos(c.pos.String())); err != nil { 141 | return fmt.Errorf("track position after copy: %w", err) 142 | } 143 | } 144 | 145 | log.Debug().Msgf("starting slot from pos: %q", c.pos) 146 | 147 | if err := slot.start(ctx); err != nil { 148 | return fmt.Errorf("start slot: %w", err) 149 | } 150 | 151 | var ( 152 | logicalMsg pglogrepl.Message 153 | query string 154 | ) 155 | 156 | stream := slot.stream() 157 | 158 | for { 159 | select { 160 | case <-ctx.Done(): 161 | slot.close() 162 | return ctx.Err() 163 | case <-slot.errs: 164 | return fmt.Errorf("slot error: %w", err) 165 | case logicalMsg = <-stream: 166 | } 167 | 168 | switch logicalMsg := logicalMsg.(type) { 169 | case *pglogrepl.RelationMessageV2: 170 | query, err = gen.Relation(logicalMsg) 171 | case *pglogrepl.BeginMessage: 172 | query, err = gen.Begin(logicalMsg) 173 | case *pglogrepl.CommitMessage: 174 | query, err = gen.Commit(logicalMsg) 175 | case *pglogrepl.InsertMessageV2: 176 | query, err = gen.Insert(logicalMsg) 177 | case *pglogrepl.UpdateMessageV2: 178 | query, err = gen.Update(logicalMsg) 179 | case *pglogrepl.DeleteMessageV2: 180 | query, err = gen.Delete(logicalMsg) 181 | case *pglogrepl.TruncateMessageV2: 182 | query, err = gen.Truncate(logicalMsg) 183 | case *pglogrepl.TypeMessageV2: 184 | case *pglogrepl.OriginMessage: 185 | case *pglogrepl.LogicalDecodingMessageV2: 186 | log.Debug().Msgf("Logical decoding message: %q, %q, %d", logicalMsg.Prefix, logicalMsg.Content, logicalMsg.Xid) 187 | case *pglogrepl.StreamStartMessageV2: 188 | query, err = gen.StreamStart(logicalMsg) 189 | case *pglogrepl.StreamStopMessageV2: 190 | query, err = gen.StreamStop(logicalMsg) 191 | case *pglogrepl.StreamCommitMessageV2: 192 | query, err = gen.StreamCommit(logicalMsg) 193 | case *pglogrepl.StreamAbortMessageV2: 194 | query, err = gen.StreamAbort(logicalMsg) 195 | default: 196 | log.Debug().Msgf("Unknown message type in pgoutput stream: %T", logicalMsg) 197 | continue 198 | } 199 | 200 | log.Debug().Msg(query) 201 | 202 | if err != nil { 203 | return fmt.Errorf("generate sql: %w", err) 204 | } 205 | 206 | if err = d.Execute(query); err != nil { 207 | return fmt.Errorf("apply sql: %w", err) 208 | } 209 | } 210 | } 211 | 212 | func (c *Conn) slot(slotName, outputPlugin string, createSlot, temporary bool, pos pglogrepl.LSN) (*slot, error) { 213 | pluginArguments := []string{ 214 | "proto_version '2'", 215 | fmt.Sprintf("publication_names '%s'", c.publication), 216 | "messages 'true'", 217 | "streaming 'false'", 218 | } 219 | 220 | s := &slot{ 221 | conn: c.conn, 222 | args: pluginArguments, 223 | name: slotName, 224 | pos: c.pos, 225 | } 226 | 227 | // TODO: automatically work out if slot exists 228 | if createSlot { 229 | res, err := pglogrepl.CreateReplicationSlot( 230 | context.Background(), 231 | c.conn, 232 | slotName, 233 | outputPlugin, 234 | pglogrepl.CreateReplicationSlotOptions{Temporary: temporary}, 235 | ) 236 | if err != nil { 237 | return nil, fmt.Errorf("create slot: %w", err) 238 | } 239 | 240 | s.startSnapshot = res.SnapshotName 241 | } 242 | 243 | return s, nil 244 | } 245 | 246 | func (c *Conn) identify() error { 247 | sysident, err := pglogrepl.IdentifySystem(context.Background(), c.conn) 248 | if err != nil { 249 | return fmt.Errorf("identify: %w", err) 250 | } 251 | 252 | c.pos = sysident.XLogPos 253 | return nil 254 | } 255 | 256 | func tableColDefs(connStr, schema string) (map[string][]sqlgen.ColDef, error) { 257 | db, err := sql.Open("pgx", strings.Replace(connStr, "replication=database", "", 1)) 258 | if err != nil { 259 | return nil, fmt.Errorf("open connection: %w", err) 260 | } 261 | 262 | defs, err := tables.TableColDefs(db, schema, nil) 263 | if err != nil { 264 | return nil, fmt.Errorf("load col definitions: %w", err) 265 | } 266 | 267 | return defs, nil 268 | } 269 | 270 | func (c *Conn) initialCopy(ctx context.Context, schema, snapshotName string, dst DBDriver, gen SQLGen) (err error) { 271 | if schema == "" { 272 | return fmt.Errorf("cannot copy for empty schema") 273 | } 274 | 275 | defs, err := tableColDefs(c.connStr, schema) 276 | if err != nil { 277 | return fmt.Errorf("load col defs: %w", err) 278 | } 279 | 280 | copyConn, err := pgconn.Connect(context.Background(), c.connStr) 281 | if err != nil { 282 | return fmt.Errorf("pgconnect: %w", err) 283 | } 284 | 285 | query := `BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY;` 286 | if snapshotName != "" { 287 | query += fmt.Sprintf("SET TRANSACTION SNAPSHOT '%s';", snapshotName) 288 | } 289 | 290 | log.Debug().Msg(query) 291 | 292 | copyConn.Exec(ctx, query).Close() 293 | 294 | defer func() { 295 | defer copyConn.Close(ctx) 296 | 297 | if e := recover(); e != nil { 298 | copyConn.Exec(ctx, `ROLLBACK;`).Close() 299 | err = fmt.Errorf("recover: %v", e) 300 | log.Debug().Msg("ROLLBACK") 301 | } 302 | 303 | if err != nil { 304 | copyConn.Exec(ctx, `ROLLBACK;`).Close() 305 | log.Debug().Msg("ROLLBACK") 306 | } 307 | 308 | copyConn.Exec(ctx, `COMMIT;`).Close() 309 | log.Debug().Msg("COMMIT") 310 | }() 311 | 312 | for table, columns := range defs { 313 | var ( 314 | query string 315 | vals [][]string 316 | ) 317 | 318 | query, err = gen.CopyCreateTable(schema, table, columns) 319 | 320 | if err = dst.Execute(query); err != nil { 321 | return fmt.Errorf("execute inital copy: %w", err) 322 | } 323 | 324 | log.Debug().Msg(query) 325 | vals, err = tables.Copy(ctx, table, columns, copyConn) 326 | if err != nil { 327 | return fmt.Errorf("copy table: %w", err) 328 | 329 | } 330 | 331 | for _, row := range vals { 332 | query, err = gen.InsertCopyRow(schema, table, columns, row) 333 | if err != nil { 334 | return fmt.Errorf("generate sql: %w", err) 335 | } 336 | 337 | log.Debug().Msg(query) 338 | 339 | if err = dst.Execute(query); err != nil { 340 | return fmt.Errorf("execute inital copy: %w", err) 341 | } 342 | } 343 | } 344 | 345 | return nil 346 | } 347 | 348 | type slot struct { 349 | conn *pgconn.PgConn 350 | 351 | args []string 352 | name string 353 | pos pglogrepl.LSN 354 | startSnapshot string 355 | 356 | msgs chan pglogrepl.Message 357 | errs chan error 358 | done chan struct{} 359 | } 360 | 361 | func (s *slot) start(ctx context.Context) error { 362 | if s.msgs != nil { 363 | // already started 364 | return nil 365 | } 366 | 367 | err := pglogrepl.StartReplication( 368 | ctx, 369 | s.conn, 370 | s.name, 371 | s.pos, 372 | pglogrepl.StartReplicationOptions{PluginArgs: s.args}, 373 | ) 374 | if err != nil { 375 | return fmt.Errorf("start replication: %w", err) 376 | } 377 | 378 | s.msgs = make(chan pglogrepl.Message) 379 | s.errs = make(chan error) 380 | s.done = make(chan struct{}) 381 | 382 | go s.listen() 383 | 384 | return nil 385 | } 386 | 387 | func (s *slot) Errors() <-chan error { 388 | return s.errs 389 | } 390 | 391 | func (s *slot) stream() <-chan pglogrepl.Message { 392 | return s.msgs 393 | } 394 | 395 | func (s *slot) listen() { 396 | standbyMessageTimeout := time.Second * 10 397 | nextStandbyMessageDeadline := time.Now().Add(standbyMessageTimeout) 398 | 399 | inStream := false 400 | 401 | for { 402 | select { 403 | case <-s.done: 404 | return 405 | default: 406 | } 407 | 408 | if time.Now().After(nextStandbyMessageDeadline) { 409 | log.Trace().Msg("status heartbeat") 410 | err := pglogrepl.SendStandbyStatusUpdate( 411 | context.Background(), 412 | s.conn, 413 | pglogrepl.StandbyStatusUpdate{WALWritePosition: s.pos}, 414 | ) 415 | if err != nil { 416 | go s.sendErr(err) 417 | } 418 | 419 | nextStandbyMessageDeadline = time.Now().Add(standbyMessageTimeout) 420 | } 421 | 422 | ctx, cancel := context.WithDeadline(context.Background(), nextStandbyMessageDeadline) 423 | 424 | rawMsg, err := s.conn.ReceiveMessage(ctx) 425 | cancel() 426 | if err != nil { 427 | if pgconn.Timeout(err) { 428 | continue 429 | } 430 | 431 | go s.sendErr(err) 432 | } 433 | 434 | if err, ok := rawMsg.(*pgproto3.ErrorResponse); ok { 435 | go s.sendErr(fmt.Errorf("postgres wal error: %v", err)) 436 | continue 437 | } 438 | 439 | msg, ok := rawMsg.(*pgproto3.CopyData) 440 | if !ok { 441 | go s.sendErr(fmt.Errorf("unexpected message: %w", err)) 442 | continue 443 | } 444 | 445 | switch msg.Data[0] { 446 | case pglogrepl.PrimaryKeepaliveMessageByteID: 447 | pkm, err := pglogrepl.ParsePrimaryKeepaliveMessage(msg.Data[1:]) 448 | if err != nil { 449 | go s.sendErr(fmt.Errorf("keep alive parse failed: %w", err)) 450 | continue 451 | } 452 | 453 | if pkm.ReplyRequested { 454 | nextStandbyMessageDeadline = time.Time{} 455 | } 456 | 457 | case pglogrepl.XLogDataByteID: 458 | log.Trace().Msg("process logical replication") 459 | 460 | xld, err := pglogrepl.ParseXLogData(msg.Data[1:]) 461 | if err != nil { 462 | go s.sendErr(fmt.Errorf("parse xlog data failed: %w", err)) 463 | continue 464 | } 465 | 466 | logicalMsg, err := pglogrepl.ParseV2(xld.WALData, inStream) 467 | if err != nil { 468 | go s.sendErr(fmt.Errorf("parse logical replication message failed: %w", err)) 469 | continue 470 | } 471 | 472 | if _, ok := logicalMsg.(*pglogrepl.StreamStartMessageV2); ok { 473 | inStream = true 474 | } 475 | 476 | if _, ok := logicalMsg.(*pglogrepl.StreamStopMessageV2); ok { 477 | inStream = false 478 | } 479 | 480 | log.Trace().Msg("sending logical message") 481 | 482 | select { 483 | case s.msgs <- logicalMsg: 484 | case <-s.done: 485 | } 486 | 487 | s.pos = xld.WALStart + pglogrepl.LSN(len(xld.WALData)) 488 | } 489 | } 490 | } 491 | 492 | func (s *slot) close() error { 493 | close(s.done) 494 | return nil 495 | } 496 | 497 | func (s *slot) sendErr(err error) { 498 | if err == nil { 499 | return 500 | } 501 | 502 | var evt *zerolog.Event 503 | 504 | if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { 505 | evt = log.Warn() 506 | } else { 507 | evt = log.Error() 508 | } 509 | 510 | evt.Err(err).Msg("send error") 511 | 512 | select { 513 | case s.errs <- err: 514 | case <-s.done: 515 | } 516 | } 517 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= 4 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= 5 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 6 | github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= 7 | github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= 8 | github.com/Microsoft/hcsshim v0.9.7 h1:mKNHW/Xvv1aFH87Jb6ERDzXTJTLPlmzfZ28VBFD/bfg= 9 | github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= 10 | github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= 11 | github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= 12 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 13 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 14 | github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= 15 | github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= 16 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 17 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 18 | github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= 19 | github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= 20 | github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= 21 | github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= 22 | github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= 23 | github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= 24 | github.com/containerd/containerd v1.6.19 h1:F0qgQPrG0P2JPgwpxWxYavrVeXAG0ezUIB9Z/4FTUAU= 25 | github.com/containerd/containerd v1.6.19/go.mod h1:HZCDMn4v/Xl2579/MvtOC2M206i+JJ6VxFWU/NetrGY= 26 | github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= 27 | github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= 28 | github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= 29 | github.com/cpuguy83/dockercfg v0.3.1 h1:/FpZ+JaygUR/lZP2NlFI2DVfrOEMAIKP5wWEJdoYe9E= 30 | github.com/cpuguy83/dockercfg v0.3.1/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= 31 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 32 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 33 | github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= 34 | github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= 35 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 36 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 37 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 38 | github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= 39 | github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= 40 | github.com/docker/docker v23.0.5+incompatible h1:DaxtlTJjFSnLOXVNUBU1+6kXGz2lpDoEAH6QoxaSg8k= 41 | github.com/docker/docker v23.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= 42 | github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= 43 | github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= 44 | github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 45 | github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= 46 | github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 47 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 48 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 49 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 50 | github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= 51 | github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= 52 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 53 | github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= 54 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 55 | github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 56 | github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 57 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 58 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 59 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 60 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 61 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 62 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 63 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 64 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 65 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 66 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 67 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 68 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 69 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 70 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 71 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 72 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 73 | github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= 74 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 75 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 76 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 77 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 78 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 79 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 80 | github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 81 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 82 | github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 83 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 84 | github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 85 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 86 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 87 | github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= 88 | github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= 89 | github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= 90 | github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= 91 | github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= 92 | github.com/jackc/pglogrepl v0.0.0-20230630212501-5fd22a600b50 h1:88/G11oNDrFAk2kZzxLDUE1jiYkFfVYHxUyWen7Ro5c= 93 | github.com/jackc/pglogrepl v0.0.0-20230630212501-5fd22a600b50/go.mod h1:Y1HIk+uK2wXiU8vuvQh0GaSzVh+MXFn2kfKBMpn6CZg= 94 | github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= 95 | github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= 96 | github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= 97 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= 98 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= 99 | github.com/jackc/pgx/v5 v5.0.3/go.mod h1:JBbvW3Hdw77jKl9uJrEDATUZIFM2VFPzRq4RWIhkF4o= 100 | github.com/jackc/pgx/v5 v5.4.2 h1:u1gmGDwbdRUZiwisBm/Ky2M14uQyUP65bG8+20nnyrg= 101 | github.com/jackc/pgx/v5 v5.4.2/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY= 102 | github.com/jackc/puddle/v2 v2.0.0/go.mod h1:itE7ZJY8xnoo0JqJEpSMprN0f+NQkMCuEV/N9j8h0oc= 103 | github.com/joeshaw/envdecode v0.0.0-20200121155833-099f1fc765bd h1:nIzoSW6OhhppWLm4yqBwZsKJlAayUu5FGozhrF3ETSM= 104 | github.com/joeshaw/envdecode v0.0.0-20200121155833-099f1fc765bd/go.mod h1:MEQrHur0g8VplbLOv5vXmDzacSaH9Z7XhcgsSh1xciU= 105 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 106 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 107 | github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= 108 | github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 109 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 110 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 111 | github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= 112 | github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= 113 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 114 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 115 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 116 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 117 | github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= 118 | github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= 119 | github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= 120 | github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= 121 | github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= 122 | github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= 123 | github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= 124 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 125 | github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= 126 | github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= 127 | github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo= 128 | github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= 129 | github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= 130 | github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= 131 | github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= 132 | github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= 133 | github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= 134 | github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= 135 | github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= 136 | github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= 137 | github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= 138 | github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= 139 | github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= 140 | github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= 141 | github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= 142 | github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= 143 | github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= 144 | github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= 145 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 146 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 147 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 148 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 149 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 150 | github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= 151 | github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= 152 | github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= 153 | github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= 154 | github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= 155 | github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= 156 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 157 | github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= 158 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 159 | github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= 160 | github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= 161 | github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= 162 | github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 163 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 164 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 165 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 166 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 167 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 168 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 169 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 170 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 171 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 172 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 173 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 174 | github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= 175 | github.com/testcontainers/testcontainers-go v0.21.0 h1:syePAxdeTzfkap+RrJaQZpJQ/s/fsUgn11xIvHrOE9U= 176 | github.com/testcontainers/testcontainers-go v0.21.0/go.mod h1:c1ez3WVRHq7T/Aj+X3TIipFBwkBaNT5iNCY8+1b83Ng= 177 | github.com/testcontainers/testcontainers-go/modules/postgres v0.21.0 h1:rFPyTR7pPMiHcDktXwd5iZ+mA1cHH/WRa+knxBcY8wU= 178 | github.com/testcontainers/testcontainers-go/modules/postgres v0.21.0/go.mod h1:Uoia8PX1RewxkJTbeXGBK6vgMjlmRbnL/4n0EXH2Z54= 179 | github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= 180 | github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= 181 | github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= 182 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 183 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 184 | go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= 185 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 186 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 187 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 188 | golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 189 | golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= 190 | golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= 191 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 192 | golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea h1:vLCWI/yYrdEHyN2JzIzPO3aaQJHQdp89IZBA/+azVC4= 193 | golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= 194 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 195 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 196 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 197 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 198 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 199 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 200 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 201 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 202 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 203 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 204 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 205 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 206 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 207 | golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 208 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 209 | golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 210 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 211 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 212 | golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= 213 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= 214 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 215 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 216 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 217 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 218 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 219 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 220 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 221 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 222 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 223 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 224 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 225 | golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 226 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 227 | golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 228 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 229 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 230 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 231 | golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 232 | golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 233 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 234 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 235 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 236 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 237 | golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 238 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 239 | golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 240 | golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 241 | golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 242 | golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 243 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 244 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 245 | golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= 246 | golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 247 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 248 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 249 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 250 | golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 251 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 252 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 253 | golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= 254 | golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= 255 | golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= 256 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 257 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 258 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 259 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 260 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 261 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 262 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 263 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 264 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 265 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 266 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 267 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 268 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 269 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 270 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 271 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 272 | google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 273 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 274 | google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad h1:kqrS+lhvaMHCxul6sKQvKJ8nAAhlVItmZV822hYFH/U= 275 | google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= 276 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 277 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 278 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 279 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 280 | google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= 281 | google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= 282 | google.golang.org/grpc v1.47.0 h1:9n77onPX5F3qfFCqjy9dhn8PbNQsIKeVU04J9G7umt8= 283 | google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= 284 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 285 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 286 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 287 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 288 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 289 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 290 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 291 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 292 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= 293 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 294 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 295 | google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 296 | google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= 297 | google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 298 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 299 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 300 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 301 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 302 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 303 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 304 | gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 305 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 306 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 307 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 308 | gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= 309 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 310 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 311 | --------------------------------------------------------------------------------