├── .gitignore ├── ptor.png ├── model └── model.go ├── catalog └── catalog.go ├── global └── global.go ├── util └── util.go ├── Makefile ├── locks └── locks.go ├── cli └── cli.go ├── go.mod ├── pool └── pool.go ├── calc └── calc.go ├── main.go ├── README.md ├── repo └── repo.go ├── task └── task.go ├── db └── db.go └── go.sum /.gitignore: -------------------------------------------------------------------------------- 1 | ptor 2 | -------------------------------------------------------------------------------- /ptor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dineshkumar02/ptor/HEAD/ptor.png -------------------------------------------------------------------------------- /model/model.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import "time" 4 | 5 | type RelationData struct { 6 | Id int 7 | T string 8 | LastUpdate time.Time 9 | } 10 | -------------------------------------------------------------------------------- /catalog/catalog.go: -------------------------------------------------------------------------------- 1 | package catalog 2 | 3 | const SchemaSQL = `CREATE SCHEMA IF NOT EXISTS ptor;` 4 | const TableSQL = `CREATE TABLE ptor.worker_%d(id bigint primary key, t char(8192), last_update timestamp without time zone default (now() at time zone 'UTC'));` 5 | -------------------------------------------------------------------------------- /global/global.go: -------------------------------------------------------------------------------- 1 | package global 2 | 3 | import ( 4 | "ptor/cli" 5 | 6 | "github.com/gammazero/workerpool" 7 | ) 8 | 9 | var ( 10 | CliOpts cli.CliArgs 11 | PrimaryWorkerPool *workerpool.WorkerPool 12 | RepoWorkerPool *workerpool.WorkerPool 13 | ) 14 | -------------------------------------------------------------------------------- /util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/signal" 7 | "ptor/db" 8 | "ptor/global" 9 | "syscall" 10 | ) 11 | 12 | type fn func() 13 | 14 | func HandleCntrlC(f fn) { 15 | c := make(chan os.Signal) 16 | signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGABRT, syscall.SIGSEGV) 17 | go func() { 18 | <-c 19 | f() 20 | os.Exit(1) 21 | }() 22 | } 23 | 24 | func QuitNice() { 25 | StopTasks() 26 | os.Exit(0) 27 | } 28 | 29 | func StopTasks() { 30 | global.PrimaryWorkerPool.Stop() 31 | err := db.ClosePrimaryPool() 32 | if err != nil { 33 | fmt.Println(err) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Go parameters 2 | GOCMD=go 3 | GOBUILD=$(GOCMD) build 4 | GOCLEAN=$(GOCMD) clean 5 | BINARY_NAME=ptor 6 | COMMIT=$(shell git rev-parse --short HEAD) 7 | DATE=$(shell git log -1 --format=%ci) 8 | Version=0.1.0 9 | 10 | # Do not set CGO_ENABLED=0 11 | # sqlite3 will throw error, if you set this 12 | 13 | all: build 14 | build: 15 | $(GOBUILD) -o $(BINARY_NAME) -v -ldflags="-X 'main.Version=${Version}' -X 'main.GitCommit=${COMMIT}' -X 'main.CommitDate=${DATE}'" 16 | 17 | linux: 18 | GOOS=linux $(GOBUILD) -o $(BINARY_NAME) -v -ldflags="-X 'main.Version=${Version}' -X 'main.GitCommit=${COMMIT}' -X 'main.CommitDate=${DATE}'" 19 | 20 | test: 21 | $(GOTEST) -v ./... 22 | clean: 23 | $(GOCLEAN) 24 | rm -f $(BINARY_NAME) 25 | -------------------------------------------------------------------------------- /locks/locks.go: -------------------------------------------------------------------------------- 1 | package locks 2 | 3 | import ( 4 | "ptor/global" 5 | "sync" 6 | ) 7 | 8 | var primaryRelationLocks = make(map[int]*sync.RWMutex) 9 | var repoRelationLocks = make(map[int]*sync.RWMutex) 10 | 11 | var PrimarySyncLock = &sync.RWMutex{} 12 | var RepoSyncLock = &sync.RWMutex{} 13 | 14 | func CreatePrimaryRelLocks() { 15 | for i := 0; i < global.CliOpts.ParallelWorkers; i++ { 16 | primaryRelationLocks[i] = &sync.RWMutex{} 17 | } 18 | } 19 | 20 | func AcquirePrimaryWorkerLock(worker_id int) { 21 | // fmt.Println("AcquireWorkerLock", worker_id) 22 | primaryRelationLocks[worker_id].Lock() 23 | } 24 | 25 | func ReleasePrimaryWorkerLock(worker_id int) { 26 | //fmt.Println("ReleaseWorkerLock", worker_id) 27 | primaryRelationLocks[worker_id].Unlock() 28 | } 29 | 30 | func CreateRepoRelLocks() { 31 | for i := 0; i < global.CliOpts.ParallelWorkers; i++ { 32 | repoRelationLocks[i] = &sync.RWMutex{} 33 | } 34 | } 35 | 36 | func AcquireRepoWorkerLock(worker_id int) { 37 | // fmt.Println("AcquireWorkerLock", worker_id) 38 | repoRelationLocks[worker_id].Lock() 39 | } 40 | 41 | func ReleaseRepoWorkerLock(worker_id int) { 42 | //fmt.Println("ReleaseWorkerLock", worker_id) 43 | repoRelationLocks[worker_id].Unlock() 44 | } 45 | -------------------------------------------------------------------------------- /cli/cli.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | type CliArgs struct { 4 | RepoPgDsn string `arg:"--repo-pgdsn,env:REPO_PGDSN" default:"postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable"` 5 | PrimaryPgDsn string `arg:"--primary-pgdsn,env:PRIMARY_PGDSN" default:"postgres://postgres:postgres@localhost:6432/postgres?sslmode=disable"` 6 | ParallelWorkers int `arg:"--parallel-workers,env:PARALLEL_WORKERS" default:"10"` 7 | Init bool `arg:"--init,env:INIT" default:"false"` 8 | Reset bool `arg:"--reset,env:RESET" default:"false"` 9 | 10 | RTOTimeout int `arg:"--rto-conn-timeout,env:TIMEOUT" default:"10"` 11 | WarmupDuration int `arg:"--warmup-duration,env:DURATION" default:"10"` 12 | 13 | InsertPercent int `arg:"--insert-percent,env:INSERT_PERCENT" default:"50"` 14 | UpdatePercent int `arg:"--update-percent,env:UPDATE_PERCENT" default:"30"` 15 | DeletePercent int `arg:"--delete-percent,env:DELETE_PERCENT" default:"20"` 16 | 17 | FullDataValidation bool `arg:"--full-data-validation,env:FULL_DATA_VALIDATION" default:"true"` 18 | AsyncRepoMode bool `arg:"--async-repo-mode,env:ASYNC_REPO_MODE" default:"true"` 19 | 20 | CheckPrimaryLatency bool `arg:"--check-primary-latency,env:CHECK_PRIMARY_LATENCY" default:"false"` 21 | ValidationDelay int `arg:"--validation-delay,env:VALIDATION_DELAY" default:"0"` 22 | 23 | Help bool `arg:"--help" default:"false"` 24 | About bool `arg:"--about" default:"false"` 25 | } 26 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module ptor 2 | 3 | go 1.19 4 | 5 | require github.com/jackc/pgx/v4 v4.17.2 6 | 7 | require ( 8 | github.com/Delta456/box-cli-maker v1.3.2 // indirect 9 | github.com/briandowns/spinner v1.21.0 // indirect 10 | github.com/fatih/color v1.14.1 // indirect 11 | github.com/gammazero/deque v0.2.0 // indirect 12 | github.com/gammazero/workerpool v1.1.3 // indirect 13 | github.com/jackc/puddle v1.3.0 // indirect 14 | github.com/lossdev/stack v1.0.5 // indirect 15 | github.com/mattn/go-colorable v0.1.13 // indirect 16 | github.com/mattn/go-isatty v0.0.17 // indirect 17 | github.com/mattn/go-runewidth v0.0.9 // indirect 18 | golang.org/x/sys v0.5.0 // indirect 19 | golang.org/x/term v0.5.0 // indirect 20 | gopkg.in/karalabe/cookiejar.v2 v2.0.0-20150724131613-8dcd6a7f4951 // indirect 21 | ) 22 | 23 | require ( 24 | github.com/alexflint/go-arg v1.4.3 // indirect 25 | github.com/alexflint/go-scalar v1.1.0 // indirect 26 | github.com/dariubs/percent v1.0.0 27 | github.com/jackc/chunkreader/v2 v2.0.1 // indirect 28 | github.com/jackc/pgconn v1.13.0 // indirect 29 | github.com/jackc/pgio v1.0.0 // indirect 30 | github.com/jackc/pgpassfile v1.0.0 // indirect 31 | github.com/jackc/pgproto3/v2 v2.3.1 // indirect 32 | github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect 33 | github.com/jackc/pgtype v1.12.0 // indirect 34 | github.com/jackc/pgx v3.6.2+incompatible // indirect 35 | github.com/pkg/errors v0.9.1 // indirect 36 | golang.org/x/crypto v0.6.0 // indirect 37 | golang.org/x/text v0.7.0 // indirect 38 | ) 39 | -------------------------------------------------------------------------------- /pool/pool.go: -------------------------------------------------------------------------------- 1 | package pool 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "ptor/global" 7 | 8 | "github.com/jackc/pgx/v4/pgxpool" 9 | ) 10 | 11 | var PrimaryPool *Pool 12 | var RepoPool *Pool 13 | 14 | type Pool struct { 15 | pool *pgxpool.Pool 16 | } 17 | 18 | func (p *Pool) GetPrimaryPool() *pgxpool.Pool { 19 | return p.pool 20 | } 21 | 22 | func (p *Pool) InitPrimaryPool() error { 23 | parseConfig, err := pgxpool.ParseConfig(global.CliOpts.PrimaryPgDsn) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | config := parseConfig 29 | config.MaxConns = int32(global.CliOpts.ParallelWorkers) 30 | config.LazyConnect = true 31 | p.pool, err = pgxpool.ConnectConfig(context.Background(), config) 32 | if err != nil { 33 | return err 34 | } 35 | 36 | return nil 37 | } 38 | 39 | func (p *Pool) GetConnectionFromPrimaryPool() (*pgxpool.Conn, error) { 40 | if p.pool == nil { 41 | return nil, fmt.Errorf("unable to get connection from an empty pool") 42 | } 43 | 44 | conn, err := p.pool.Acquire(context.Background()) 45 | if err != nil { 46 | return nil, err 47 | } 48 | return conn, nil 49 | } 50 | 51 | func (p *Pool) ClosePrimaryPool() error { 52 | if p.pool != nil { 53 | p.pool.Close() 54 | } 55 | return nil 56 | } 57 | 58 | func (p *Pool) GetRepoPool() *pgxpool.Pool { 59 | return p.pool 60 | } 61 | 62 | func (p *Pool) InitRepoPool() error { 63 | parseConfig, err := pgxpool.ParseConfig(global.CliOpts.RepoPgDsn) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | config := parseConfig 69 | config.MaxConns = int32(global.CliOpts.ParallelWorkers) 70 | config.LazyConnect = true 71 | p.pool, err = pgxpool.ConnectConfig(context.Background(), config) 72 | if err != nil { 73 | return err 74 | } 75 | 76 | return nil 77 | } 78 | 79 | func (p *Pool) GetConnectionFromRepoPool() (*pgxpool.Conn, error) { 80 | if p.pool == nil { 81 | return nil, fmt.Errorf("unable to get connection from an empty pool") 82 | } 83 | 84 | conn, err := p.pool.Acquire(context.Background()) 85 | if err != nil { 86 | return nil, err 87 | } 88 | return conn, nil 89 | } 90 | 91 | func (p *Pool) CloseRepoPool() error { 92 | if p.pool != nil { 93 | p.pool.Close() 94 | } 95 | return nil 96 | } 97 | -------------------------------------------------------------------------------- /calc/calc.go: -------------------------------------------------------------------------------- 1 | package calc 2 | 3 | import ( 4 | "fmt" 5 | "ptor/db" 6 | "ptor/global" 7 | "ptor/repo" 8 | "time" 9 | ) 10 | 11 | func ServiceAvailable(lost_err_time time.Time) time.Duration { 12 | var service_conn_time *time.Time 13 | var err error 14 | // Make a fresh connection to primary/read-write postgres 15 | for { 16 | service_conn_time, err = db.CheckPrimaryConnTime() 17 | if err != nil { 18 | //fmt.Println("Error while making new primary read-write connection: ", err) 19 | } else { 20 | break 21 | } 22 | } 23 | 24 | return service_conn_time.Sub(lost_err_time) 25 | } 26 | 27 | func RTO(lost_err_time time.Time) time.Duration { 28 | var fresh_conn_time *time.Time 29 | var err error 30 | // Make a fresh connection to primary/read-write postgres 31 | for { 32 | fresh_conn_time, err = db.MakeNewPrimaryReadWriteConn() 33 | if err != nil { 34 | //fmt.Println("Error while making new primary read-write connection: ", err) 35 | } else { 36 | break 37 | } 38 | } 39 | 40 | // If there is a validation delay specified, then add that delay to the RTO 41 | if global.CliOpts.ValidationDelay > 0 { 42 | fresh_conn_time.Add(time.Duration(global.CliOpts.ValidationDelay) * time.Second) 43 | } 44 | 45 | // Calculate RTO 46 | return fresh_conn_time.Sub(lost_err_time) 47 | } 48 | 49 | func RPO() (time.Duration, int) { 50 | // RPO calculation is bit tricky. 51 | // Once we get a fresh new connection to primary, then we have to compare the results with the repo results. 52 | 53 | var total_duration time.Duration 54 | var total_data_los_bytes int 55 | 56 | for i := 0; i < global.CliOpts.ParallelWorkers; i++ { 57 | // Get the max date from primary 58 | max_primary_date, err := db.GetMaxDateFromPrimary(i) 59 | if err != nil { 60 | fmt.Println("Error while getting max date from primary: ", err) 61 | } 62 | 63 | // Compare the results with repo 64 | duration, data_los_bytes, err := repo.CompareDataWithPrimary(i, max_primary_date) 65 | if err != nil { 66 | fmt.Println("Error while comparing data with primary: ", err) 67 | } 68 | 69 | total_duration += duration 70 | total_data_los_bytes += data_los_bytes 71 | } 72 | 73 | return total_duration, total_data_los_bytes 74 | } 75 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "ptor/cli" 7 | "ptor/db" 8 | "ptor/global" 9 | "ptor/locks" 10 | "ptor/pool" 11 | "ptor/repo" 12 | "ptor/task" 13 | "ptor/util" 14 | 15 | "github.com/alexflint/go-arg" 16 | "github.com/gammazero/workerpool" 17 | ) 18 | 19 | var args cli.CliArgs 20 | 21 | var Version string 22 | var GitCommit string 23 | var CommitDate string 24 | 25 | func init() { 26 | arg.MustParse(&args) 27 | global.CliOpts = args 28 | 29 | if args.About { 30 | fmt.Println("Version: ", Version) 31 | fmt.Println("Git Commit: ", GitCommit) 32 | fmt.Println("Git Commit Date: ", CommitDate) 33 | os.Exit(0) 34 | } 35 | 36 | if args.Init { 37 | err := db.InitSchema() 38 | if err != nil { 39 | fmt.Println(err) 40 | } 41 | 42 | err = repo.InitSchema() 43 | if err != nil { 44 | fmt.Println(err) 45 | } 46 | os.Exit(0) 47 | } 48 | 49 | if args.Reset { 50 | err := db.DropSchema() 51 | if err != nil { 52 | fmt.Println(err) 53 | } 54 | 55 | err = repo.DropSchema() 56 | if err != nil { 57 | fmt.Println(err) 58 | } 59 | os.Exit(0) 60 | } 61 | 62 | //Create lock map 63 | locks.CreatePrimaryRelLocks() 64 | locks.CreateRepoRelLocks() 65 | 66 | //Create connection pool for the primary 67 | pool.PrimaryPool = new(pool.Pool) 68 | err := pool.PrimaryPool.InitPrimaryPool() 69 | if err != nil { 70 | fmt.Println(err) 71 | os.Exit(1) 72 | } 73 | 74 | //Create connection pool for the repo 75 | pool.RepoPool = new(pool.Pool) 76 | err = pool.RepoPool.InitRepoPool() 77 | if err != nil { 78 | fmt.Println(err) 79 | os.Exit(1) 80 | } 81 | 82 | util.HandleCntrlC(func() { 83 | util.QuitNice() 84 | }) 85 | 86 | // Check the percentages of ins,upd and del 87 | if args.InsertPercent+args.DeletePercent+args.UpdatePercent != 100 { 88 | fmt.Println("Insert, Update and Delete percentages should add up to 100") 89 | os.Exit(1) 90 | } 91 | 92 | } 93 | 94 | func bootstrapChecks() { 95 | // Check repo and primary connections 96 | fmt.Println("Trying to connect to primary instance ...") 97 | err := db.CheckConnection() 98 | if err != nil { 99 | fmt.Println("Error connecting to primary instance: ", err) 100 | os.Exit(1) 101 | } 102 | 103 | fmt.Println("Trying to connect to repo instance ...") 104 | err = repo.CheckConnection() 105 | if err != nil { 106 | fmt.Println("Error connecting to repo instance: ", err) 107 | os.Exit(1) 108 | } 109 | 110 | // Check if schema present 111 | fmt.Println("Checking for ptor schema on primary instance ...") 112 | err = db.CheckPtorSchema() 113 | if err != nil { 114 | fmt.Println(err) 115 | os.Exit(1) 116 | } 117 | 118 | fmt.Println("Checking for ptor schema on repo instance ...") 119 | err = repo.CheckPtorSchema() 120 | if err != nil { 121 | fmt.Println(err) 122 | os.Exit(1) 123 | } 124 | 125 | } 126 | 127 | func main() { 128 | global.PrimaryWorkerPool = workerpool.New(global.CliOpts.ParallelWorkers) 129 | global.RepoWorkerPool = workerpool.New(1) 130 | bootstrapChecks() 131 | 132 | if global.CliOpts.CheckPrimaryLatency { 133 | duration, err := task.CheckPrimaryLatency() 134 | 135 | if err != nil { 136 | fmt.Println(err) 137 | os.Exit(1) 138 | } 139 | fmt.Println("Primary latency: ", duration) 140 | os.Exit(0) 141 | } 142 | 143 | task.Run() 144 | } 145 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Design 2 | The `ptor` tool is designed to validate the `RPO`, `RTO` of a PostgreSQL HA instance. By using this tool, we can put some load into the primary instance, which will get replicated to it's secondary. 3 | 4 | While the data loading happens, we can either turn off the primary or it's vm instance to trigger the underlying `auto failover` system. Once the `auto failover` happens, `ptor` tool will evaluate the `RPO(Data Loss)` and `RTO(Recovery Time)` of the HA system. 5 | 6 | 7 | To use `ptor` tool, we need below two instances. 8 | 9 | 1. `PRIMARY_PGDSN` which points to the primary endpoint 10 | 2. `REPO_PGDSN` which makes a copy of the transactions, which parallel workers execute on the `PRIMARY_PGDSN`. 11 | We need this `REPO_PGDSN` instance, to validate the data after we trigger the failover/switchover. 12 | 13 | ![](./ptor.png) 14 | ## Quick Test 15 | 16 | ### Local 17 | Quick test performed between the two local instance, where primary and repo instances are in `sync streaming` replication mode. In this demo, we restarted the local `primary` instance to mimic the `failover/switchover`. 18 | 19 | [![asciicast](https://asciinema.org/a/2MRLVcmL2cm7V4eWCtM9rj0Yf.svg)](https://asciinema.org/a/2MRLVcmL2cm7V4eWCtM9rj0Yf) 20 | 21 | 22 | ## Other Demos 23 | All the demos are done with a `sync streaming` replication between the primary and secondary nodes. All the instances are configured to be in the same network. 24 | 25 | ### pg_auto_failover 26 | Configure `PRIMARY_PGDSN` as a `multi host` connection string as like below. 27 | 28 | host=host1,host2 port=5432,5432 user=postgres password=password target_session_attrs=read-write 29 | 30 | [Demo Ptor pg_auto_failover](https://youtu.be/_0vhXn0HbWU) 31 | 32 | ### Stolon 33 | Configure `PRIMARY_PGDSN` as to point the `stolon proxy`. 34 | 35 | [Demo Ptor Stolon](https://youtu.be/SDriI00HNbM) 36 | 37 | ### Patroni 38 | Configure `PRIMARY_PGDSN` as to point the `haproxy`. 39 | Below is the `haproxy.cfg` used for this demo. 40 | 41 | global 42 | maxconn 100 43 | 44 | defaults 45 | log global 46 | mode tcp 47 | retries 1 48 | timeout client 30m 49 | timeout connect 1s 50 | timeout server 30m 51 | timeout check 1s 52 | 53 | listen stats 54 | mode http 55 | bind *:2361 56 | stats enable 57 | stats uri / 58 | 59 | listen production 60 | bind 172.31.46.52:2360 61 | option httpchk OPTIONS/master 62 | http-check expect status 200 63 | default-server inter 1s fall 1 rise 1 on-marked-down shutdown-sessions 64 | server postgresql_192.168.56.104_5432 172.31.34.9:5432 maxconn 100 check port 8008 65 | server postgresql_192.168.56.105_5432 172.31.43.176:5432 maxconn 100 check port 8008 66 | 67 | 68 | [Demo Ptor Patroni, HaProxy](https://youtu.be/NoDMljx8_Q0) 69 | 70 | 71 | 72 | ## Installation 73 | 74 | Below are the installation steps, which are prepared on RHEL instance. If you are using debain flavour, then use the platform specific package tools like `apt-get` or `brew` to install the below components. 75 | 76 | 77 | 1. Install `git` 78 | 79 | $ sudo yum install git -y 80 | 81 | 2. Install `golang` 82 | 83 | $ sudo yum install golang -y 84 | 85 | 86 | 3. Install `PostgreSQL server` (Optional) 87 | 88 | $ sudo yum install postgresql-server -y 89 | 90 | This is for the repo server, where we save a copy of primary transactions. 91 | 92 | 4. Download the copy of `ptor` source 93 | 94 | $ git clone https://github.com/dineshkumar02/ptor.git 95 | 96 | 5. Build the `ptor` binary 97 | 98 | $ cd ptor 99 | $ make 100 | 101 | ## Usage 102 | | Option | Usage | 103 | |------------------------|---------------------------------------------------------------------------------------------------------------------------| 104 | | --repo-pgdsn | The `repo` PostgreSQL connection string, where it syncs primary data. | 105 | | --primary-pgdsn | The primary or service PostgreSQL connection string, where we run switchover/failover. | 106 | | --parallel-workers | Number of parallel workers to run data loading. It will create these many individual tables. | 107 | | --init | Initialize the `paralle-workers` tables. | 108 | | --reset | Delete all data from `repo` and `primary` instances. | 109 | | --warmup-duration | Initial data loading duration in seconds. | 110 | | --insert-percent | Percentage number of insert operations. | 111 | | --update-percent | Percentage number of update operations. | 112 | | --delete-percent | Percentage number of delete operations. | 113 | | --full-data-validation | Run full data validation on both `repo` and `primary` in the end of the test case. | 114 | | --async-repo-mode | All `primary` events will get in sync to `repo` asynchronously. This improves more data generation on the `primary` side. | 115 | | --rto-conn-timeout | `Primary` dns connection timeout value. This helps in calculating the `RTO` | 116 | | --check-primary-latency | Check network connectivity latency between `ptor` and `primary` dns | 117 | | --validation-delay | Delay the data validation these many seconds. This value is going to be added to the `RTO` | -------------------------------------------------------------------------------- /repo/repo.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "ptor/catalog" 7 | "ptor/global" 8 | "ptor/locks" 9 | "ptor/pool" 10 | "time" 11 | 12 | "github.com/jackc/pgx/v4" 13 | ) 14 | 15 | func CheckConnection() error { 16 | 17 | conn, err := pool.RepoPool.GetConnectionFromRepoPool() 18 | if err != nil { 19 | return err 20 | } 21 | defer conn.Release() 22 | 23 | err = conn.Ping(context.Background()) 24 | if err != nil { 25 | return err 26 | } 27 | return nil 28 | } 29 | 30 | func CheckPtorSchema() error { 31 | 32 | conn, err := pool.RepoPool.GetConnectionFromRepoPool() 33 | if err != nil { 34 | return err 35 | } 36 | 37 | defer conn.Release() 38 | 39 | // Check if ptor schema exists 40 | var schema string 41 | err = conn.QueryRow(context.Background(), "SELECT COALESCE(schema_name,'') FROM information_schema.schemata WHERE schema_name = 'ptor'").Scan(&schema) 42 | if err != nil { 43 | 44 | if err == pgx.ErrNoRows { 45 | return fmt.Errorf("repo instance is missing the ptor schema, run the initialization command first") 46 | } 47 | return err 48 | } 49 | 50 | return nil 51 | } 52 | 53 | //XXX 54 | //Repo database should always be up and running. 55 | //Repo database should not throw any error. 56 | 57 | func SaveInsState(worker_id *int, id *int, t *time.Time) error { 58 | 59 | // Acquire lock on the worker relation 60 | locks.AcquireRepoWorkerLock(*worker_id) 61 | defer locks.ReleaseRepoWorkerLock(*worker_id) 62 | 63 | conn, err := pool.RepoPool.GetConnectionFromRepoPool() 64 | if err != nil { 65 | return err 66 | } 67 | defer conn.Release() 68 | 69 | // Save insert state 70 | _, err = conn.Exec(context.Background(), fmt.Sprintf("INSERT INTO ptor.worker_%d (id, t, last_update) VALUES( $1, 'a', $2)", *worker_id), *id, *t) 71 | return err 72 | } 73 | 74 | func SaveUpdState(worker_id *int, rec_id *int, t *time.Time) error { 75 | // Acquire lock on the worker relation 76 | locks.AcquireRepoWorkerLock(*worker_id) 77 | defer locks.ReleaseRepoWorkerLock(*worker_id) 78 | 79 | // Connect to repo postgres 80 | conn, err := pool.RepoPool.GetConnectionFromRepoPool() 81 | if err != nil { 82 | return err 83 | } 84 | defer conn.Release() 85 | 86 | // Save update state 87 | _, err = conn.Exec(context.Background(), fmt.Sprintf("UPDATE ptor.worker_%d SET t = 'b', last_update=$2 WHERE id = $1", *worker_id), *rec_id, *t) 88 | return err 89 | } 90 | 91 | func SaveDelState(worker_id *int, rec_id *int) error { 92 | // Acquire lock on the worker relation 93 | locks.AcquireRepoWorkerLock(*worker_id) 94 | defer locks.ReleaseRepoWorkerLock(*worker_id) 95 | // Connect to repo postgres 96 | // Connect to repo postgres 97 | conn, err := pool.RepoPool.GetConnectionFromRepoPool() 98 | if err != nil { 99 | return err 100 | } 101 | defer conn.Release() 102 | 103 | // Save delete state 104 | _, err = conn.Exec(context.Background(), fmt.Sprintf("DELETE FROM ptor.worker_%d WHERE id=$1", *worker_id), *rec_id) 105 | if err != nil { 106 | return err 107 | } 108 | 109 | // wait for commit 110 | return err 111 | } 112 | 113 | func InitSchema() error { 114 | conn, err := pgx.Connect(context.Background(), global.CliOpts.RepoPgDsn) 115 | if err != nil { 116 | return err 117 | } 118 | defer conn.Close(context.Background()) 119 | 120 | _, err = conn.Exec(context.Background(), catalog.SchemaSQL) 121 | if err != nil { 122 | return err 123 | } 124 | 125 | for i := 0; i < global.CliOpts.ParallelWorkers; i++ { 126 | _, err = conn.Exec(context.Background(), fmt.Sprintf(catalog.TableSQL, i)) 127 | if err != nil { 128 | return err 129 | } 130 | } 131 | return nil 132 | } 133 | 134 | func DropSchema() error { 135 | conn, err := pgx.Connect(context.Background(), global.CliOpts.RepoPgDsn) 136 | if err != nil { 137 | return err 138 | } 139 | defer conn.Close(context.Background()) 140 | 141 | _, err = conn.Exec(context.Background(), "DROP SCHEMA IF EXISTS ptor CASCADE") 142 | if err != nil { 143 | return err 144 | } 145 | 146 | return nil 147 | } 148 | 149 | func GetMaxDateFromRepo(worker_id int) (time.Time, error) { 150 | conn, err := pgx.Connect(context.Background(), global.CliOpts.RepoPgDsn) 151 | if err != nil { 152 | return time.Time{}, err 153 | } 154 | defer conn.Close(context.Background()) 155 | 156 | var maxDate time.Time 157 | err = conn.QueryRow(context.Background(), fmt.Sprintf("SELECT max(last_update) FROM ptor.worker_%d", worker_id)).Scan(&maxDate) 158 | if err != nil { 159 | return time.Time{}, err 160 | } 161 | 162 | return maxDate, nil 163 | } 164 | 165 | func CompareDataWithPrimary(worker_id int, primary_max_time time.Time) (duration time.Duration, data_los_bytes int, err error) { 166 | conn, err := pgx.Connect(context.Background(), global.CliOpts.RepoPgDsn) 167 | if err != nil { 168 | return 0, 0, err 169 | } 170 | defer conn.Close(context.Background()) 171 | 172 | var repo_max_time time.Time 173 | 174 | err = conn.QueryRow(context.Background(), fmt.Sprintf("SELECT COALESCE(max(last_update),'12-12-1212 12:12:12') FROM ptor.worker_%d WHERE last_update >= $1", worker_id), primary_max_time).Scan(&repo_max_time) 175 | if err != nil { 176 | return 0, 0, err 177 | } 178 | 179 | if repo_max_time == primary_max_time { 180 | return 0, 0, nil 181 | } 182 | 183 | err = conn.QueryRow(context.Background(), fmt.Sprintf("SELECT sum(a) FROM (SELECT pg_column_size(id)+pg_column_size(t)+pg_column_size(last_update) a FROM ptor.worker_%d WHERE last_update > $1) foo;", worker_id), primary_max_time).Scan(&data_los_bytes) 184 | if err != nil { 185 | return 0, 0, err 186 | } 187 | 188 | return repo_max_time.Sub(primary_max_time), data_los_bytes, nil 189 | } 190 | 191 | func GetRepoRowCount(worker_id int) (int, error) { 192 | conn, err := pgx.Connect(context.Background(), global.CliOpts.RepoPgDsn) 193 | if err != nil { 194 | return 0, err 195 | } 196 | defer conn.Close(context.Background()) 197 | 198 | var rowCount int 199 | err = conn.QueryRow(context.Background(), fmt.Sprintf("SELECT count(*) FROM ptor.worker_%d", worker_id)).Scan(&rowCount) 200 | if err != nil { 201 | return 0, err 202 | } 203 | 204 | return rowCount, nil 205 | } 206 | 207 | func GetRepoRelationCheckSum(worker_id int) (string, error) { 208 | conn, err := pgx.Connect(context.Background(), global.CliOpts.RepoPgDsn) 209 | if err != nil { 210 | return "", err 211 | } 212 | defer conn.Close(context.Background()) 213 | 214 | var checkSum string 215 | err = conn.QueryRow(context.Background(), fmt.Sprintf("WITH sort_data AS (SELECT id||trim(t)||last_update as data FROM ptor.worker_%d ORDER BY last_update DESC) SELECT COALESCE(md5(array_agg(sort_data.data)::text), '') FROM sort_data;", worker_id)).Scan(&checkSum) 216 | if err != nil { 217 | return "", err 218 | } 219 | 220 | return checkSum, nil 221 | } 222 | -------------------------------------------------------------------------------- /task/task.go: -------------------------------------------------------------------------------- 1 | package task 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "math/rand" 7 | "ptor/calc" 8 | "ptor/db" 9 | "ptor/global" 10 | "ptor/repo" 11 | "ptor/util" 12 | "strconv" 13 | "text/tabwriter" 14 | "time" 15 | 16 | "github.com/Delta456/box-cli-maker" 17 | "github.com/briandowns/spinner" 18 | "github.com/dariubs/percent" 19 | "github.com/fatih/color" 20 | ) 21 | 22 | func getRandomNumber() int { 23 | rand.Seed(time.Now().UnixNano()) 24 | min := 1 25 | max := 100 26 | return rand.Intn(max-min+1) + min 27 | } 28 | 29 | func Run() { 30 | task_done := make(chan bool) 31 | 32 | var insert_range int 33 | var update_range int 34 | var delete_range int 35 | 36 | insert_count := 0 37 | update_count := 0 38 | delete_count := 0 39 | 40 | var workerCnt = 0 41 | var err error 42 | var error_time time.Time 43 | 44 | // defer func() { 45 | // fmt.Println("Insert count: ", insert_count) 46 | // fmt.Println("Update count: ", update_count) 47 | // fmt.Println("Delete count: ", delete_count) 48 | // }() 49 | 50 | insert_range = (int)(percent.Percent(global.CliOpts.InsertPercent, 100)) 51 | update_range = (int)(insert_range + (int)(percent.Percent(global.CliOpts.UpdatePercent, 100))) 52 | delete_range = (int)(update_range + (int)(percent.Percent(global.CliOpts.DeletePercent, 100))) 53 | 54 | s := spinner.New(spinner.CharSets[9], 100*time.Millisecond) 55 | 56 | go func() { 57 | s.Prefix = color.HiCyanString("Warming up instance...") 58 | s.Start() 59 | time.Sleep(time.Duration(global.CliOpts.WarmupDuration) * time.Second) 60 | s.Stop() 61 | task_done <- true 62 | }() 63 | 64 | for { 65 | if (workerCnt + 1) == global.CliOpts.ParallelWorkers { 66 | workerCnt = 0 67 | } else { 68 | workerCnt++ 69 | } 70 | 71 | select { 72 | case <-task_done: 73 | fmt.Println(color.HiBlueString("\nDo the failover/switchover")) 74 | s.Prefix = color.HiCyanString("Data loading...") 75 | s.Start() 76 | //util.StopTasks() 77 | //return 78 | default: 79 | } 80 | 81 | // Generate random number between 1 to 100 82 | random_number := getRandomNumber() 83 | if random_number <= insert_range { 84 | if !global.PrimaryWorkerPool.Stopped() { 85 | global.PrimaryWorkerPool.Submit(func() { 86 | err = db.DoInsert(workerCnt) 87 | insert_count++ 88 | }) 89 | } 90 | } else if random_number <= update_range { 91 | if !global.PrimaryWorkerPool.Stopped() { 92 | global.PrimaryWorkerPool.Submit(func() { 93 | err = db.DoUpdate(workerCnt) 94 | update_count++ 95 | }) 96 | } 97 | } else if random_number <= delete_range { 98 | if !global.PrimaryWorkerPool.Stopped() { 99 | global.PrimaryWorkerPool.Submit(func() { 100 | err = db.DoDelete(workerCnt) 101 | delete_count++ 102 | }) 103 | } 104 | } else { 105 | // We should not come here 106 | } 107 | 108 | if err != nil { 109 | //Stop spinner 110 | s.Stop() 111 | error_time = time.Now() 112 | fmt.Println(color.HiRedString("Got the connectivity error")) 113 | util.StopTasks() 114 | break 115 | } 116 | } 117 | fmt.Println(color.HiGreenString("Checking Connection Availability...")) 118 | serviceAvailable := calc.ServiceAvailable(error_time) 119 | 120 | if global.CliOpts.ValidationDelay > 0 { 121 | fmt.Println(color.HiGreenString(fmt.Sprintf("Sleeping %d seconds validation delay...", global.CliOpts.ValidationDelay))) 122 | time.Sleep(time.Duration(global.CliOpts.ValidationDelay) * time.Second) 123 | } 124 | 125 | // Calculate RTO 126 | fmt.Println(color.HiGreenString("Checking RTO...")) 127 | rto := calc.RTO(error_time) 128 | // Wait for the repo to get sync 129 | for { 130 | //Wait until the repo worker pool is empty 131 | var queueSize = global.RepoWorkerPool.WaitingQueueSize() 132 | s.Prefix = color.HiCyanString(fmt.Sprintf("Waiting for the repo db sync. Queue size %d", queueSize)) 133 | s.Start() 134 | if queueSize == 0 { 135 | s.Stop() 136 | // Waiting for any pending tasks to complete 137 | s.Prefix = "Waiting for any pending tasks to get complete..." 138 | s.Start() 139 | global.RepoWorkerPool.StopWait() 140 | // Stop the spinner 141 | s.Stop() 142 | break 143 | } 144 | time.Sleep(1 * time.Second) 145 | } 146 | 147 | fmt.Println(color.HiGreenString("Checking RPO...")) 148 | // Calculate RPO 149 | rpo, data_los_bytes := calc.RPO() 150 | buf := new(bytes.Buffer) 151 | w := tabwriter.NewWriter(buf, 0, 0, 1, ' ', 0) 152 | fmt.Fprintln(w, "\t", "") 153 | fmt.Fprintln(w, "SLA\t", fmt.Sprintf("%.5f", percent.PercentOf(86400*1000-int(rto.Milliseconds()), 86400*1000))) 154 | fmt.Fprintln(w, "Service Available\t", serviceAvailable) 155 | fmt.Fprintln(w, "RTO\t", rto) 156 | fmt.Fprintln(w, "RPO\t", rpo) 157 | fmt.Fprintln(w, "Quick Data Loss Check (bytes)\t", data_los_bytes) 158 | 159 | w.Flush() 160 | 161 | summary := box.New(box.Config{Px: 1, Py: 1, Type: "Single", Color: "Cyan", TitlePos: "Top", ContentAlign: "Left"}) 162 | summary.Println("Summary", buf.String()) 163 | 164 | if global.CliOpts.FullDataValidation { 165 | s.Prefix = color.HiCyanString("Data validation is in progress...") 166 | s.Start() 167 | tab, success := fullDataValidation() 168 | 169 | //Stop the spinner 170 | s.Stop() 171 | if success { 172 | bx := box.New(box.Config{Px: 1, Py: 1, Type: "Single", Color: "Green", TitlePos: "Top", ContentAlign: "Left"}) 173 | bx.Println("Data Loss", "NO DATA LOSS") 174 | } else { 175 | bx := box.New(box.Config{Px: 1, Py: 1, Type: "Single", Color: "Red", TitlePos: "Top", ContentAlign: "Left"}) 176 | bx.Println("Data Loss", "DATA LOSS DETECTED") 177 | fmt.Println(color.HiRedString("Table " + tab + " is not in sync")) 178 | } 179 | } 180 | } 181 | 182 | func fullDataValidation() (string, bool) { 183 | for i := 0; i < global.CliOpts.ParallelWorkers; i++ { 184 | 185 | primaryCheckSum, err := db.GetPrimaryRelationCheckSum(i) 186 | if err != nil { 187 | fmt.Println(err) 188 | break 189 | } 190 | repoCheckSum, err := repo.GetRepoRelationCheckSum(i) 191 | if err != nil { 192 | fmt.Println(err) 193 | break 194 | } 195 | 196 | primaryRowCount, err := db.GetPrimaryRowCount(i) 197 | if err != nil { 198 | fmt.Println(err) 199 | break 200 | } 201 | 202 | repoRowCount, err := repo.GetRepoRowCount(i) 203 | if err != nil { 204 | fmt.Println(err) 205 | break 206 | } 207 | 208 | if primaryRowCount != repoRowCount { 209 | return "ptor.worker_" + strconv.Itoa(i), false 210 | } 211 | 212 | if primaryCheckSum != repoCheckSum { 213 | return "ptor.worker_" + strconv.Itoa(i), false 214 | } 215 | } 216 | return "", true 217 | } 218 | 219 | func CheckPrimaryLatency() (time.Duration, error) { 220 | // Create a new spinner 221 | s := spinner.New(spinner.CharSets[9], 100*time.Millisecond) 222 | 223 | // Start the spinner 224 | s.Prefix = color.HiCyanString("Checking primary latency...") 225 | s.Start() 226 | defer s.Stop() 227 | 228 | t, err := db.MakeNewPrimaryReadWriteConn() 229 | if err != nil { 230 | return 0, err 231 | } 232 | 233 | return time.Since(*t), nil 234 | } 235 | -------------------------------------------------------------------------------- /db/db.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "ptor/catalog" 8 | "ptor/global" 9 | "ptor/locks" 10 | "ptor/pool" 11 | "ptor/repo" 12 | "time" 13 | 14 | "github.com/jackc/pgx/v4" 15 | ) 16 | 17 | func CheckConnection() error { 18 | 19 | conn, err := pool.PrimaryPool.GetConnectionFromPrimaryPool() 20 | if err != nil { 21 | return err 22 | } 23 | defer conn.Release() 24 | 25 | err = conn.Ping(context.Background()) 26 | if err != nil { 27 | return err 28 | } 29 | return nil 30 | } 31 | 32 | func CheckPtorSchema() error { 33 | 34 | conn, err := pool.PrimaryPool.GetConnectionFromPrimaryPool() 35 | if err != nil { 36 | return err 37 | } 38 | 39 | defer conn.Release() 40 | 41 | // Check if ptor schema exists 42 | var schema string 43 | err = conn.QueryRow(context.Background(), "SELECT COALESCE(schema_name,'') FROM information_schema.schemata WHERE schema_name = 'ptor'").Scan(&schema) 44 | if err != nil { 45 | 46 | if err == pgx.ErrNoRows { 47 | return fmt.Errorf("primary instance is missing the ptor schema, run the initialization command first") 48 | } 49 | 50 | return err 51 | } 52 | 53 | return nil 54 | } 55 | 56 | func DoInsert(worker_id int) error { 57 | 58 | // Acquire lock on the worker relation 59 | locks.AcquirePrimaryWorkerLock(worker_id) 60 | defer locks.ReleasePrimaryWorkerLock(worker_id) 61 | 62 | conn, err := pool.PrimaryPool.GetConnectionFromPrimaryPool() 63 | if err != nil { 64 | return err 65 | } 66 | defer conn.Release() 67 | 68 | // Start transaction 69 | tx, err := conn.Begin(context.Background()) 70 | if err != nil { 71 | return err 72 | } 73 | defer tx.Rollback(context.Background()) 74 | 75 | var id int 76 | var t time.Time 77 | 78 | // Insert record 79 | err = tx.QueryRow(context.Background(), fmt.Sprintf("INSERT INTO ptor.worker_%d (id, t) VALUES( (SELECT COALESCE(max(id),0)+1 FROM ptor.worker_%d), 'a') RETURNING id,last_update", worker_id, worker_id)).Scan(&id, &t) 80 | if err != nil { 81 | return err 82 | } 83 | 84 | err = tx.Commit(context.Background()) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | // Save record into repo database asynchronously, if it is successful then commit the primary transaction 90 | if global.CliOpts.AsyncRepoMode { 91 | locks.RepoSyncLock.Lock() 92 | global.RepoWorkerPool.Submit(func() { 93 | err = repo.SaveInsState(&worker_id, &id, &t) 94 | if err != nil { 95 | fmt.Println(err) 96 | } 97 | }) 98 | locks.RepoSyncLock.Unlock() 99 | } else { 100 | err = repo.SaveInsState(&worker_id, &id, &t) 101 | if err != nil { 102 | fmt.Println(err) 103 | } 104 | } 105 | return nil 106 | } 107 | 108 | func DoUpdate(worker_id int) error { 109 | 110 | // Acquire lock on the worker relation 111 | locks.AcquirePrimaryWorkerLock(worker_id) 112 | defer locks.ReleasePrimaryWorkerLock(worker_id) 113 | 114 | var rec_id int 115 | var t time.Time 116 | 117 | conn, err := pool.PrimaryPool.GetConnectionFromPrimaryPool() 118 | if err != nil { 119 | return err 120 | } 121 | defer conn.Release() 122 | 123 | // Start transaction 124 | tx, err := conn.Begin(context.Background()) 125 | if err != nil { 126 | return err 127 | } 128 | defer tx.Rollback(context.Background()) 129 | 130 | // Update record 131 | err = tx.QueryRow(context.Background(), fmt.Sprintf(`WITH random_rec AS (SELECT (random() * ((SELECT max(id) FROM ptor.worker_%d)))::int as id) UPDATE ptor.worker_%d SET t='b', last_update=(now() AT TIME ZONE 'UTC'::text) WHERE id = (SELECT CASE WHEN id=0 THEN 1 ELSE id END FROM random_rec) RETURNING id, last_update;`, worker_id, worker_id)).Scan(&rec_id, &t) 132 | if err != nil { 133 | 134 | //XXX 135 | // If no rows are updated, then return nil 136 | // We may get into this situation if the table is empty 137 | // While we are running the `ptor` first time, then we may get into this situation 138 | // Where `update` wins over the parallel `insert` 139 | if err == pgx.ErrNoRows { 140 | return nil 141 | } 142 | 143 | return err 144 | } 145 | 146 | err = tx.Commit(context.Background()) 147 | if err != nil { 148 | return err 149 | } 150 | 151 | if global.CliOpts.AsyncRepoMode { 152 | locks.RepoSyncLock.Lock() 153 | global.RepoWorkerPool.Submit(func() { 154 | err = repo.SaveUpdState(&worker_id, &rec_id, &t) 155 | if err != nil { 156 | fmt.Println(err) 157 | } 158 | }) 159 | locks.RepoSyncLock.Unlock() 160 | } else { 161 | err = repo.SaveUpdState(&worker_id, &rec_id, &t) 162 | if err != nil { 163 | fmt.Println(err) 164 | } 165 | } 166 | 167 | return nil 168 | } 169 | 170 | func DoDelete(worker_id int) error { 171 | 172 | // Acquire lock on the worker relation 173 | locks.AcquirePrimaryWorkerLock(worker_id) 174 | defer locks.ReleasePrimaryWorkerLock(worker_id) 175 | 176 | var rec_id int 177 | 178 | conn, err := pool.PrimaryPool.GetConnectionFromPrimaryPool() 179 | if err != nil { 180 | return err 181 | } 182 | defer conn.Release() 183 | 184 | // Start transaction 185 | tx, err := conn.Begin(context.Background()) 186 | if err != nil { 187 | return err 188 | } 189 | defer tx.Rollback(context.Background()) 190 | 191 | // Delete Record 192 | err = tx.QueryRow(context.Background(), fmt.Sprintf(`WITH random_rec AS (SELECT (random() * ( (SELECT max(id) FROM ptor.worker_%d)))::int as id) DELETE FROM ptor.worker_%d WHERE id = (SELECT CASE WHEN id=0 THEN 1 ELSE id END FROM random_rec) RETURNING id;`, worker_id, worker_id)).Scan(&rec_id) 193 | if err != nil { 194 | if err == pgx.ErrNoRows { 195 | return nil 196 | } 197 | return err 198 | } 199 | 200 | // Delete the record in repo database, if it is successful then commit the primary transaction 201 | err = tx.Commit(context.Background()) 202 | if err != nil { 203 | return err 204 | } 205 | 206 | if global.CliOpts.AsyncRepoMode { 207 | locks.RepoSyncLock.Lock() 208 | global.RepoWorkerPool.Submit(func() { 209 | err = repo.SaveDelState(&worker_id, &rec_id) 210 | if err != nil { 211 | fmt.Println(err) 212 | } 213 | }) 214 | locks.RepoSyncLock.Unlock() 215 | } else { 216 | err = repo.SaveDelState(&worker_id, &rec_id) 217 | if err != nil { 218 | fmt.Println(err) 219 | } 220 | } 221 | 222 | return nil 223 | } 224 | 225 | func InitSchema() error { 226 | conn, err := pgx.Connect(context.Background(), global.CliOpts.PrimaryPgDsn) 227 | if err != nil { 228 | return err 229 | } 230 | defer conn.Close(context.Background()) 231 | 232 | _, err = conn.Exec(context.Background(), catalog.SchemaSQL) 233 | if err != nil { 234 | return err 235 | } 236 | 237 | for i := 0; i < global.CliOpts.ParallelWorkers; i++ { 238 | _, err = conn.Exec(context.Background(), fmt.Sprintf(catalog.TableSQL, i)) 239 | if err != nil { 240 | return err 241 | } 242 | } 243 | return nil 244 | } 245 | 246 | func DropSchema() error { 247 | conn, err := pgx.Connect(context.Background(), global.CliOpts.PrimaryPgDsn) 248 | if err != nil { 249 | return err 250 | } 251 | defer conn.Close(context.Background()) 252 | 253 | _, err = conn.Exec(context.Background(), "DROP SCHEMA IF EXISTS ptor CASCADE") 254 | if err != nil { 255 | return err 256 | } 257 | 258 | return nil 259 | } 260 | 261 | func ClosePrimaryPool() error { 262 | return pool.PrimaryPool.ClosePrimaryPool() 263 | } 264 | 265 | func CloseRepoPool() error { 266 | return pool.RepoPool.CloseRepoPool() 267 | } 268 | 269 | func CheckPrimaryConnTime() (*time.Time, error) { 270 | 271 | ctx, _ := context.WithTimeout(context.Background(), time.Duration(global.CliOpts.RTOTimeout)*time.Millisecond) 272 | conn, err := pgx.Connect(ctx, global.CliOpts.PrimaryPgDsn) 273 | if err != nil { 274 | return nil, err 275 | } 276 | defer conn.Close(ctx) 277 | connTime := time.Now() 278 | return &connTime, nil 279 | } 280 | 281 | func MakeNewPrimaryReadWriteConn() (*time.Time, error) { 282 | 283 | ctx, _ := context.WithTimeout(context.Background(), time.Duration(global.CliOpts.RTOTimeout)*time.Millisecond) 284 | conn, err := pgx.Connect(ctx, global.CliOpts.PrimaryPgDsn) 285 | if err != nil { 286 | return nil, err 287 | } 288 | defer conn.Close(ctx) 289 | 290 | var isRecovery bool 291 | // Check whether the database is in read-write mode 292 | err = conn.QueryRow(context.Background(), "SELECT pg_is_in_recovery()").Scan(&isRecovery) 293 | if err != nil { 294 | return nil, err 295 | } 296 | 297 | if isRecovery { 298 | return nil, errors.New("database is in recovery mode") 299 | } 300 | 301 | primTime := time.Now() 302 | return &primTime, nil 303 | } 304 | 305 | func GetMaxDateFromPrimary(worker_id int) (time.Time, error) { 306 | conn, err := pgx.Connect(context.Background(), global.CliOpts.PrimaryPgDsn) 307 | if err != nil { 308 | return time.Time{}, err 309 | } 310 | defer conn.Close(context.Background()) 311 | 312 | var maxDate time.Time 313 | err = conn.QueryRow(context.Background(), fmt.Sprintf("SELECT COALESCE(max(last_update), '12-12-1212 12:12:12') FROM ptor.worker_%d", worker_id)).Scan(&maxDate) 314 | if err != nil { 315 | return time.Time{}, err 316 | } 317 | 318 | return maxDate, nil 319 | } 320 | 321 | func GetPrimaryRowCount(worker_id int) (int, error) { 322 | conn, err := pgx.Connect(context.Background(), global.CliOpts.PrimaryPgDsn) 323 | if err != nil { 324 | return 0, err 325 | } 326 | defer conn.Close(context.Background()) 327 | 328 | var rowCount int 329 | err = conn.QueryRow(context.Background(), fmt.Sprintf("SELECT count(*) FROM ptor.worker_%d", worker_id)).Scan(&rowCount) 330 | if err != nil { 331 | return 0, err 332 | } 333 | 334 | return rowCount, nil 335 | } 336 | 337 | func GetPrimaryRelationCheckSum(worker_id int) (string, error) { 338 | conn, err := pgx.Connect(context.Background(), global.CliOpts.PrimaryPgDsn) 339 | if err != nil { 340 | return "", err 341 | } 342 | defer conn.Close(context.Background()) 343 | 344 | var checkSum string 345 | err = conn.QueryRow(context.Background(), fmt.Sprintf("WITH sort_data AS (SELECT id||trim(t)||last_update as data FROM ptor.worker_%d ORDER BY last_update DESC) SELECT COALESCE(md5(array_agg(sort_data.data)::text),'') FROM sort_data;", worker_id)).Scan(&checkSum) 346 | if err != nil { 347 | return "", err 348 | } 349 | 350 | return checkSum, nil 351 | } 352 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/Delta456/box-cli-maker v1.3.2 h1:p3LJ67tYrkWOqd3Xju1rYWxx7cRcfHL42poOxdmOLi8= 3 | github.com/Delta456/box-cli-maker v1.3.2/go.mod h1:MsanLNTlPCfUdYyhfPY4Aond1NbPGnRxJiAWFcDBIsA= 4 | github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= 5 | github.com/alexflint/go-arg v1.4.3 h1:9rwwEBpMXfKQKceuZfYcwuc/7YY7tWJbFsgG5cAU/uo= 6 | github.com/alexflint/go-arg v1.4.3/go.mod h1:3PZ/wp/8HuqRZMUUgu7I+e1qcpUbvmS258mRXkFH4IA= 7 | github.com/alexflint/go-scalar v1.1.0 h1:aaAouLLzI9TChcPXotr6gUhq+Scr8rl0P9P4PnltbhM= 8 | github.com/alexflint/go-scalar v1.1.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o= 9 | github.com/briandowns/spinner v1.21.0 h1:2lVBzf3iZ3cT/ulVXljc4BzlL3yTWZDzsGsamI7si+A= 10 | github.com/briandowns/spinner v1.21.0/go.mod h1:TcwZHb7Wb6vn/+bcVv1UXEzaA4pLS7yznHlkY/HzH44= 11 | github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= 12 | github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 13 | github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 14 | github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= 15 | github.com/dariubs/percent v1.0.0 h1:fY8q40FRYaCiFZ0gTOa73Cmp21hS32w+tSSmqbGnUzc= 16 | github.com/dariubs/percent v1.0.0/go.mod h1:NDZpkezJ8QqyIW/510MywB5T2KdC8v/0oTlEoPcMsRM= 17 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 18 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 19 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 20 | github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= 21 | github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= 22 | github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= 23 | github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= 24 | github.com/gammazero/deque v0.2.0 h1:SkieyNB4bg2/uZZLxvya0Pq6diUlwx7m2TeT7GAIWaA= 25 | github.com/gammazero/deque v0.2.0/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= 26 | github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q= 27 | github.com/gammazero/workerpool v1.1.3/go.mod h1:wPjyBLDbyKnUn2XwwyD3EEwo9dHutia9/fwNmSHWACc= 28 | github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= 29 | github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= 30 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 31 | github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= 32 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 33 | github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= 34 | github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= 35 | github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= 36 | github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= 37 | github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= 38 | github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= 39 | github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= 40 | github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= 41 | github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= 42 | github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= 43 | github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= 44 | github.com/jackc/pgconn v1.13.0 h1:3L1XMNV2Zvca/8BYhzcRFS70Lr0WlDg16Di6SFGAbys= 45 | github.com/jackc/pgconn v1.13.0/go.mod h1:AnowpAqO4CMIIJNZl2VJp+KrkAZciAkhEl0W0JIobpI= 46 | github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= 47 | github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= 48 | github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= 49 | github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= 50 | github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= 51 | github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= 52 | github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= 53 | github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= 54 | github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= 55 | github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= 56 | github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= 57 | github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= 58 | github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= 59 | github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= 60 | github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= 61 | github.com/jackc/pgproto3/v2 v2.3.1 h1:nwj7qwf0S+Q7ISFfBndqeLwSwxs+4DPsbRFjECT1Y4Y= 62 | github.com/jackc/pgproto3/v2 v2.3.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= 63 | github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= 64 | github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= 65 | github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= 66 | github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= 67 | github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= 68 | github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= 69 | github.com/jackc/pgtype v1.12.0 h1:Dlq8Qvcch7kiehm8wPGIW0W3KsCCHJnRacKW0UM8n5w= 70 | github.com/jackc/pgtype v1.12.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= 71 | github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o= 72 | github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= 73 | github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= 74 | github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= 75 | github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= 76 | github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= 77 | github.com/jackc/pgx/v4 v4.17.2 h1:0Ut0rpeKwvIVbMQ1KbMBU4h6wxehBI535LK6Flheh8E= 78 | github.com/jackc/pgx/v4 v4.17.2/go.mod h1:lcxIZN44yMIrWI78a5CpucdD14hX0SBDbNRvjDBItsw= 79 | github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= 80 | github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= 81 | github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= 82 | github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= 83 | github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= 84 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 85 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 86 | github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 87 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 88 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 89 | github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= 90 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 91 | github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 92 | github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 93 | github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 94 | github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 95 | github.com/lossdev/stack v1.0.5 h1:/kaKhI0Reoi7y7hLj2toLpDHNpobA9iUp3GzEj/T6TE= 96 | github.com/lossdev/stack v1.0.5/go.mod h1:hfhRy+k8UG39xygWSQwuguM4KGSxzM6rlgGFCJd2I+4= 97 | github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= 98 | github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 99 | github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 100 | github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= 101 | github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= 102 | github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= 103 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 104 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 105 | github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 106 | github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 107 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 108 | github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= 109 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= 110 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 111 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 112 | github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= 113 | github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 114 | github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= 115 | github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= 116 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 117 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 118 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 119 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 120 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 121 | github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= 122 | github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= 123 | github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= 124 | github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= 125 | github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= 126 | github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= 127 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= 128 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 129 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 130 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 131 | github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= 132 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 133 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 134 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 135 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 136 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 137 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 138 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 139 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 140 | github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= 141 | go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 142 | go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 143 | go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= 144 | go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= 145 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 146 | go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= 147 | go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= 148 | go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= 149 | go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 150 | go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 151 | go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= 152 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 153 | golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= 154 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 155 | golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 156 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 157 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 158 | golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= 159 | golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 160 | golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 161 | golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 162 | golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= 163 | golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= 164 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 165 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 166 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 167 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 168 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 169 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 170 | golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 171 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 172 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 173 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 174 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 175 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 176 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 177 | golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 178 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 179 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 180 | golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 181 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 182 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 183 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 184 | golang.org/x/sys v0.0.0-20200724161237-0e2f3a69832c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 185 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 186 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 187 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 188 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 189 | golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= 190 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 191 | golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= 192 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 193 | golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 194 | golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= 195 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 196 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 197 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 198 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 199 | golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 200 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 201 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 202 | golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= 203 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 204 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 205 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 206 | golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 207 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 208 | golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 209 | golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 210 | golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 211 | golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 212 | golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 213 | golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 214 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 215 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 216 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 217 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 218 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 219 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 220 | gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= 221 | gopkg.in/karalabe/cookiejar.v2 v2.0.0-20150724131613-8dcd6a7f4951 h1:DMTcQRFbEH62YPRWwOI647s2e5mHda3oBPMHfrLs2bw= 222 | gopkg.in/karalabe/cookiejar.v2 v2.0.0-20150724131613-8dcd6a7f4951/go.mod h1:owOxCRGGeAx1uugABik6K9oeNu1cgxP/R9ItzLDxNWA= 223 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 224 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 225 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 226 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 227 | --------------------------------------------------------------------------------