├── .circleci └── config.yml ├── .github └── workflows │ └── build-integration.yml ├── .gitignore ├── .goreleaser.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── VERSION ├── cmd └── pgreplay │ └── main.go ├── compose.yml ├── dev.dockerfile ├── go.mod ├── go.sum ├── pkg └── pgreplay │ ├── database.go │ ├── integration │ ├── integration_test.go │ ├── suite_test.go │ └── testdata │ │ ├── single_user.go │ │ ├── single_user.json │ │ ├── single_user.log │ │ └── structure.sql │ ├── parse.go │ ├── parse_test.go │ ├── prom_server.go │ ├── streamer.go │ ├── suite_test.go │ ├── types.go │ └── types_test.go └── res ├── grafana-dashboard-pgreplay-go.json └── grafana.jpg /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2 3 | 4 | references: 5 | docker_golang: &docker_golang 6 | docker: 7 | - image: golang:1.13 8 | environment: 9 | PGHOST: "127.0.0.1" 10 | PGUSER: "postgres" 11 | - image: postgres:11.2 12 | environment: 13 | POSTGRES_USER: postgres 14 | POSTGRES_DB: pgreplay_test 15 | POSTGRES_PASSWORD: "" 16 | working_directory: /go/src/github.com/gocardless/pgreplay-go 17 | 18 | jobs: 19 | unit-integration: 20 | <<: *docker_golang 21 | steps: 22 | - checkout 23 | - run: 24 | name: Install ginkgo test runner 25 | command: go get github.com/onsi/ginkgo/ginkgo 26 | - run: 27 | name: Install Postgres 28 | command: apt-get update && apt-get install -y postgresql-client libpq-dev 29 | - run: 30 | name: Create test database 31 | command: make recreatedb 32 | - run: 33 | name: Run tests 34 | command: ginkgo -race -r 35 | 36 | release: 37 | <<: *docker_golang 38 | working_directory: /go/src/github.com/gocardless/pgreplay-go 39 | steps: 40 | - checkout 41 | - run: 42 | name: Release 43 | command: | 44 | CURRENT_VERSION="v$(cat VERSION)" 45 | 46 | if [[ $(git tag -l "${CURRENT_VERSION}") == "${CURRENT_VERSION}" ]]; then 47 | echo "Version ${CURRENT_VERSION} is already released" 48 | exit 0 49 | fi 50 | 51 | curl -L -o /tmp/goreleaser_Linux_x86_64.tar.gz https://github.com/goreleaser/goreleaser/releases/download/v0.101.0/goreleaser_Linux_x86_64.tar.gz 52 | tar zxf /tmp/goreleaser_Linux_x86_64.tar.gz -C /tmp 53 | 54 | git log --pretty=oneline --abbrev-commit --no-decorate --no-color "$(git describe --tags --abbrev=0)..HEAD" -- pkg cmd vendor internal > /tmp/release-notes 55 | git tag "${CURRENT_VERSION}" 56 | git push --tags 57 | 58 | /tmp/goreleaser --rm-dist --release-notes /tmp/release-notes 59 | 60 | workflows: 61 | version: 2 62 | build-integration: 63 | jobs: 64 | - unit-integration 65 | - release: 66 | requires: 67 | - unit-integration 68 | filters: 69 | branches: {only: master} 70 | -------------------------------------------------------------------------------- /.github/workflows/build-integration.yml: -------------------------------------------------------------------------------- 1 | name: gocardless/pgreplay-go/build-integration 2 | on: 3 | push: 4 | branches: 5 | - master 6 | env: 7 | GITHUB_TOKEN: xxxx599f 8 | jobs: 9 | unit-integration: 10 | defaults: 11 | run: 12 | working-directory: "/go/src/github.com/gocardless/pgreplay-go" 13 | runs-on: ubuntu-latest 14 | container: 15 | image: golang:1.21.4 16 | env: 17 | PGHOST: 127.0.0.1 18 | PGUSER: postgres 19 | services: 20 | postgres: 21 | image: postgres:14 22 | env: 23 | POSTGRES_USER: postgres 24 | POSTGRES_DB: pgreplay_test 25 | POSTGRES_PASSWORD: '' 26 | steps: 27 | - uses: actions/checkout@v4.1.1 28 | - name: Install ginkgo test runner 29 | run: go get github.com/onsi/ginkgo/ginkgo 30 | - name: Install Postgres 31 | run: apt-get update && apt-get install -y postgresql-client libpq-dev 32 | - name: Create test database 33 | run: make recreatedb 34 | - name: Run tests 35 | run: ginkgo -race -r 36 | release: 37 | if: github.ref == 'refs/heads/master' 38 | defaults: 39 | run: 40 | working-directory: "/go/src/github.com/gocardless/pgreplay-go" 41 | runs-on: ubuntu-latest 42 | container: 43 | image: golang:1.21.4 44 | env: 45 | PGHOST: 127.0.0.1 46 | PGUSER: postgres 47 | services: 48 | postgres: 49 | image: postgres:14 50 | env: 51 | POSTGRES_USER: postgres 52 | POSTGRES_DB: pgreplay_test 53 | POSTGRES_PASSWORD: '' 54 | needs: 55 | - unit-integration 56 | steps: 57 | - uses: actions/checkout@v3.1.0 58 | - name: Release 59 | run: |- 60 | CURRENT_VERSION="v$(cat VERSION)" 61 | if [[ $(git tag -l "${CURRENT_VERSION}") == "${CURRENT_VERSION}" ]]; then 62 | echo "Version ${CURRENT_VERSION} is already released" 63 | exit 0 64 | fi 65 | curl -L -o /tmp/goreleaser_Linux_x86_64.tar.gz https://github.com/goreleaser/goreleaser/releases/download/v0.101.0/goreleaser_Linux_x86_64.tar.gz 66 | tar zxf /tmp/goreleaser_Linux_x86_64.tar.gz -C /tmp 67 | git log --pretty=oneline --abbrev-commit --no-decorate --no-color "$(git describe --tags --abbrev=0)..HEAD" -- pkg cmd vendor internal > /tmp/release-notes 68 | git tag "${CURRENT_VERSION}" 69 | git push --tags 70 | /tmp/goreleaser --rm-dist --release-notes /tmp/release-notes 71 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | bin/ 2 | dist/ 3 | vendor/ 4 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | # View goreleaser docs for configuration settings 2 | # https://goreleaser.com 3 | 4 | --- 5 | project_name: pgreplay 6 | 7 | builds: 8 | - binary: pgreplay 9 | main: cmd/pgreplay/main.go 10 | goos: 11 | - darwin 12 | - linux 13 | goarch: 14 | - amd64 15 | ldflags: > 16 | -X main.Version={{.Version}} 17 | -X main.Commit={{.Commit}} 18 | -X main.Date={{.Date}} 19 | -a 20 | -installsuffix cgo 21 | env: 22 | - CGO_ENABLED=0 23 | 24 | nfpm: 25 | name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" 26 | vendor: GoCardless 27 | homepage: https://github.com/gocardless/pgreplay-go 28 | maintainer: GoCardless Engineering 29 | description: &description PostgreSQL load testing tool 30 | formats: 31 | - deb 32 | 33 | brew: 34 | github: 35 | owner: gocardless 36 | name: homebrew-taps 37 | commit_author: 38 | name: GoCardless Engineering 39 | email: engineering@gocardless.com 40 | folder: Formula 41 | homepage: https://github.com/gocardless/pgreplay-go 42 | description: *description 43 | test: system "#{bin}/pgreplay version" 44 | install: bin.install "pgreplay" 45 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.21.4-alpine AS build-stage 2 | 3 | WORKDIR /app 4 | 5 | COPY go.mod go.sum ./ 6 | RUN go mod download 7 | 8 | COPY ./ ./ 9 | 10 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ./pgreplay ./cmd/pgreplay/main.go 11 | 12 | # Deploy the application binary into a lean image 13 | FROM alpine:latest 14 | RUN adduser -D pgreplay-user 15 | 16 | COPY --from=build-stage /app/pgreplay /bin/pgreplay 17 | 18 | USER pgreplay-user 19 | 20 | CMD [ "sh" ] 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2018-2019 GoCardless, Ltd. https://gocardless.com 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROG=bin/pgreplay 2 | PROJECT=github.com/gocardless/pgreplay-go 3 | VERSION=$(shell git rev-parse --short HEAD)-dev 4 | BUILD_COMMAND=go build -ldflags "-X main.Version=$(VERSION)" 5 | DB_CONN_CONFIG=-h localhost -p 5432 -U postgres 6 | DOCKER_CONN_CONFIG=-h postgres -p 5432 -U postgres 7 | 8 | .PHONY: all darwin linux test clean 9 | 10 | all: darwin linux 11 | darwin: $(PROG) 12 | linux: $(PROG:=.linux_amd64) 13 | 14 | bin/%.linux_amd64: 15 | CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(BUILD_COMMAND) -a -o $@ cmd/$*/main.go 16 | 17 | bin/%: 18 | $(BUILD_COMMAND) -o $@ cmd/$*/main.go 19 | 20 | createdb: 21 | psql $(DB_CONN_CONFIG) -d postgres -c "CREATE DATABASE pgreplay_test;" 22 | 23 | dropdb: 24 | psql $(DB_CONN_CONFIG) -c "DROP DATABASE IF EXISTS pgreplay_test;" 25 | 26 | structure: 27 | psql $(DB_CONN_CONFIG) -d pgreplay_test -f pkg/pgreplay/integration/testdata/structure.sql 28 | 29 | recreatedb: dropdb createdb structure 30 | 31 | createdbdocker: 32 | psql $(DOCKER_CONN_CONFIG) -c "DROP DATABASE IF EXISTS pgreplay_test;" 33 | psql $(DOCKER_CONN_CONFIG) -d postgres -c "CREATE DATABASE pgreplay_test;" 34 | psql $(DOCKER_CONN_CONFIG) -d pgreplay_test -f pkg/pgreplay/integration/testdata/structure.sql 35 | 36 | # go get -u github.com/onsi/ginkgo/ginkgo 37 | test: 38 | ginkgo -v -r 39 | 40 | clean: 41 | rm -rvf $(PROG) $(PROG:%=%.linux_amd64) 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pgreplay-go [![CircleCI](https://circleci.com/gh/gocardless/pgreplay-go.svg?style=svg&circle-token=d020aaec823388b8e4debe552960450402964ae7)](https://circleci.com/gh/gocardless/pgreplay-go) 2 | 3 | > See a discussion of building this tool at https://blog.lawrencejones.dev/building-a-postgresql-load-tester/ 4 | 5 | This tool is a different take on the existing [pgreplay]( 6 | https://github.com/laurenz/pgreplay) project. Where pgreplay will playback 7 | Postgres log files while respecting relative chronological order, pgreplay-go 8 | plays back statements at approximately the same rate they were originally sent 9 | to the database. 10 | 11 | When benchmarking database performance, it is better for a replay tool to 12 | continue sending traffic than to halt execution until a strangling query has 13 | complete. If your new cluster isn't performing, users won't politely wait for 14 | your service to catch-up before issuing new commands. 15 | 16 | ## Benchmark strategy 17 | 18 | You have an existing cluster and want to trial new hardware/validate 19 | configuration changes/move infrastructure providers. Your production services 20 | depend on ensuring the new change is safe and doesn't degrade performance. 21 | 22 | 23 | ## Building this tool 24 | Ensure you have go installed (`brew install go`). 25 | 26 | ```bash 27 | $ make all 28 | ``` 29 | This will put `pgreplay` in `/bin`, if you need to rebuild delete the contents of `/bin` & remake. 30 | 31 | ### 1. Configure production logging 32 | 33 | First capture the logs from your running Postgres instance. You may need to add 34 | additional storage to your machine before starting- we often attach a new disk 35 | and change our log location to that disk for experiments. 36 | 37 | You can turn on logging in a compatible format like so: 38 | 39 | ```sql 40 | ALTER SYSTEM SET log_directory='/postgres-logs'; 41 | ALTER SYSTEM SET log_connections='on'; 42 | ALTER SYSTEM SET log_disconnections='on'; 43 | ALTER SYSTEM SET log_line_prefix='%m|%u|%d|%c|'; 44 | ALTER SYSTEM SET log_min_error_statement='log'; 45 | ALTER SYSTEM SET log_min_messages='error'; 46 | ALTER SYSTEM SET log_statement='all'; 47 | ALTER SYSTEM SET log_min_duration_statement=0; 48 | SELECT pg_reload_conf(); 49 | ``` 50 | 51 | Or, if you need to capture logs for an RDS instance, you can use these parameters in your 52 | instances parameter group: 53 | 54 | ``` 55 | log_destination = csvlog 56 | log_connections = 1 57 | log_disconnections = 1 58 | log_min_error_statement = log 59 | log_min_messages = error 60 | log_statement = all 61 | log_min_duration_statement = 0 62 | ``` 63 | 64 | ### 2. Take snapshot 65 | 66 | Now we're emitting logs we need to snapshot the database so that we can later 67 | restore it to the same moment in time on our benchmark clusters. If you're 68 | running in a cloud provider with disk snapshot facilities then this is likely 69 | the easiest of options (remember to checkpoint first, to reduce recovery time) 70 | but you can also achieve this using `pg_basebackup` and point-in-time recovery 71 | configuration, or by taking a physical copy of a paused replica. 72 | 73 | Whatever method used must produce an image that can be restored into new 74 | machines later. 75 | 76 | ### 3. Extract and process logs 77 | 78 | Once you've captured logs for your desired benchmark window, you can optionally 79 | pre-process them to create a more realistic sample. Complex database 80 | interactions are likely to have transactions that may fail when we play them 81 | back out-of-order. Recalling that our primary goal is to create representative 82 | load on the database, not a totally faithful replay, we suggest applying the 83 | following filters: 84 | 85 | ``` 86 | $ cat postgresql.log \ 87 | | pv --progress --rate --size "$(du postgresql.log | cut -f1)" \ 88 | | grep -v "LOG: statement: BEGIN" \ 89 | | grep -v "LOG: statement: COMMIT" \ 90 | | grep -v "LOG: statement: ROLLBACK TO SAVEPOINT" \ 91 | | grep -v "LOG: statement: SAVEPOINT" \ 92 | | grep -v "LOG: statement: RELEASE SAVEPOINT" \ 93 | | grep -v "LOG: statement: SET LOCAL" \ 94 | | sed 's/pg_try_advisory_lock/bool/g' \ 95 | | sed 's/pg_advisory_unlock/pg_advisory_unlock_shared/g' \ 96 | > postgresql-filtered.log 97 | ``` 98 | 99 | By removing transactions we avoid skipping work if any of the transaction 100 | queries were to fail - the same goes for savepoints. 101 | We remove any `SET LOCAL` statements, as having removed transactions these 102 | configuration settings would be present for the duration of the connection. 103 | We then modify any advisory lock queries to be shared, preventing us from 104 | needlessly blocking while still requiring the database to perform similar levels 105 | of work as the exclusive locking. 106 | 107 | These transformations mean our replayed queries won't exactly simulate what we 108 | saw in production, but that's why we'll compare the performance of these 109 | filtered logs against the two clusters rather than the original performance of 110 | the production cluster. 111 | 112 | ### 4. pgreplay-go against copy of production cluster 113 | 114 | Now create a copy of the original production cluster using the snapshot from 115 | (2). The aim is to have a cluster that exactly replicates production, providing 116 | a reliable control for our experiment. 117 | 118 | The goal of this run will be to output Postgres logs that can be parsed by 119 | [pgBadger](https://github.com/darold/pgbadger) to provide an analysis of the 120 | benchmark run. See the pgBadger readme for details, or apply the following 121 | configuration for defaults that will work: 122 | 123 | ```sql 124 | ALTER SYSTEM SET log_min_duration_statement = 0; 125 | ALTER SYSTEM SET log_line_prefix = '%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h ' 126 | ALTER SYSTEM SET log_checkpoints = on; 127 | ALTER SYSTEM SET log_connections = on; 128 | ALTER SYSTEM SET log_disconnections = on; 129 | ALTER SYSTEM SET log_lock_waits = on; 130 | ALTER SYSTEM SET log_temp_files = 0; 131 | ALTER SYSTEM SET log_autovacuum_min_duration = 0; 132 | ALTER SYSTEM SET log_error_verbosity = default; 133 | SELECT pg_reload_conf(); 134 | ``` 135 | 136 | Run the benchmark using the pgreplay-go binary from the primary Postgres machine 137 | under test. pgreplay-go requires minimal system resource and is unlikely to 138 | affect the benchmark when running on the same machine, though you can run it 139 | from an alternative location if this is a concern. An example run would look 140 | like this: 141 | 142 | ``` 143 | $ pgreplay-go/bin/pgreplay run \ 144 | --debug \ 145 | --errlog-input ./postgresql-filtered.log \ 146 | --host 127.0.0.1 \ 147 | --metrics-address 0.0.0.0 \ 148 | --port 5433 \ 149 | --user postgres \ 150 | --password postgres \ 151 | --start 2023-07-25\ 03:10:05.000\ UTC \ 152 | --finish 2024-01-01\ 15:04:05.000\ UTC 153 | ``` 154 | 155 | If you run Prometheus then pgreplay-go exposes a metrics that can be used to 156 | report progress on the benchmark. See [Observability](#observability) for more 157 | details. 158 | 159 | Once the benchmark is complete, store the logs somewhere for safekeeping. We 160 | usually upload the logs to Google cloud storage, though you should use whatever 161 | service you are most familiar with. 162 | 163 | ### 5. pgreplay-go against new cluster 164 | 165 | After provisioning your new candidate cluster, with the hardware/software 166 | changes you wish to test, we repeat step (4) for our new cluster. You should use 167 | exactly the same pgreplay logs and run the benchmark for the same time-window. 168 | 169 | As in step (4), upload your logs in preparation for the next step. 170 | 171 | ### 6. User pgBadger to compare performance 172 | 173 | We use pgBadger to perform analysis of our performance during the benchmark, 174 | along with taking measurements from the 175 | [node_exporter](https://github.com/prometheus/node_exporter) and 176 | [postgres_exporter](https://github.com/rnaveiras/postgres_exporter) running 177 | during the experiment. 178 | 179 | pgBadger reports can be used to calculate a query duration histogram for both 180 | clusters - these can indicate general speed-up/degradation. Digging into 181 | specific queries and the worst performers can provide more insight into which 182 | type of queries have degraded, and explaining those query plans on your clusters 183 | can help indicate what might have caused the change. 184 | 185 | This is the least prescriptive part of our experiment, and answering whether the 186 | performance changes are acceptable - and what they may be - will depend on your 187 | knowledge of the applications using your database. We've found pgBadger to 188 | provide sufficient detail for us to be confident in answering this question, and 189 | hope you do too. 190 | 191 | ## Observability 192 | 193 | Running benchmarks can be a long process. pgreplay-go provides Prometheus 194 | metrics that can help determine how far through the benchmark has progressed, 195 | along with estimating how long remains. 196 | 197 | Hooking these metrics into a Grafana dashboard can give the following output: 198 | 199 | ![pgreplay-go Grafana dashboard](res/grafana.jpg) 200 | 201 | We'd suggest that you integrate these panels into your own dashboard; for 202 | example by showing them alongside key PostgreSQL metrics such as transaction 203 | commits, active backends and buffer cache hit ratio, as well as node-level 204 | metrics to show CPU and IO saturation. 205 | 206 | A sample dashboard with the `pgreplay-go`-specific panels has been provided that 207 | may help you get started. Import it into your Grafana dashboard by downloading 208 | the [dashboard JSON file](res/grafana-dashboard-pgreplay-go.json). 209 | 210 | ## Types of Log 211 | 212 | ### Simple 213 | 214 | This is a basic query that is executed directly against Postgres with no 215 | other steps. 216 | 217 | ``` 218 | 2010-12-31 10:59:52.243 UTC|postgres|postgres|4d1db7a8.4227|LOG: statement: set client_encoding to 'LATIN9' 219 | ``` 220 | 221 | ### Prepared statement 222 | 223 | ``` 224 | 2010-12-31 10:59:57.870 UTC|postgres|postgres|4d1db7a8.4227|LOG: execute einf"ug: INSERT INTO runtest (id, c, t, b) VALUES ($1, $2, $3, $4) 225 | 2010-12-31 10:59:57.870 UTC|postgres|postgres|4d1db7a8.4227|DETAIL: parameters: $1 = '6', $2 = 'mit Tabulator', $3 = '2050-03-31 22:00:00+00', $4 = NULL 226 | ``` 227 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.1.3 2 | -------------------------------------------------------------------------------- /cmd/pgreplay/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | stdlog "log" 8 | "os" 9 | "runtime" 10 | "time" 11 | 12 | kingpin "github.com/alecthomas/kingpin/v2" 13 | kitlog "github.com/go-kit/log" 14 | "github.com/go-kit/log/level" 15 | "github.com/gocardless/pgreplay-go/pkg/pgreplay" 16 | "github.com/pkg/errors" 17 | ) 18 | 19 | var logger kitlog.Logger 20 | 21 | var ( 22 | app = kingpin.New("pgreplay", "Replay Postgres logs against database").Version(versionStanza()) 23 | 24 | // Global flags applying to every command 25 | debug = app.Flag("debug", "Enable debug logging").Default("false").Bool() 26 | startFlag = app.Flag("start", "Play logs from this time onward ("+pgreplay.PostgresTimestampFormat+")").String() 27 | finishFlag = app.Flag("finish", "Stop playing logs at this time ("+pgreplay.PostgresTimestampFormat+")").String() 28 | metricsAddress = app.Flag("metrics-address", "Address to bind HTTP metrics listener").Default("0.0.0.0").String() 29 | metricsPort = app.Flag("metrics-port", "Port to bind HTTP metrics listener").Default("9445").Uint16() 30 | 31 | filter = app.Command("filter", "Process an errlog file into a pgreplay preprocessed JSON log") 32 | filterJsonInput = filter.Flag("json-input", "JSON input file").ExistingFile() 33 | filterErrlogInput = filter.Flag("errlog-input", "Postgres errlog input file").ExistingFile() 34 | filterCsvLogInput = filter.Flag("csvlog-input", "Postgres CSV log input file").ExistingFile() 35 | filterOutput = filter.Flag("output", "JSON output file").String() 36 | filterNullOutput = filter.Flag("null-output", "Don't output anything, for testing parsing only").Bool() 37 | 38 | run = app.Command("run", "Replay from log files against a real database") 39 | runHost = run.Flag("host", "PostgreSQL database host").Required().String() 40 | runPort = run.Flag("port", "PostgreSQL database port").Default("5432").Uint16() 41 | runDatname = run.Flag("database", "PostgreSQL root database").Default("postgres").String() 42 | runUser = run.Flag("user", "PostgreSQL root user").Default("postgres").String() 43 | runPassword = run.Flag("password", "PostgreSQl password user (the default value is obtained from the DB_PASSWORD env var)").Default(os.Getenv("DB_PASSWORD")).String() 44 | runReplayRate = run.Flag("replay-rate", "Rate of playback, will execute queries at Nx speed").Default("1").Float() 45 | runErrlogInput = run.Flag("errlog-input", "Path to PostgreSQL errlog").ExistingFile() 46 | runCsvLogInput = run.Flag("csvlog-input", "Path to PostgreSQL CSV log").ExistingFile() 47 | runJsonInput = run.Flag("json-input", "Path to preprocessed pgreplay JSON log file").ExistingFile() 48 | ) 49 | 50 | func main() { 51 | command := kingpin.MustParse(app.Parse(os.Args[1:])) 52 | 53 | logger = kitlog.NewLogfmtLogger(kitlog.NewSyncWriter(os.Stderr)) 54 | logger = kitlog.With(logger, "ts", kitlog.DefaultTimestampUTC, "caller", kitlog.DefaultCaller) 55 | stdlog.SetOutput(kitlog.NewStdlibAdapter(logger)) 56 | 57 | if *debug { 58 | logger = level.NewFilter(logger, level.AllowDebug()) 59 | } else { 60 | logger = level.NewFilter(logger, level.AllowInfo()) 61 | } 62 | 63 | // Starting the Prometheus Server 64 | server := pgreplay.StartPrometheusServer(logger, *metricsAddress, *metricsPort) 65 | 66 | var err error 67 | var start, finish *time.Time 68 | 69 | if start, err = parseTimestamp(*startFlag); err != nil { 70 | kingpin.Fatalf("--start flag %s", err) 71 | } 72 | 73 | if finish, err = parseTimestamp(*finishFlag); err != nil { 74 | kingpin.Fatalf("--finish flag %s", err) 75 | } 76 | 77 | switch command { 78 | case filter.FullCommand(): 79 | var items chan pgreplay.Item 80 | 81 | switch checkSingleFormat(filterJsonInput, filterErrlogInput, filterCsvLogInput) { 82 | case filterJsonInput: 83 | items = parseLog(*filterJsonInput, pgreplay.ParseJSON) 84 | case filterErrlogInput: 85 | items = parseLog(*filterErrlogInput, pgreplay.ParseErrlog) 86 | case filterCsvLogInput: 87 | items = parseLog(*filterCsvLogInput, pgreplay.ParseCsvLog) 88 | default: 89 | logger.Log("event", "postgres.error", "error", "you must provide an input") 90 | os.Exit(255) 91 | } 92 | 93 | // Apply the start and end filters 94 | items = pgreplay.NewStreamer(start, finish, logger).Filter(items) 95 | 96 | if *filterNullOutput { 97 | logger.Log("event", "filter.null_output", "msg", "Null output enabled, logs won't be serialized") 98 | for range items { 99 | // no-op 100 | } 101 | 102 | return 103 | } 104 | 105 | if *filterOutput == "" { 106 | kingpin.Fatalf("must provide output file when no --null-output") 107 | } 108 | 109 | outputFile, err := os.Create(*filterOutput) 110 | if err != nil { 111 | kingpin.Fatalf("failed to create output file: %v", err) 112 | } 113 | 114 | // Buffer the writes by 32MB to enable much faster filtering 115 | buffer := bufio.NewWriterSize(outputFile, 32*1000*1000) 116 | 117 | for item := range items { 118 | bytes, err := pgreplay.ItemMarshalJSON(item) 119 | if err != nil { 120 | kingpin.Fatalf("failed to serialize item: %v", err) 121 | } 122 | 123 | if _, err := buffer.Write(append(bytes, byte('\n'))); err != nil { 124 | kingpin.Fatalf("failed to write to output file: %v", err) 125 | } 126 | } 127 | 128 | buffer.Flush() 129 | outputFile.Close() 130 | 131 | case run.FullCommand(): 132 | ctx := context.Background() 133 | database, err := pgreplay.NewDatabase( 134 | ctx, 135 | pgreplay.DatabaseConnConfig{ 136 | Host: *runHost, 137 | Port: *runPort, 138 | Database: *runDatname, 139 | User: *runUser, 140 | Password: *runPassword, 141 | }, 142 | ) 143 | 144 | if err != nil { 145 | logger.Log("event", "postgres.error", "error", err) 146 | os.Exit(255) 147 | } 148 | 149 | var items chan pgreplay.Item 150 | 151 | switch checkSingleFormat(runJsonInput, runErrlogInput, runCsvLogInput) { 152 | case runJsonInput: 153 | items = parseLog(*runJsonInput, pgreplay.ParseJSON) 154 | case runErrlogInput: 155 | items = parseLog(*runErrlogInput, pgreplay.ParseErrlog) 156 | case runCsvLogInput: 157 | items = parseLog(*runCsvLogInput, pgreplay.ParseCsvLog) 158 | default: 159 | logger.Log("event", "postgres.error", "error", "you must provide an input") 160 | os.Exit(255) 161 | } 162 | 163 | replay_started := time.Now() 164 | stream, err := pgreplay.NewStreamer(start, finish, logger).Stream(items, *runReplayRate) 165 | if err != nil { 166 | kingpin.Fatalf("failed to start streamer: %s", err) 167 | } 168 | 169 | errs, done := database.Consume(ctx, stream) 170 | 171 | var status int 172 | 173 | for { 174 | select { 175 | case err := <-errs: 176 | if err != nil { 177 | logger.Log("event", "consume.error", "error", err) 178 | } 179 | case err := <-done: 180 | if err != nil { 181 | status = 255 182 | } 183 | 184 | logger.Log("event", "consume.finished", "error", err, "status", status) 185 | logger.Log("event", "time.elapsed", "total", buildTimeElapsed(replay_started)) 186 | logger.Log("event", "server.status", "message", "shutting down the server!") 187 | err = pgreplay.ShutdownServer(ctx, server) 188 | if err != nil { 189 | logger.Log("error", "server.shutdown", "message", err.Error()) 190 | } 191 | 192 | os.Exit(status) 193 | } 194 | } 195 | } 196 | } 197 | 198 | // Set by goreleaser 199 | var ( 200 | Version = "dev" 201 | Commit = "none" 202 | Date = "unknown" 203 | GoVersion = runtime.Version() 204 | ) 205 | 206 | func versionStanza() string { 207 | return fmt.Sprintf( 208 | "pgreplay Version: %v\nGit SHA: %v\nGo Version: %v\nGo OS/Arch: %v/%v\nBuilt at: %v", 209 | Version, Commit, GoVersion, runtime.GOOS, runtime.GOARCH, Date, 210 | ) 211 | } 212 | 213 | func checkSingleFormat(formats ...*string) (result *string) { 214 | var supplied = 0 215 | for _, format := range formats { 216 | if *format != "" { 217 | result = format 218 | supplied++ 219 | } 220 | } 221 | 222 | if supplied != 1 { 223 | kingpin.Fatalf("must provide exactly one input format") 224 | } 225 | 226 | return result // which becomes the one that isn't empty 227 | } 228 | 229 | func parseLog(path string, parser pgreplay.ParserFunc) chan pgreplay.Item { 230 | file, err := os.Open(path) 231 | if err != nil { 232 | kingpin.Fatalf("failed to open logfile: %s", err) 233 | } 234 | 235 | items, logerrs, done := parser(file) 236 | 237 | go func() { 238 | logger.Log("event", "parse.finished", "error", <-done) 239 | }() 240 | 241 | go func() { 242 | for err := range logerrs { 243 | level.Debug(logger).Log("event", "parse.error", "error", err) 244 | } 245 | }() 246 | 247 | return items 248 | } 249 | 250 | // parseTimestamp parsed a Postgres friendly timestamp 251 | func parseTimestamp(in string) (*time.Time, error) { 252 | if in == "" { 253 | return nil, nil 254 | } 255 | 256 | t, err := time.Parse(pgreplay.PostgresTimestampFormat, in) 257 | return &t, errors.Wrapf( 258 | err, "must be a valid timestamp (%s)", pgreplay.PostgresTimestampFormat, 259 | ) 260 | } 261 | 262 | func buildTimeElapsed(start time.Time) string { 263 | const day = time.Minute * 60 * 24 264 | 265 | duration := time.Since(start) 266 | 267 | if duration < 0 { 268 | duration *= -1 269 | } 270 | 271 | if duration < day { 272 | return duration.String() 273 | } 274 | 275 | n := duration / day 276 | duration -= n * day 277 | 278 | if duration == 0 { 279 | return fmt.Sprintf("%dd", n) 280 | } 281 | 282 | return fmt.Sprintf("%dd%s", n, duration) 283 | } 284 | -------------------------------------------------------------------------------- /compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | services: 3 | pgreplay: 4 | command: bash -c "while true; do sleep 10; done" 5 | build: 6 | context: ./ 7 | dockerfile: ./dev.dockerfile 8 | environment: 9 | PGHOST: postgres 10 | PGUSER: postgres 11 | volumes: 12 | - ./:/app 13 | expose: 14 | - 9445 15 | 16 | postgres: 17 | image: postgres:15-alpine 18 | environment: 19 | POSTGRES_USER: postgres 20 | POSTGRES_PASSWORD: password 21 | POSTGRES_DB: users 22 | volumes: 23 | - postgres-db-volume:/var/lib/postgresql/data 24 | expose: 25 | - 5432 26 | 27 | volumes: 28 | postgres-db-volume: 29 | -------------------------------------------------------------------------------- /dev.dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.21.4 2 | 3 | RUN apt update && apt -y upgrade 4 | RUN apt install -y make postgresql-client 5 | 6 | WORKDIR /app 7 | 8 | COPY ./ ./ 9 | RUN make bin/pgreplay.linux_amd64 10 | 11 | EXPOSE 9445 12 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/gocardless/pgreplay-go 2 | 3 | go 1.21.4 4 | 5 | require ( 6 | github.com/alecthomas/kingpin/v2 v2.3.2 7 | github.com/eapache/channels v1.1.0 8 | github.com/go-kit/log v0.2.1 9 | github.com/jackc/pgx/v5 v5.4.3 10 | github.com/json-iterator/go v1.1.12 11 | github.com/onsi/ginkgo v1.16.5 12 | github.com/onsi/gomega v1.27.10 13 | github.com/pkg/errors v0.9.1 14 | github.com/prometheus/client_golang v1.16.0 15 | ) 16 | 17 | require ( 18 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect 19 | github.com/beorn7/perks v1.0.1 // indirect 20 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 21 | github.com/eapache/queue v1.1.0 // indirect 22 | github.com/fsnotify/fsnotify v1.6.0 // indirect 23 | github.com/go-logfmt/logfmt v0.5.1 // indirect 24 | github.com/golang/protobuf v1.5.3 // indirect 25 | github.com/google/go-cmp v0.5.9 // indirect 26 | github.com/jackc/pgpassfile v1.0.0 // indirect 27 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect 28 | github.com/kr/text v0.1.0 // indirect 29 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 30 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 31 | github.com/modern-go/reflect2 v1.0.2 // indirect 32 | github.com/nxadm/tail v1.4.8 // indirect 33 | github.com/prometheus/client_model v0.3.0 // indirect 34 | github.com/prometheus/common v0.42.0 // indirect 35 | github.com/prometheus/procfs v0.10.1 // indirect 36 | github.com/rogpeppe/go-internal v1.11.0 // indirect 37 | github.com/xhit/go-str2duration/v2 v2.1.0 // indirect 38 | golang.org/x/crypto v0.17.0 // indirect 39 | golang.org/x/net v0.15.0 // indirect 40 | golang.org/x/sys v0.15.0 // indirect 41 | golang.org/x/text v0.14.0 // indirect 42 | golang.org/x/tools v0.13.0 // indirect 43 | google.golang.org/protobuf v1.30.0 // indirect 44 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect 45 | gopkg.in/yaml.v3 v3.0.1 // indirect 46 | ) 47 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/alecthomas/kingpin/v2 v2.3.2 h1:H0aULhgmSzN8xQ3nX1uxtdlTHYoPLu5AhHxWrKI6ocU= 2 | github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= 3 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= 4 | github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= 5 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 6 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 7 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 8 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 9 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 11 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/eapache/channels v1.1.0 h1:F1taHcn7/F0i8DYqKXJnyhJcVpp2kgFcNePxXtnyu4k= 13 | github.com/eapache/channels v1.1.0/go.mod h1:jMm2qB5Ubtg9zLd+inMZd2/NUvXgzmWXsDaLyQIGfH0= 14 | github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= 15 | github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= 16 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 17 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 18 | github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= 19 | github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= 20 | github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= 21 | github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= 22 | github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= 23 | github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= 24 | github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= 25 | github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 26 | github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= 27 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= 28 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= 29 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 30 | github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= 31 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 32 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 33 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 34 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 35 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 36 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 37 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 38 | github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= 39 | github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 40 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 41 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 42 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 43 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 44 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 45 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 46 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 47 | github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= 48 | github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= 49 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 50 | github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= 51 | github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= 52 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= 53 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= 54 | github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY= 55 | github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= 56 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 57 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 58 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 59 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 60 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 61 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 62 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 63 | github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= 64 | github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 65 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 66 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 67 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 68 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 69 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 70 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 71 | github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= 72 | github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= 73 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 74 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= 75 | github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= 76 | github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= 77 | github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= 78 | github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= 79 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 80 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 81 | github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= 82 | github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= 83 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 84 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 85 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 86 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 87 | github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= 88 | github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= 89 | github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= 90 | github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= 91 | github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= 92 | github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= 93 | github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= 94 | github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= 95 | github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= 96 | github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= 97 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 98 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 99 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 100 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 101 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 102 | github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= 103 | github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 104 | github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= 105 | github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= 106 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 107 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 108 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 109 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 110 | golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= 111 | golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= 112 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 113 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 114 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 115 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 116 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 117 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 118 | golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= 119 | golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= 120 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 121 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 122 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 123 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 124 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 125 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 126 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 127 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 128 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 129 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 130 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 131 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 132 | golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 133 | golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 134 | golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= 135 | golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 136 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 137 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 138 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 139 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 140 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 141 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 142 | golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 143 | golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= 144 | golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= 145 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 146 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 147 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 148 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 149 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 150 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 151 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 152 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 153 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 154 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 155 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 156 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 157 | google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= 158 | google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 159 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 160 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 161 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 162 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 163 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 164 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 165 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 166 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 167 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 168 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 169 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 170 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 171 | -------------------------------------------------------------------------------- /pkg/pgreplay/database.go: -------------------------------------------------------------------------------- 1 | package pgreplay 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/eapache/channels" 9 | pgx "github.com/jackc/pgx/v5" 10 | "github.com/prometheus/client_golang/prometheus" 11 | "github.com/prometheus/client_golang/prometheus/promauto" 12 | ) 13 | 14 | var ( 15 | connectionsActive = promauto.NewGauge( 16 | prometheus.GaugeOpts{ 17 | Name: "pgreplay_connections_active", 18 | Help: "Number of connections currently open against Postgres", 19 | }, 20 | ) 21 | connectionsEstablishedTotal = promauto.NewCounter( 22 | prometheus.CounterOpts{ 23 | Name: "pgreplay_connections_established_total", 24 | Help: "Number of connections established against Postgres", 25 | }, 26 | ) 27 | itemsProcessedTotal = promauto.NewCounter( 28 | prometheus.CounterOpts{ 29 | Name: "pgreplay_items_processed_total", 30 | Help: "Total count of replay items that have been sent to the database", 31 | }, 32 | ) 33 | itemsMostRecentTimestamp = promauto.NewGauge( 34 | prometheus.GaugeOpts{ 35 | Name: "pgreplay_items_most_recent_timestamp", 36 | Help: "Most recent timestamp of processed items", 37 | }, 38 | ) 39 | ) 40 | 41 | func NewDatabase(ctx context.Context, cfg DatabaseConnConfig) (*Database, error) { 42 | connConfig, err := pgx.ParseConfig(ParseConnData(cfg)) 43 | if err != nil { 44 | return nil, err 45 | } 46 | 47 | conn, err := pgx.ConnectConfig(ctx, connConfig) 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | return &Database{connConfig, map[SessionID]*Conn{}}, conn.Close(ctx) 53 | } 54 | 55 | func ParseConnData(cfg DatabaseConnConfig) string { 56 | return fmt.Sprintf( 57 | "postgres://%s:%s@%s:%d/%s", 58 | cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.Database, 59 | ) 60 | } 61 | 62 | type Database struct { 63 | cfg *pgx.ConnConfig 64 | conns map[SessionID]*Conn 65 | } 66 | 67 | // Consume iterates through all the items in the given channel and attempts to process 68 | // them against the item's session connection. Consume returns two error channels, the 69 | // first for per item errors that should be used for diagnostics only, and the second to 70 | // indicate unrecoverable failures. 71 | // 72 | // Once all items have finished processing, both channels will be closed. 73 | func (d *Database) Consume(ctx context.Context, items chan Item) (chan error, chan error) { 74 | var wg sync.WaitGroup 75 | 76 | errs, done := make(chan error, 10), make(chan error) 77 | 78 | go func() { 79 | for item := range items { 80 | var err error 81 | conn, ok := d.conns[item.GetSessionID()] 82 | 83 | // Connection did not exist, so create a new one 84 | if !ok { 85 | if conn, err = d.Connect(ctx, item); err != nil { 86 | errs <- err 87 | continue 88 | } 89 | 90 | d.conns[item.GetSessionID()] = conn 91 | 92 | wg.Add(1) 93 | connectionsEstablishedTotal.Inc() 94 | connectionsActive.Inc() 95 | 96 | go func(conn *Conn) { 97 | defer wg.Done() 98 | defer connectionsActive.Dec() 99 | 100 | if err := conn.Start(ctx); err != nil { 101 | errs <- err 102 | } 103 | }(conn) 104 | } 105 | 106 | conn.In() <- item 107 | } 108 | 109 | for _, conn := range d.conns { 110 | conn.Close() 111 | } 112 | 113 | // Wait for every connection to terminate 114 | wg.Wait() 115 | 116 | close(errs) 117 | close(done) 118 | }() 119 | 120 | return errs, done 121 | } 122 | 123 | // Connect establishes a new connection to the database, reusing the ConnInfo that was 124 | // generated when the Database was constructed. The wg is incremented whenever we 125 | // establish a new connection and decremented when we disconnect. 126 | func (d *Database) Connect(ctx context.Context, item Item) (*Conn, error) { 127 | cfg := d.cfg.Copy() 128 | cfg.Database, cfg.User = item.GetDatabase(), item.GetUser() 129 | 130 | conn, err := pgx.Connect(ctx, cfg.ConnString()) 131 | if err != nil { 132 | return nil, err 133 | } 134 | 135 | return &Conn{conn, channels.NewInfiniteChannel(), sync.Once{}}, nil 136 | } 137 | 138 | // Conn represents a single database connection handling a stream of work Items 139 | type Conn struct { 140 | *pgx.Conn 141 | channels.Channel 142 | sync.Once 143 | } 144 | 145 | func (c *Conn) Close() { 146 | c.Once.Do(c.Channel.Close) 147 | } 148 | 149 | // Start begins to process the items that are placed into the Conn's channel. We'll finish 150 | // once the connection has died or we run out of items to process. 151 | func (c *Conn) Start(ctx context.Context) error { 152 | items := make(chan Item) 153 | channels.Unwrap(c.Channel, items) 154 | defer c.Close() 155 | 156 | for item := range items { 157 | if item == nil { 158 | continue 159 | } 160 | 161 | itemsProcessedTotal.Inc() 162 | itemsMostRecentTimestamp.Set(float64(item.GetTimestamp().Unix())) 163 | 164 | err := item.Handle(ctx, c.Conn) 165 | 166 | // If we're no longer alive, then we know we can no longer process items 167 | if c.IsClosed() { 168 | return err 169 | } 170 | } 171 | 172 | // If we're still alive after consuming all our items, assume that we finished 173 | // processing our logs before we saw this connection be disconnected. We should 174 | // terminate ourselves by handling our own disconnect, so we can know when all our 175 | // connection are done. 176 | if !c.IsClosed() { 177 | Disconnect{}.Handle(ctx, c.Conn) 178 | } 179 | 180 | return nil 181 | } 182 | -------------------------------------------------------------------------------- /pkg/pgreplay/integration/integration_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "strconv" 7 | 8 | kitlog "github.com/go-kit/log" 9 | "github.com/gocardless/pgreplay-go/pkg/pgreplay" 10 | pgx "github.com/jackc/pgx/v5" 11 | "github.com/onsi/gomega/types" 12 | 13 | . "github.com/onsi/ginkgo" 14 | . "github.com/onsi/ginkgo/extensions/table" 15 | . "github.com/onsi/gomega" 16 | . "github.com/onsi/gomega/gstruct" 17 | ) 18 | 19 | var _ = Describe("pgreplay", func() { 20 | var ( 21 | conn *pgx.Conn 22 | logger = kitlog.NewLogfmtLogger(GinkgoWriter) 23 | err error 24 | 25 | // We expect a Postgres database to be running for integration tests, and that 26 | // environment variables are appropriately configured to permit access. 27 | cfg = pgreplay.DatabaseConnConfig{ 28 | Database: tryEnviron("PGDATABASE", "pgreplay_test"), 29 | Host: tryEnviron("PGHOST", "localhost"), 30 | User: tryEnviron("PGUSER", "pgreplay_test_users"), 31 | Password: tryEnviron("PGPASSWORD", "password"), 32 | Port: uint16(mustAtoi(tryEnviron("PGPORT", "5432"))), 33 | } 34 | ctx = context.Background() 35 | ) 36 | 37 | DescribeTable("Replaying logfiles", 38 | func(parser pgreplay.ParserFunc, fixture string, matchLogs []types.GomegaMatcher) { 39 | conn, err = pgx.Connect(ctx, pgreplay.ParseConnData(cfg)) 40 | Expect(err).NotTo(HaveOccurred(), "failed to connect to postgres") 41 | 42 | _, err = conn.Exec(ctx, `TRUNCATE logs;`) 43 | Expect(err).NotTo(HaveOccurred(), "failed to truncate logs table") 44 | 45 | database, err := pgreplay.NewDatabase(ctx, cfg) 46 | Expect(err).NotTo(HaveOccurred()) 47 | 48 | log, err := os.Open(fixture) 49 | Expect(err).NotTo(HaveOccurred()) 50 | 51 | items, logerrs, parsingDone := parser(log) 52 | go func() { 53 | defer GinkgoRecover() 54 | for err := range logerrs { 55 | logger.Log("event", "parse.error", "error", err) 56 | } 57 | }() 58 | 59 | stream, err := pgreplay.NewStreamer(nil, nil, logger).Stream(items, 1.0) 60 | Expect(err).NotTo(HaveOccurred()) 61 | 62 | errs, consumeDone := database.Consume(ctx, stream) 63 | 64 | // Expect that we finish with no errors 65 | Eventually(consumeDone).Should(BeClosed()) 66 | Eventually(errs).Should(BeClosed()) 67 | 68 | // Parsing should complete 69 | Eventually(parsingDone).Should(BeClosed()) 70 | 71 | // Extract the logs that our test will have placed in the database 72 | logs, err := getLogs(ctx, conn) 73 | 74 | Expect(err).NotTo(HaveOccurred()) 75 | Expect(len(logs)).To(Equal(len(matchLogs))) 76 | 77 | for idx, matchLog := range matchLogs { 78 | Expect(logs[idx]).To(matchLog) 79 | } 80 | }, 81 | Entry("Single user (errlog)", pgreplay.ParseErrlog, "testdata/single_user.log", []types.GomegaMatcher{ 82 | matchLog("alice", "says hello"), 83 | matchLog("alice", "sees 1 logs"), 84 | matchLog("alice", "sees 2 of alice's logs"), 85 | matchLog("alice", "sees 0 of bob's logs"), 86 | }), 87 | Entry("Single user (json)", pgreplay.ParseJSON, "testdata/single_user.json", []types.GomegaMatcher{ 88 | matchLog("alice", "says hello"), 89 | matchLog("alice", "sees 1 logs"), 90 | matchLog("alice", "sees 2 of alice's logs"), 91 | matchLog("alice", "sees 0 of bob's logs"), 92 | }), 93 | ) 94 | }) 95 | 96 | func tryEnviron(key, otherwise string) string { 97 | if value, found := os.LookupEnv(key); found { 98 | return value 99 | } 100 | 101 | return otherwise 102 | } 103 | 104 | func mustAtoi(numstr string) int { 105 | num, err := strconv.Atoi(numstr) 106 | if err != nil { 107 | panic(err) 108 | } 109 | 110 | return num 111 | } 112 | 113 | func getLogs(ctx context.Context, conn *pgx.Conn) ([]interface{}, error) { 114 | rows, err := conn.Query( 115 | ctx, `SELECT id::text, author, message FROM logs ORDER BY id;`, 116 | ) 117 | if err != nil { 118 | return nil, err 119 | } 120 | 121 | defer rows.Close() 122 | var id, author, message string 123 | var logs = []interface{}{} 124 | 125 | for rows.Next() { 126 | if err := rows.Scan(&id, &author, &message); err != nil { 127 | return nil, err 128 | } 129 | 130 | logs = append(logs, &struct{ ID, Author, Message string }{id, author, message}) 131 | } 132 | 133 | return logs, nil 134 | } 135 | 136 | func matchLog(author, message string) types.GomegaMatcher { 137 | return PointTo( 138 | MatchFields(IgnoreExtras, Fields{"Author": Equal(author), "Message": Equal(message)}), 139 | ) 140 | } 141 | -------------------------------------------------------------------------------- /pkg/pgreplay/integration/suite_test.go: -------------------------------------------------------------------------------- 1 | package integration 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestSuite(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "pkg/pgreplay/integration") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/pgreplay/integration/testdata/single_user.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | pgx "github.com/jackc/pgx/v5" 8 | ) 9 | 10 | func connect(ctx context.Context, user string) *pgx.Conn { 11 | connConfig, _ := pgx.ParseConfig( 12 | fmt.Sprintf("postgres://%s@%s:%d/%s", user, "127.0.0.1", 5432, "pgreplay_test"), 13 | ) 14 | 15 | conn, _ := pgx.ConnectConfig( 16 | ctx, connConfig, 17 | ) 18 | 19 | return conn 20 | } 21 | 22 | var someoneSeesUser = `insert into logs (author, message) ( 23 | select $1, format('sees %s of %s''s logs', count(*), $2::text) from logs where author = $2 24 | );` 25 | 26 | func main() { 27 | ctx := context.Background() 28 | alice := connect(ctx, "alice") 29 | alice.Exec(ctx, `insert into logs (author, message) values ('alice', 'says hello');`) 30 | alice.Exec(ctx, `insert into logs (author, message) ( 31 | select 'alice', format('sees %s logs', count(*)) from logs 32 | );`) 33 | 34 | // Named prepared statement 35 | alice.Prepare(ctx, "someone_sees_user", someoneSeesUser) 36 | alice.Exec(ctx, "someone_sees_user", "alice", "alice") 37 | 38 | // Unnamed prepared statement 39 | alice.Exec(ctx, someoneSeesUser, "alice", "bob") 40 | } 41 | -------------------------------------------------------------------------------- /pkg/pgreplay/integration/testdata/single_user.json: -------------------------------------------------------------------------------- 1 | {"type":"Connect","item":{"timestamp":"2019-02-25T15:08:27.233Z","session_id":"5c7404eb.d6bd","user":"alice","database":"pgreplay_test"}} 2 | {"type":"BoundExecute","item":{"timestamp":"2019-02-25T15:08:27.237Z","session_id":"5c7404eb.d6bd","user":"alice","database":"pgreplay_test","query":"select t.oid,\n\tcase when nsp.nspname in ('pg_catalog', 'public') then t.typname\n\t\telse nsp.nspname||'.'||t.typname\n\tend\nfrom pg_type t\nleft join pg_type base_type on t.typelem=base_type.oid\nleft join pg_namespace nsp on t.typnamespace=nsp.oid\nwhere (\n\t t.typtype in('b', 'p', 'r', 'e')\n\t and (base_type.oid is null or base_type.typtype in('b', 'p', 'r'))\n\t)","parameters":[]}} 3 | {"type":"BoundExecute","item":{"timestamp":"2019-02-25T15:08:27.238Z","session_id":"5c7404eb.d6bd","user":"alice","database":"pgreplay_test","query":"select t.oid, t.typname\nfrom pg_type t\n join pg_type base_type on t.typelem=base_type.oid\nwhere t.typtype = 'b'\n and base_type.typtype = 'e'","parameters":[]}} 4 | {"type":"BoundExecute","item":{"timestamp":"2019-02-25T15:08:27.238Z","session_id":"5c7404eb.d6bd","user":"alice","database":"pgreplay_test","query":"select t.oid, t.typname, t.typbasetype\nfrom pg_type t\n join pg_type base_type on t.typbasetype=base_type.oid\nwhere t.typtype = 'd'\n and base_type.typtype = 'b'","parameters":[]}} 5 | {"type":"Statement","item":{"timestamp":"2019-02-25T15:08:27.239Z","session_id":"5c7404eb.d6bd","user":"alice","database":"pgreplay_test","query":"insert into logs (author, message) values ('alice', 'says hello');"}} 6 | {"type":"Statement","item":{"timestamp":"2019-02-25T15:08:27.239Z","session_id":"5c7404eb.d6bd","user":"alice","database":"pgreplay_test","query":"insert into logs (author, message) (\n select 'alice', format('sees %s logs', count(*)) from logs\n);"}} 7 | {"type":"BoundExecute","item":{"timestamp":"2019-02-25T15:08:27.24Z","session_id":"5c7404eb.d6bd","user":"alice","database":"pgreplay_test","query":" insert into logs (author, message) (\n select $1, format('sees %s of %s''s logs', count(*), $2::text) from logs where author = $2\n);","parameters":["alice","alice"]}} 8 | {"type":"BoundExecute","item":{"timestamp":"2019-02-25T15:08:27.24Z","session_id":"5c7404eb.d6bd","user":"alice","database":"pgreplay_test","query":"insert into logs (author, message) (\n select $1, format('sees %s of %s''s logs', count(*), $2::text) from logs where author = $2\n);","parameters":["alice","bob"]}} 9 | {"type":"Disconnect","item":{"timestamp":"2019-02-25T15:08:27.241Z","session_id":"5c7404eb.d6bd","user":"alice","database":"pgreplay_test"}} 10 | -------------------------------------------------------------------------------- /pkg/pgreplay/integration/testdata/single_user.log: -------------------------------------------------------------------------------- 1 | 2019-02-25 15:08:27.232 GMT|[unknown]|[unknown]|5c7404eb.d6bd|LOG: connection received: host=127.0.0.1 port=59103 2 | 2019-02-25 15:08:27.233 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: connection authorized: user=alice database=pgreplay_test 3 | 2019-02-25 15:08:27.236 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.968 ms parse : select t.oid, 4 | case when nsp.nspname in ('pg_catalog', 'public') then t.typname 5 | else nsp.nspname||'.'||t.typname 6 | end 7 | from pg_type t 8 | left join pg_type base_type on t.typelem=base_type.oid 9 | left join pg_namespace nsp on t.typnamespace=nsp.oid 10 | where ( 11 | t.typtype in('b', 'p', 'r', 'e') 12 | and (base_type.oid is null or base_type.typtype in('b', 'p', 'r')) 13 | ) 14 | 2019-02-25 15:08:27.237 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 1.100 ms bind : select t.oid, 15 | case when nsp.nspname in ('pg_catalog', 'public') then t.typname 16 | else nsp.nspname||'.'||t.typname 17 | end 18 | from pg_type t 19 | left join pg_type base_type on t.typelem=base_type.oid 20 | left join pg_namespace nsp on t.typnamespace=nsp.oid 21 | where ( 22 | t.typtype in('b', 'p', 'r', 'e') 23 | and (base_type.oid is null or base_type.typtype in('b', 'p', 'r')) 24 | ) 25 | 2019-02-25 15:08:27.237 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: execute : select t.oid, 26 | case when nsp.nspname in ('pg_catalog', 'public') then t.typname 27 | else nsp.nspname||'.'||t.typname 28 | end 29 | from pg_type t 30 | left join pg_type base_type on t.typelem=base_type.oid 31 | left join pg_namespace nsp on t.typnamespace=nsp.oid 32 | where ( 33 | t.typtype in('b', 'p', 'r', 'e') 34 | and (base_type.oid is null or base_type.typtype in('b', 'p', 'r')) 35 | ) 36 | 2019-02-25 15:08:27.238 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.326 ms 37 | 2019-02-25 15:08:27.238 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.080 ms parse : select t.oid, t.typname 38 | from pg_type t 39 | join pg_type base_type on t.typelem=base_type.oid 40 | where t.typtype = 'b' 41 | and base_type.typtype = 'e' 42 | 2019-02-25 15:08:27.238 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.124 ms bind : select t.oid, t.typname 43 | from pg_type t 44 | join pg_type base_type on t.typelem=base_type.oid 45 | where t.typtype = 'b' 46 | and base_type.typtype = 'e' 47 | 2019-02-25 15:08:27.238 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: execute : select t.oid, t.typname 48 | from pg_type t 49 | join pg_type base_type on t.typelem=base_type.oid 50 | where t.typtype = 'b' 51 | and base_type.typtype = 'e' 52 | 2019-02-25 15:08:27.238 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.060 ms 53 | 2019-02-25 15:08:27.238 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.055 ms parse : select t.oid, t.typname, t.typbasetype 54 | from pg_type t 55 | join pg_type base_type on t.typbasetype=base_type.oid 56 | where t.typtype = 'd' 57 | and base_type.typtype = 'b' 58 | 2019-02-25 15:08:27.238 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.157 ms bind : select t.oid, t.typname, t.typbasetype 59 | from pg_type t 60 | join pg_type base_type on t.typbasetype=base_type.oid 61 | where t.typtype = 'd' 62 | and base_type.typtype = 'b' 63 | 2019-02-25 15:08:27.238 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: execute : select t.oid, t.typname, t.typbasetype 64 | from pg_type t 65 | join pg_type base_type on t.typbasetype=base_type.oid 66 | where t.typtype = 'd' 67 | and base_type.typtype = 'b' 68 | 2019-02-25 15:08:27.239 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.102 ms 69 | 2019-02-25 15:08:27.239 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: statement: insert into logs (author, message) values ('alice', 'says hello'); 70 | 2019-02-25 15:08:27.239 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.601 ms 71 | 2019-02-25 15:08:27.239 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: statement: insert into logs (author, message) ( 72 | select 'alice', format('sees %s logs', count(*)) from logs 73 | ); 74 | 2019-02-25 15:08:27.240 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.362 ms 75 | 2019-02-25 15:08:27.240 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.088 ms parse someone_sees_user: insert into logs (author, message) ( 76 | select $1, format('sees %s of %s''s logs', count(*), $2::text) from logs where author = $2 77 | ); 78 | 2019-02-25 15:08:27.240 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.093 ms bind someone_sees_user: insert into logs (author, message) ( 79 | select $1, format('sees %s of %s''s logs', count(*), $2::text) from logs where author = $2 80 | ); 81 | 2019-02-25 15:08:27.240 GMT|alice|pgreplay_test|5c7404eb.d6bd|DETAIL: parameters: $1 = 'alice', $2 = 'alice' 82 | 2019-02-25 15:08:27.240 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: execute someone_sees_user: insert into logs (author, message) ( 83 | select $1, format('sees %s of %s''s logs', count(*), $2::text) from logs where author = $2 84 | ); 85 | 2019-02-25 15:08:27.240 GMT|alice|pgreplay_test|5c7404eb.d6bd|DETAIL: parameters: $1 = 'alice', $2 = 'alice' 86 | 2019-02-25 15:08:27.240 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.079 ms 87 | 2019-02-25 15:08:27.240 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.042 ms parse : insert into logs (author, message) ( 88 | select $1, format('sees %s of %s''s logs', count(*), $2::text) from logs where author = $2 89 | ); 90 | 2019-02-25 15:08:27.240 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.045 ms bind : insert into logs (author, message) ( 91 | select $1, format('sees %s of %s''s logs', count(*), $2::text) from logs where author = $2 92 | ); 93 | 2019-02-25 15:08:27.240 GMT|alice|pgreplay_test|5c7404eb.d6bd|DETAIL: parameters: $1 = 'alice', $2 = 'bob' 94 | 2019-02-25 15:08:27.240 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: execute : insert into logs (author, message) ( 95 | select $1, format('sees %s of %s''s logs', count(*), $2::text) from logs where author = $2 96 | ); 97 | 2019-02-25 15:08:27.240 GMT|alice|pgreplay_test|5c7404eb.d6bd|DETAIL: parameters: $1 = 'alice', $2 = 'bob' 98 | 2019-02-25 15:08:27.240 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.042 ms 99 | 2019-02-25 15:08:27.241 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: disconnection: session time: 0:00:00.009 user=alice database=pgreplay_test host=127.0.0.1 port=59103 100 | -------------------------------------------------------------------------------- /pkg/pgreplay/integration/testdata/structure.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- PostgreSQL database dump 3 | -- 4 | 5 | -- Dumped from database version 10.5 6 | -- Dumped by pg_dump version 10.5 7 | 8 | SET statement_timeout = 0; 9 | SET lock_timeout = 0; 10 | SET idle_in_transaction_session_timeout = 0; 11 | SET client_encoding = 'UTF8'; 12 | SET standard_conforming_strings = on; 13 | SELECT pg_catalog.set_config('search_path', '', false); 14 | SET check_function_bodies = false; 15 | SET client_min_messages = warning; 16 | SET row_security = off; 17 | 18 | -- 19 | -- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: 20 | -- 21 | 22 | CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; 23 | 24 | 25 | -- 26 | -- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: 27 | -- 28 | 29 | COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; 30 | 31 | 32 | SET default_tablespace = ''; 33 | 34 | SET default_with_oids = false; 35 | 36 | -- 37 | -- Name: logs; Type: TABLE; Schema: public; Owner: postgres 38 | -- 39 | 40 | CREATE TABLE public.logs ( 41 | id integer NOT NULL, 42 | author text, 43 | message text 44 | ); 45 | 46 | 47 | ALTER TABLE public.logs OWNER TO postgres; 48 | 49 | -- 50 | -- Name: logs_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres 51 | -- 52 | 53 | CREATE SEQUENCE public.logs_id_seq 54 | AS integer 55 | START WITH 1 56 | INCREMENT BY 1 57 | NO MINVALUE 58 | NO MAXVALUE 59 | CACHE 1; 60 | 61 | 62 | ALTER TABLE public.logs_id_seq OWNER TO postgres; 63 | 64 | -- 65 | -- Name: logs_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: postgres 66 | -- 67 | 68 | ALTER SEQUENCE public.logs_id_seq OWNED BY public.logs.id; 69 | 70 | 71 | -- 72 | -- Name: logs id; Type: DEFAULT; Schema: public; Owner: postgres 73 | -- 74 | 75 | ALTER TABLE ONLY public.logs ALTER COLUMN id SET DEFAULT nextval('public.logs_id_seq'::regclass); 76 | 77 | 78 | -- 79 | -- Name: logs logs_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres 80 | -- 81 | 82 | ALTER TABLE ONLY public.logs 83 | ADD CONSTRAINT logs_pkey PRIMARY KEY (id); 84 | 85 | 86 | -- 87 | -- PostgreSQL database dump complete 88 | -- 89 | 90 | -------------------------------------------------------------------------------- /pkg/pgreplay/parse.go: -------------------------------------------------------------------------------- 1 | package pgreplay 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/csv" 7 | "fmt" 8 | "io" 9 | "regexp" 10 | "strings" 11 | "time" 12 | 13 | "github.com/prometheus/client_golang/prometheus" 14 | "github.com/prometheus/client_golang/prometheus/promauto" 15 | ) 16 | 17 | var ( 18 | logLinesParsedTotal = promauto.NewCounter( 19 | prometheus.CounterOpts{ 20 | Name: "pgreplay_log_lines_parsed_total", 21 | Help: "Number of log lines parsed since boot", 22 | }, 23 | ) 24 | logLinesErrorTotal = promauto.NewCounter( 25 | prometheus.CounterOpts{ 26 | Name: "pgreplay_log_lines_error_total", 27 | Help: "Number of log lines that failed to parse", 28 | }, 29 | ) 30 | ) 31 | 32 | const ( 33 | // ItemBufferSize defines the size of the channel buffer when parsing Items. 34 | // Allowing the channel to buffer makes a significant throughput improvement to the 35 | // parsing. 36 | ItemBufferSize = 100 37 | 38 | // MaxLogLineSize denotes the maximum size, in bytes, that we can scan in a single log 39 | // line. It is possible to pass really large arrays of parameters to Postgres queries 40 | // which is why this has to be so large. 41 | MaxLogLineSize = 10 * 1024 * 1024 42 | InitialScannerBufferSize = 10 * 10 43 | 44 | // PostgresTimestampFormat is the Go template format that we expect to find our errlog 45 | PostgresTimestampFormat = "2006-01-02 15:04:05.000 MST" 46 | ) 47 | 48 | // ParserFunc is the standard interface to provide items from a parsing source 49 | type ParserFunc func(io.Reader) (items chan Item, errs chan error, done chan error) 50 | 51 | // ParseJSON operates on a file of JSON serialized Item elements, and pushes the parsed 52 | // items down the returned channel. 53 | func ParseJSON(jsonlog io.Reader) (items chan Item, errs chan error, done chan error) { 54 | items, errs, done = make(chan Item, ItemBufferSize), make(chan error), make(chan error) 55 | 56 | go func() { 57 | scanner := bufio.NewScanner(jsonlog) 58 | scanner.Buffer(make([]byte, InitialScannerBufferSize), MaxLogLineSize) 59 | 60 | for scanner.Scan() { 61 | line := scanner.Text() 62 | item, err := ItemUnmarshalJSON([]byte(line)) 63 | if err != nil { 64 | errs <- err 65 | } else { 66 | items <- item 67 | } 68 | } 69 | 70 | close(items) 71 | close(errs) 72 | 73 | done <- scanner.Err() 74 | close(done) 75 | }() 76 | 77 | return 78 | } 79 | 80 | func ParseCsvLog(csvlog io.Reader) (items chan Item, errs chan error, done chan error) { 81 | reader := csv.NewReader(csvlog) 82 | unbounds := map[SessionID]*Execute{} 83 | parsebuffer := make([]byte, MaxLogLineSize) 84 | items, errs, done = make(chan Item, ItemBufferSize), make(chan error), make(chan error) 85 | 86 | go func() { 87 | for { 88 | logline, err := reader.Read() 89 | if err == io.EOF { 90 | break 91 | } 92 | if err != nil { 93 | logLinesErrorTotal.Inc() 94 | errs <- err 95 | } 96 | 97 | item, err := ParseCsvItem(logline, unbounds, parsebuffer) 98 | if err != nil { 99 | logLinesErrorTotal.Inc() 100 | errs <- err 101 | } 102 | 103 | if item != nil { 104 | logLinesParsedTotal.Inc() 105 | items <- item 106 | } 107 | } 108 | 109 | close(items) 110 | close(errs) 111 | close(done) 112 | }() 113 | 114 | return 115 | } 116 | 117 | // ParseErrlog generates a stream of Items from the given PostgreSQL errlog. Log line 118 | // parsing errors are returned down the errs channel, and we signal having finished our 119 | // parsing by sending a value down the done channel. 120 | func ParseErrlog(errlog io.Reader) (items chan Item, errs chan error, done chan error) { 121 | unbounds := map[SessionID]*Execute{} 122 | loglinebuffer, parsebuffer := make([]byte, MaxLogLineSize), make([]byte, MaxLogLineSize) 123 | scanner := NewLogScanner(errlog, loglinebuffer) 124 | 125 | items, errs, done = make(chan Item, ItemBufferSize), make(chan error), make(chan error) 126 | 127 | go func() { 128 | for scanner.Scan() { 129 | item, err := ParseItem(scanner.Text(), unbounds, parsebuffer) 130 | if err != nil { 131 | logLinesErrorTotal.Inc() 132 | errs <- err 133 | } 134 | 135 | if item != nil { 136 | logLinesParsedTotal.Inc() 137 | items <- item 138 | } 139 | } 140 | 141 | close(items) 142 | close(errs) 143 | 144 | done <- scanner.Err() 145 | close(done) 146 | }() 147 | 148 | return 149 | } 150 | 151 | const ( 152 | // File Type Conversion 153 | ParsedFromCsv = "csv" 154 | ParsedFromErrLog = "errlog" 155 | // Log Detail Message 156 | ActionLog = "LOG: " 157 | ActionDetail = "DETAIL: " 158 | ActionError = "ERROR: " 159 | ) 160 | 161 | var ( 162 | LogConnectionAuthorized = LogMessage{ 163 | ActionLog, "connection authorized: ", 164 | regexp.MustCompile(`^connection authorized\: `), 165 | } 166 | LogConnectionReceived = LogMessage{ 167 | ActionLog, "connection received: ", 168 | regexp.MustCompile(`^connection received\: `), 169 | } 170 | LogConnectionDisconnect = LogMessage{ 171 | ActionLog, "disconnection: ", 172 | regexp.MustCompile(`^disconnection\: `), 173 | } 174 | LogStatement = LogMessage{ 175 | ActionLog, "statement: ", 176 | regexp.MustCompile(`^.*statement\: `), 177 | } 178 | LogDuration = LogMessage{ 179 | ActionLog, "duration: ", 180 | regexp.MustCompile(`^duration\: (\d+)\.(\d+) ms$`), 181 | } 182 | LogExtendedProtocolExecute = LogMessage{ 183 | ActionLog, "execute : ", 184 | regexp.MustCompile(`^.*execute \: `), 185 | } 186 | LogExtendedProtocolParameters = LogMessage{ 187 | ActionDetail, "parameters: ", 188 | regexp.MustCompile(`^parameters\: `), 189 | } 190 | LogNamedPrepareExecute = LogMessage{ 191 | ActionLog, "execute ", 192 | regexp.MustCompile(`^.*execute (\w+)\: `), 193 | } 194 | LogError = LogMessage{ActionError, "", regexp.MustCompile(`^ERROR\: .+`)} 195 | LogDetail = LogMessage{ActionDetail, "", regexp.MustCompile(`^DETAIL\: .+`)} 196 | ) 197 | 198 | // ParseCsvItem constructs a Item from a CSV log line. The format we accept is log_destination='csvlog'. 199 | func ParseCsvItem(logline []string, unbounds map[SessionID]*Execute, buffer []byte) (Item, error) { 200 | if len(logline) < 15 { 201 | return nil, fmt.Errorf("failed to parse log line: '%s'", logline) 202 | } 203 | 204 | ts, err := time.Parse(PostgresTimestampFormat, logline[0]) 205 | if err != nil { 206 | return nil, fmt.Errorf("failed to parse log timestamp: '%s': %v", logline[0], err) 207 | } 208 | 209 | // 2023-06-09 01:50:01.825 UTC,"postgres","postgres",,,64828549.7698,,,,,,,,,, .... 210 | user, database, session, actionLog, msg, params := logline[1], logline[2], logline[5], logline[11], logline[13], logline[14] 211 | 212 | extractedLog := ExtractedLog{ 213 | Details: Details{ 214 | Timestamp: ts, 215 | SessionID: SessionID(session), 216 | User: user, 217 | Database: database, 218 | }, 219 | ActionLog: actionLog, 220 | Message: msg, 221 | Parameters: params, 222 | } 223 | 224 | return parseDetailToItem(extractedLog, ParsedFromCsv, unbounds, buffer) 225 | } 226 | 227 | // ParseItem constructs a Item from Postgres errlogs. The format we accept is 228 | // log_line_prefix='%m|%u|%d|%c|', so we can split by | to discover each component. 229 | // 230 | // The unbounds map allows retrieval of an Execute that was previously parsed for a 231 | // session, as we expect following log lines to complete the Execute with the parameters 232 | // it should use. 233 | func ParseItem(logline string, unbounds map[SessionID]*Execute, buffer []byte) (Item, error) { 234 | tokens := strings.SplitN(logline, "|", 5) 235 | if len(tokens) != 5 { 236 | return nil, fmt.Errorf("failed to parse log line: '%s'", logline) 237 | } 238 | 239 | ts, err := time.Parse(PostgresTimestampFormat, tokens[0]) 240 | if err != nil { 241 | return nil, fmt.Errorf("failed to parse log timestamp: '%s': %v", tokens[0], err) 242 | } 243 | 244 | // 2018-06-04 13:00:52.366 UTC|postgres|postgres|5b153804.964| 245 | user, database, session, msg := tokens[1], tokens[2], tokens[3], tokens[4] 246 | 247 | extractedLog := ExtractedLog{ 248 | Details: Details{ 249 | Timestamp: ts, 250 | SessionID: SessionID(session), 251 | User: user, 252 | Database: database, 253 | }, 254 | ActionLog: "", 255 | Message: msg, 256 | Parameters: "", 257 | } 258 | 259 | return parseDetailToItem(extractedLog, ParsedFromErrLog, unbounds, buffer) 260 | } 261 | 262 | func parseDetailToItem(el ExtractedLog, parsedFrom string, unbounds map[SessionID]*Execute, buff []byte) (Item, error) { 263 | // LOG: duration: 0.043 ms 264 | // Duration logs mark completion of replay items, and are not of interest for 265 | // reproducing traffic. We should only take an action if there exists an unbound item 266 | // for this session, as this log line will confirm the unbound query has no parameters. 267 | if LogDuration.Match(el.Message, parsedFrom) { 268 | if unbound, ok := unbounds[el.SessionID]; ok { 269 | delete(unbounds, el.SessionID) 270 | return unbound.Bind(nil), nil 271 | } 272 | 273 | return nil, nil 274 | } 275 | 276 | // LOG: statement: select pg_reload_conf(); 277 | if LogStatement.Match(el.Message, parsedFrom) { 278 | return Statement{el.Details, LogStatement.RenderQuery(el.Message, parsedFrom)}, nil 279 | } 280 | 281 | // LOG: execute : select pg_sleep($1) 282 | // An execute log represents a potential statement. When running the extended protocol, 283 | // even queries that don't have any arguments will be sent as an unamed prepared 284 | // statement. We need to wait for a following DETAIL or duration log to confirm the 285 | // statement has been executed. 286 | if LogExtendedProtocolExecute.Match(el.Message, parsedFrom) { 287 | query := LogExtendedProtocolExecute.RenderQuery(el.Message, parsedFrom) 288 | 289 | if parsedFrom == ParsedFromCsv { 290 | params, err := ParseBindParameters(LogExtendedProtocolParameters.RenderQuery(el.Parameters, parsedFrom), buff) 291 | if err != nil { 292 | return nil, fmt.Errorf("[UnNamedExecute]: failed to parse bind parameters: %s", err.Error()) 293 | } 294 | 295 | return Execute{el.Details, query}.Bind(params), nil 296 | } 297 | 298 | unbounds[el.SessionID] = &Execute{el.Details, query} 299 | 300 | return nil, nil 301 | } 302 | 303 | // LOG: execute name: select pg_sleep($1) 304 | if LogNamedPrepareExecute.Match(el.Message, parsedFrom) { 305 | if parsedFrom == ParsedFromCsv { 306 | query := LogNamedPrepareExecute.RenderQuery(el.Message, parsedFrom) 307 | params, err := ParseBindParameters(LogExtendedProtocolParameters.RenderQuery(el.Parameters, parsedFrom), buff) 308 | if err != nil { 309 | return nil, fmt.Errorf("[NamedExecute]: failed to parse bind parameters: %s", err.Error()) 310 | } 311 | 312 | return Execute{el.Details, query}.Bind(params), nil 313 | } 314 | 315 | query := strings.SplitN( 316 | LogNamedPrepareExecute.RenderQuery(el.Message, parsedFrom), ":", 2, 317 | )[1] 318 | 319 | // TODO: This doesn't exactly replicate what we'd expect from named prepares. Instead 320 | // of creating a genuine named prepare, we implement them as unnamed prepared 321 | // statements instead. If this parse signature allowed us to return arbitrary items 322 | // then we'd be able to create an initial prepare statement followed by a matching 323 | // execute, but we can hold off doing this until it becomes a problem. 324 | unbounds[el.SessionID] = &Execute{el.Details, query} 325 | 326 | return nil, nil 327 | } 328 | 329 | // DETAIL: parameters: $1 = '1', $2 = NULL 330 | if LogExtendedProtocolParameters.Match(el.Message, parsedFrom) { 331 | if unbound, ok := unbounds[el.SessionID]; ok { 332 | parameters, err := ParseBindParameters(LogExtendedProtocolParameters.RenderQuery(el.Message, parsedFrom), buff) 333 | if err != nil { 334 | return nil, fmt.Errorf("failed to parse bind parameters: %s", err.Error()) 335 | } 336 | 337 | // Remove the unbound from our cache and bind it 338 | delete(unbounds, el.SessionID) 339 | return unbound.Bind(parameters), nil 340 | } 341 | 342 | // It's quite normal for us to get here, as Postgres will log the following when 343 | // log_min_duration_statement = 0: 344 | // 345 | // 1. LOG: duration: 0.XXX ms parse name: 346 | // 2. LOG: duration: 0.XXX ms bind name: 347 | // 3. DETAIL: parameters: $1 = '', $2 = '', ... 348 | // 4. LOG: execute name: 349 | // 5. DETAIL: parameters: $1 = '', $2 = '', ... 350 | // 351 | // The 3rd and 5th entry are the same, but we expect to be matching our detail against 352 | // a prior execute log-line. This is just an artifact of Postgres extended query 353 | // protocol and the activation of two logging systems which duplicate the same entry. 354 | return nil, fmt.Errorf("cannot process bind parameters without previous execute item: %s", el.Message) 355 | } 356 | 357 | // LOG: connection authorized: user=postgres database=postgres 358 | if LogConnectionAuthorized.Match(el.Message, parsedFrom) { 359 | return Connect{el.Details}, nil 360 | } 361 | 362 | // LOG: disconnection: session time: 0:00:03.861 user=postgres database=postgres host=192.168.99.1 port=51529 363 | if LogConnectionDisconnect.Match(el.Message, parsedFrom) { 364 | return Disconnect{el.Details}, nil 365 | } 366 | 367 | // LOG: connection received: host=192.168.99.1 port=52188 368 | // We use connection authorized for replay, and can safely ignore connection received 369 | if LogConnectionReceived.Match(el.Message, parsedFrom) { 370 | return nil, nil 371 | } 372 | 373 | // ERROR: invalid value for parameter \"log_destination\": \"/var\" 374 | // We don't replicate errors as this should be the minority of our traffic. Can safely 375 | // ignore. 376 | if el.ActionLog == "ERROR" || LogError.Match(el.Message, parsedFrom) { 377 | return nil, nil 378 | } 379 | 380 | // DETAIL: Unrecognized key word: \"/var/log/postgres/postgres.log\" 381 | // The previous condition catches the extended query bind detail statements, and any 382 | // other DETAIL logs we can safely ignore. 383 | if el.ActionLog == "DETAIL" || LogDetail.Match(el.Message, parsedFrom) { 384 | return nil, nil 385 | } 386 | 387 | return nil, fmt.Errorf("no parser matches line: %s", el.Message) 388 | } 389 | 390 | // ParseBindParameters constructs an interface slice from the suffix of a DETAIL parameter 391 | // Postgres errlog. An example input to this function would be: 392 | // 393 | // $1 = ”, $2 = '30', $3 = '2018-05-03 10:26:27.905086+00' 394 | // 395 | // ...and this would be parsed into []interface{"", "30", "2018-05-03 10:26:27.905086+00"} 396 | func ParseBindParameters(input string, buffer []byte) ([]interface{}, error) { 397 | if buffer == nil { 398 | buffer = make([]byte, InitialScannerBufferSize) 399 | } 400 | 401 | scanner := bufio.NewScanner(strings.NewReader(input)) 402 | scanner.Buffer(buffer, MaxLogLineSize) 403 | scanner.Split(bindParametersSplitFunc) 404 | 405 | parameters := make([]interface{}, 0) 406 | 407 | for scanner.Scan() { 408 | token := scanner.Text() 409 | switch token { 410 | case "NULL": 411 | parameters = append(parameters, nil) 412 | default: 413 | parameters = append(parameters, strings.Replace( 414 | token[1:len(token)-1], "''", "'", -1, 415 | )) 416 | } 417 | } 418 | 419 | return parameters, scanner.Err() 420 | } 421 | 422 | var prefixMatcher = regexp.MustCompile(`^(, )?\$\d+ = `) 423 | 424 | func bindParametersSplitFunc(data []byte, atEOF bool) (advance int, token []byte, err error) { 425 | if atEOF && len(data) == 0 { 426 | return 0, nil, nil 427 | } 428 | 429 | prefix := prefixMatcher.Find(data) 430 | if len(prefix) == 0 { 431 | return 0, nil, fmt.Errorf("could not parse parameter: %s", string(data)) 432 | } 433 | 434 | advance = len(prefix) 435 | if bytes.HasPrefix(data[advance:], []byte("NULL")) { 436 | return advance + 4, []byte("NULL"), nil 437 | } 438 | 439 | closingIdx := findClosingTag(string(data[advance+1:]), "'", "''") 440 | if closingIdx == -1 { 441 | return 0, nil, fmt.Errorf("could not find closing ' for parameter: %s", string(data)) 442 | } 443 | 444 | token = data[advance : advance+2+closingIdx] 445 | advance += 2 + closingIdx 446 | 447 | return 448 | } 449 | 450 | // findClosingTag will search the given input for the provided marker, finding the first 451 | // index of the marker that does not also match the escapeSequence. 452 | func findClosingTag(input, marker, escapeSequence string) (idx int) { 453 | for idx < len(input) { 454 | if strings.HasPrefix(input[idx:], marker) { 455 | if strings.HasPrefix(input[idx:], escapeSequence) { 456 | idx += len(escapeSequence) 457 | continue 458 | } 459 | 460 | return idx 461 | } 462 | 463 | idx++ 464 | } 465 | 466 | return -1 467 | } 468 | 469 | // NewLogScanner constructs a scanner that will produce a single token per errlog line 470 | // from Postgres logs. Postgres errlog format looks like this: 471 | // 472 | // 2018-05-03|gc|LOG: duration: 0.096 ms parse : 473 | // 474 | // DELETE FROM que_jobs 475 | // WHERE queue = $1::text 476 | // 477 | // ...where a log line can spill over multiple lines, with trailing lines marked with a 478 | // preceding \t. 479 | func NewLogScanner(input io.Reader, buffer []byte) *bufio.Scanner { 480 | if buffer == nil { 481 | buffer = make([]byte, InitialScannerBufferSize) 482 | } 483 | 484 | scanner := bufio.NewScanner(input) 485 | scanner.Buffer(buffer, MaxLogLineSize) 486 | scanner.Split(logLineSplitFunc) 487 | 488 | return scanner 489 | } 490 | 491 | func logLineSplitFunc(data []byte, atEOF bool) (advance int, token []byte, err error) { 492 | if atEOF && len(data) == 0 { 493 | return 0, nil, nil 494 | } 495 | 496 | for { 497 | // Only seek to the penultimate byte in data, because we want to safely access the 498 | // byte beyond this if we find a newline. 499 | offset := bytes.Index(data[advance:len(data)-1], []byte("\n")) 500 | 501 | // If we have no more data to consume but we can't find our delimiter, assume the 502 | // entire data slice is our token. 503 | if atEOF && offset == -1 { 504 | advance = len(data) 505 | break 506 | } 507 | 508 | // If we can't peek past our newline, then we'll never know if this is a genuine log 509 | // line terminator. Signal that we need more content and try again. 510 | if offset == -1 { 511 | return 0, nil, nil 512 | } 513 | 514 | advance += offset + 1 515 | 516 | // If the character immediately proceeding our newline is not a tab, then we know 517 | // we've come to the end of a valid log, and should break out of our loop. We should 518 | // only do this if we've genuinely consumed input, i.e., our token is not just 519 | // whitespace. 520 | if data[advance] != '\t' && len(bytes.TrimSpace(data[0:advance])) > 0 { 521 | break 522 | } 523 | } 524 | 525 | // We should replace any \n\t with just \n, as that is what a leading \t implies. 526 | token = bytes.Replace(data[0:advance], []byte("\n\t"), []byte("\n"), -1) 527 | token = bytes.TrimSpace(token) 528 | 529 | return advance, token, nil 530 | } 531 | -------------------------------------------------------------------------------- /pkg/pgreplay/parse_test.go: -------------------------------------------------------------------------------- 1 | package pgreplay 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | . "github.com/onsi/ginkgo" 8 | . "github.com/onsi/ginkgo/extensions/table" 9 | . "github.com/onsi/gomega" 10 | ) 11 | 12 | var time20190225, _ = time.Parse(PostgresTimestampFormat, "2019-02-25 15:08:27.222 GMT") 13 | 14 | var _ = Describe("ParseCsvLog", func() { 15 | DescribeTable("Parses", 16 | func(input string, expected []Item) { 17 | var items = []Item{} 18 | itemsChan, errs, done := ParseCsvLog(strings.NewReader(input)) 19 | go func() { 20 | for range errs { 21 | // no-op, just drain the channel 22 | } 23 | }() 24 | 25 | for item := range itemsChan { 26 | if item != nil { 27 | items = append(items, item) 28 | } 29 | } 30 | 31 | Eventually(done).Should(BeClosed()) 32 | Expect(len(items)).To(Equal(len(expected))) 33 | 34 | for idx, item := range items { 35 | Expect(item).To(BeEquivalentTo(expected[idx])) 36 | } 37 | }, 38 | Entry( 39 | "queries and duration logs", 40 | ` 41 | 2019-02-25 15:08:27.222 GMT,"postgres","postgres",7283,"199.167.158.43:57426",6480e39e.1c73,6374,"SELECT",2019-02-25 15:08:27.222 GMT,4/286618,0,LOG,00000,"connection received: host=127.0.0.1 port=59103",,,,,,,,,"","client backend" 42 | 2019-02-25 15:08:27.222 GMT,"postgres","postgres",7283,"199.167.158.43:57426",6480e39e.1c73,6374,"SELECT",2019-02-25 15:08:27.222 GMT,4/286618,0,LOG,00000,"connection authorized: user=alice database=pgreplay_test",,,,,,,,,"","client backend" 43 | 2019-02-25 15:08:27.222 GMT,"postgres","postgres",26223,"172.31.237.67:40680",65391eda.666f,39505,"SELECT",2023-10-25 13:57:46 UTC,706/2676024,0,LOG,00000,"duration: 0.029 ms execute : SELECT 1 AS one FROM ""mural_files"" WHERE (""mural_files"".""mural_id"" = $1) AND (""mural_files"".""embedded"" = $2) LIMIT $3","parameters: $1 = '1072', $2 = 'f', $3 = '1'",,,,,,,"exec_execute_message, postgres.c:2342","puma: [app]","client backend",,5774081526858323261 44 | 2019-02-25 15:08:27.222 GMT,"postgres","postgres",5081,"172.31.210.83:57006",6539311d.13d9,955,"SELECT",2023-10-25 15:15:41 UTC,595/2810709,0,LOG,00000,"duration: 0.028 ms execute a127: SELECT ""roles"".* FROM ""roles"" WHERE ""roles"".""id"" = $1 LIMIT $2","parameters: $1 = '65', $2 = '1'",,,,,,,"exec_execute_message, postgres.c:2342","puma: [app]","client backend",,-1029561919799294166 45 | 2019-02-25 15:08:27.222 GMT,"postgres","postgres",7283,"199.167.158.43:57426",6480e39e.1c73,6374,"SELECT",2019-02-25 15:08:27.222 GMT,4/286618,0,LOG,00000,"duration: 71.963 ms",,,,,,,,,"","client backend" 46 | 2019-02-25 15:08:27.222 GMT,"postgres","postgres",7283,"199.167.158.43:57426",6480e39e.1c73,6374,"SELECT",2019-02-25 15:08:27.222 GMT,4/286618,0,LOG,00000,"execute : select t.oid",,,,,,,,,"","client backend" 47 | 2019-02-25 15:08:27.222 GMT,"postgres","postgres",7283,"199.167.158.43:57426",6480e39e.1c73,6374,"SELECT",2019-02-25 15:08:27.222 GMT,4/286618,0,LOG,00000,"execute : select t.oid from test t where id = $1","parameters: $1 = '41145'",,,,,,,,"","client backend" 48 | 2019-02-25 15:08:27.222 GMT,"postgres","postgres",7283,"199.167.158.43:57426",6480e39e.1c73,6375,"idle in transaction",2019-02-25 15:08:27.222 GMT,4/286618,0,LOG,00000,"statement: SELECT p.name, r.rating 49 | FROM products p 50 | JOIN reviews r ON p.id = r.product_id 51 | WHERE r.rating IN ( 52 | SELECT MIN(rating) FROM reviews 53 | UNION 54 | SELECT MAX(rating) FROM reviews 55 | ); 56 | ",,,,,,,,,"","client backend" 57 | 2019-02-25 15:08:27.222 GMT,"postgres","postgres",7283,"199.167.158.43:57426",6480e39e.1c73,6376,"SELECT",2019-02-25 15:08:27.222 GMT,4/286618,0,LOG,00000,"duration: 53.774 ms",,,,,,,,,"","client backend" 58 | 2019-02-25 15:08:27.222 GMT,"postgres","postgres",7283,"199.167.158.43:57426",6480e39e.1c73,6377,"idle in transaction",2019-02-25 15:08:27.222 GMT,4/286618,0,LOG,00000,"statement: SELECT name, email 59 | FROM users 60 | WHERE email LIKE '@gmail.com'; 61 | ",,,,,,,,,"","client backend"`, 62 | []Item{ 63 | Connect{ 64 | Details{ 65 | Timestamp: time20190225, 66 | SessionID: "6480e39e.1c73", 67 | User: "postgres", 68 | Database: "postgres", 69 | }, 70 | }, 71 | BoundExecute{ 72 | Execute: Execute{ 73 | Details: Details{ 74 | Timestamp: time20190225, 75 | SessionID: "65391eda.666f", 76 | User: "postgres", 77 | Database: "postgres", 78 | }, 79 | Query: "SELECT 1 AS one FROM \"mural_files\" WHERE (\"mural_files\".\"mural_id\" = $1) AND (\"mural_files\".\"embedded\" = $2) LIMIT $3", 80 | }, 81 | Parameters: []interface{}{"1072", "f", "1"}, 82 | }, 83 | BoundExecute{ 84 | Execute: Execute{ 85 | Details: Details{ 86 | Timestamp: time20190225, 87 | SessionID: "6539311d.13d9", 88 | User: "postgres", 89 | Database: "postgres", 90 | }, 91 | Query: "SELECT \"roles\".* FROM \"roles\" WHERE \"roles\".\"id\" = $1 LIMIT $2", 92 | }, 93 | Parameters: []interface{}{"65", "1"}, 94 | }, 95 | BoundExecute{ 96 | Execute: Execute{ 97 | Details: Details{ 98 | Timestamp: time20190225, 99 | SessionID: "6480e39e.1c73", 100 | User: "postgres", 101 | Database: "postgres", 102 | }, 103 | Query: "select t.oid", 104 | }, 105 | Parameters: []interface{}{}, 106 | }, 107 | BoundExecute{ 108 | Execute: Execute{ 109 | Details: Details{ 110 | Timestamp: time20190225, 111 | SessionID: "6480e39e.1c73", 112 | User: "postgres", 113 | Database: "postgres", 114 | }, 115 | Query: "select t.oid from test t where id = $1", 116 | }, 117 | Parameters: []interface{}{"41145"}, 118 | }, 119 | Statement{ 120 | Details: Details{ 121 | Timestamp: time20190225, 122 | SessionID: "6480e39e.1c73", 123 | User: "postgres", 124 | Database: "postgres", 125 | }, 126 | Query: "SELECT p.name, r.rating\n\t\t\t\t\t\tFROM products p\n\t\t\t\t\t\tJOIN reviews r ON p.id = r.product_id\n\t\t\t\t\t\tWHERE r.rating IN (\n\t\t\t\t\t\tSELECT MIN(rating) FROM reviews\n\t\t\t\t\t\tUNION\n\t\t\t\t\t\tSELECT MAX(rating) FROM reviews\n\t\t\t\t\t\t);\n\t\t\t\t", 127 | }, 128 | Statement{ 129 | Details: Details{ 130 | Timestamp: time20190225, 131 | SessionID: "6480e39e.1c73", 132 | User: "postgres", 133 | Database: "postgres", 134 | }, 135 | Query: "SELECT name, email\n\t\t\t\t\t\tFROM users\n\t\t\t\t\t\tWHERE email LIKE '@gmail.com';\n\t\t\t\t", 136 | }, 137 | }, 138 | ), 139 | ) 140 | }) 141 | 142 | var _ = Describe("ParseErrlog", func() { 143 | DescribeTable("Parses", 144 | func(input string, expected []Item) { 145 | var items = []Item{} 146 | itemsChan, errs, done := ParseErrlog(strings.NewReader(input)) 147 | go func() { 148 | for range errs { 149 | // no-op, just drain the channel 150 | } 151 | }() 152 | 153 | for item := range itemsChan { 154 | if item != nil { 155 | items = append(items, item) 156 | } 157 | } 158 | 159 | Eventually(done).Should(BeClosed()) 160 | Expect(len(items)).To(Equal(len(expected))) 161 | 162 | for idx, item := range items { 163 | Expect(item).To(BeEquivalentTo(expected[idx])) 164 | } 165 | }, 166 | Entry( 167 | "Extended protocol with duration logs", 168 | ` 169 | 2019-02-25 15:08:27.222 GMT|[unknown]|[unknown]|5c7404eb.d6bd|LOG: connection received: host=127.0.0.1 port=59103 170 | 2019-02-25 15:08:27.222 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: connection authorized: user=alice database=pgreplay_test 171 | 2019-02-25 15:08:27.222 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.968 ms parse : select t.oid 172 | 2019-02-25 15:08:27.222 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 1.100 ms bind : select t.oid 173 | 2019-02-25 15:08:27.222 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: execute : select t.oid 174 | 2019-02-25 15:08:27.222 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.326 ms 175 | 176 | 2019-02-25 15:08:27.222 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.042 ms parse : insert into logs (author, message) ($1, $2) 177 | 2019-02-25 15:08:27.222 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.045 ms bind : insert into logs (author, message) ($1, $2) 178 | 2019-02-25 15:08:27.222 GMT|alice|pgreplay_test|5c7404eb.d6bd|DETAIL: parameters: $1 = 'alice', $2 = 'bob' 179 | 2019-02-25 15:08:27.222 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: execute : insert into logs (author, message) ($1, $2) 180 | 2019-02-25 15:08:27.222 GMT|alice|pgreplay_test|5c7404eb.d6bd|DETAIL: parameters: $1 = 'alice', $2 = 'bob' 181 | 2019-02-25 15:08:27.222 GMT|alice|pgreplay_test|5c7404eb.d6bd|LOG: duration: 0.042 ms`, 182 | []Item{ 183 | Connect{ 184 | Details{ 185 | Timestamp: time20190225, 186 | SessionID: "5c7404eb.d6bd", 187 | User: "alice", 188 | Database: "pgreplay_test", 189 | }, 190 | }, 191 | BoundExecute{ 192 | Execute: Execute{ 193 | Details: Details{ 194 | Timestamp: time20190225, 195 | SessionID: "5c7404eb.d6bd", 196 | User: "alice", 197 | Database: "pgreplay_test", 198 | }, 199 | Query: "select t.oid", 200 | }, 201 | Parameters: []interface{}{}, 202 | }, 203 | BoundExecute{ 204 | Execute: Execute{ 205 | Details: Details{ 206 | Timestamp: time20190225, 207 | SessionID: "5c7404eb.d6bd", 208 | User: "alice", 209 | Database: "pgreplay_test", 210 | }, 211 | Query: "insert into logs (author, message) ($1, $2)", 212 | }, 213 | Parameters: []interface{}{"alice", "bob"}, 214 | }, 215 | }, 216 | ), 217 | ) 218 | }) 219 | 220 | var _ = Describe("ParseBindParameters", func() { 221 | DescribeTable("Parses", 222 | func(input string, expected []interface{}) { 223 | Expect(ParseBindParameters(input, nil)).To( 224 | BeEquivalentTo(expected), 225 | ) 226 | }, 227 | Entry("Single string parameter", "$1 = 'hello'", []interface{}{"hello"}), 228 | Entry("Single escaped string parameter", "$1 = 'hel''lo'", []interface{}{"hel'lo"}), 229 | Entry("NULL to nil", "$2 = NULL", []interface{}{nil}), 230 | Entry("Many string parameters", "$1 = 'hello', $2 = 'world'", []interface{}{"hello", "world"}), 231 | Entry("Many string parameters", "$1 = '41145', $2 = '2018-05-03 10:26:27.905086+00'", []interface{}{"41145", "2018-05-03 10:26:27.905086+00"}), 232 | ) 233 | }) 234 | 235 | var _ = Describe("LogScanner", func() { 236 | DescribeTable("Scans", 237 | func(input string, expected []string) { 238 | scanner := NewLogScanner(strings.NewReader(input), nil) 239 | lines := []string{} 240 | 241 | for scanner.Scan() { 242 | lines = append(lines, scanner.Text()) 243 | } 244 | 245 | Expect(scanner.Err()).NotTo(HaveOccurred()) 246 | Expect(lines).To(Equal(expected)) 247 | }, 248 | Entry( 249 | "Single lines", 250 | `2010-12-31 10:59:52.243 UTC|postgres`, 251 | []string{ 252 | `2010-12-31 10:59:52.243 UTC|postgres`, 253 | }, 254 | ), 255 | Entry( 256 | "Multiple lines", 257 | ` 258 | 2010-12-31 10:59:52.243 UTC|postgres 259 | 2010-12-31 10:59:53.000 UTC|paysvc`, 260 | []string{ 261 | `2010-12-31 10:59:52.243 UTC|postgres`, 262 | `2010-12-31 10:59:53.000 UTC|paysvc`, 263 | }, 264 | ), 265 | Entry( 266 | "Multi-line lines", 267 | ` 268 | 2018-05-03|gc|LOG: statement: select max(id),min(id) from pg2pubsub.update_log; 269 | 2018-05-03|gc|LOG: duration: 0.096 ms parse : 270 | DELETE FROM que_jobs 271 | WHERE queue = $1::text 272 | 273 | 2018-05-03|gc|LOG: duration: 0.248 ms 274 | `, 275 | []string{ 276 | `2018-05-03|gc|LOG: statement: select max(id),min(id) from pg2pubsub.update_log;`, 277 | "2018-05-03|gc|LOG: duration: 0.096 ms parse :\nDELETE FROM que_jobs\nWHERE queue = $1::text", 278 | `2018-05-03|gc|LOG: duration: 0.248 ms`, 279 | }, 280 | ), 281 | ) 282 | }) 283 | -------------------------------------------------------------------------------- /pkg/pgreplay/prom_server.go: -------------------------------------------------------------------------------- 1 | package pgreplay 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "time" 8 | 9 | kitlog "github.com/go-kit/log" 10 | "github.com/prometheus/client_golang/prometheus/promhttp" 11 | ) 12 | 13 | func StartPrometheusServer(logger kitlog.Logger, address string, port uint16) *http.Server { 14 | // Server Configuration 15 | mux := http.NewServeMux() 16 | mux.Handle("/metrics", promhttp.Handler()) 17 | server := &http.Server{ 18 | Addr: fmt.Sprintf("%s:%v", address, port), 19 | Handler: mux, 20 | } 21 | 22 | // Starting the servier 23 | go func() { 24 | logger.Log("event", "metrics.listen", "address", address, "port", port) 25 | if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { 26 | logger.Log("error", "server.not-started", "message", err.Error()) 27 | return 28 | } 29 | }() 30 | 31 | return server 32 | } 33 | 34 | func ShutdownServer(ctx context.Context, server *http.Server) error { 35 | // Waiting for Prometheus to get all the data left 36 | time.Sleep(5 * time.Second) 37 | 38 | // Shutdown the server gracefully 39 | if err := server.Shutdown(ctx); err != nil { 40 | return err 41 | } 42 | 43 | return nil 44 | } 45 | -------------------------------------------------------------------------------- /pkg/pgreplay/streamer.go: -------------------------------------------------------------------------------- 1 | package pgreplay 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | kitlog "github.com/go-kit/log" 8 | "github.com/go-kit/log/level" 9 | "github.com/prometheus/client_golang/prometheus" 10 | "github.com/prometheus/client_golang/prometheus/promauto" 11 | ) 12 | 13 | var ( 14 | itemsFilteredTotal = promauto.NewCounter( 15 | prometheus.CounterOpts{ 16 | Name: "pgreplay_items_filtered_total", 17 | Help: "Number of items filtered by start/finish range", 18 | }, 19 | ) 20 | itemsFilterProgressFraction = promauto.NewGauge( 21 | prometheus.GaugeOpts{ 22 | Name: "pgreplay_items_filter_progress_fraction", 23 | Help: "Fractional progress through filter range, assuming linear distribution", 24 | }, 25 | ) 26 | _ = promauto.NewGauge( 27 | prometheus.GaugeOpts{ 28 | Name: "pgreplay_items_last_streamed_timestamp", 29 | Help: "Timestamp of last streamed item", 30 | }, 31 | ) 32 | ) 33 | 34 | // StreamFilterBufferSize is the size of the channel buffer when filtering items for a 35 | // time range 36 | var StreamFilterBufferSize = 100 37 | 38 | type Streamer struct { 39 | start *time.Time 40 | finish *time.Time 41 | logger kitlog.Logger 42 | } 43 | 44 | func NewStreamer(start, finish *time.Time, logger kitlog.Logger) Streamer { 45 | return Streamer{start, finish, logger} 46 | } 47 | 48 | // Stream takes all the items from the given items channel and returns a channel that will 49 | // receive those events at a simulated given rate. 50 | func (s Streamer) Stream(items chan Item, rate float64) (chan Item, error) { 51 | if rate < 0 { 52 | return nil, fmt.Errorf("cannot support negative rates: %v", rate) 53 | } 54 | 55 | out := make(chan Item) 56 | 57 | go func() { 58 | var first, start time.Time 59 | var seenItem bool 60 | 61 | for item := range s.Filter(items) { 62 | if !seenItem { 63 | first = item.GetTimestamp() 64 | start = time.Now() 65 | seenItem = true 66 | } 67 | 68 | elapsedSinceStart := time.Duration(rate) * time.Since(start) 69 | elapsedSinceFirst := item.GetTimestamp().Sub(first) 70 | 71 | if diff := elapsedSinceFirst - elapsedSinceStart; diff > 0 { 72 | time.Sleep(time.Duration(float64(diff) / rate)) 73 | } 74 | 75 | level.Debug(s.logger).Log( 76 | "event", "queing.item", 77 | "sessionID", string(item.GetSessionID()), 78 | "user", string(item.GetUser()), 79 | ) 80 | out <- item 81 | } 82 | 83 | close(out) 84 | }() 85 | 86 | return out, nil 87 | } 88 | 89 | // Filter takes a Item stream and filters all items that don't match the desired 90 | // time range, along with any items that are nil. Filtering of items before our start 91 | // happens synchronously on first call, which will block initially until matching items 92 | // are found. 93 | // 94 | // This function assumes that items are pushed down the channel in chronological order. 95 | func (s Streamer) Filter(items chan Item) chan Item { 96 | if s.start != nil { 97 | for item := range items { 98 | if item == nil { 99 | continue 100 | } 101 | 102 | if item.GetTimestamp().After(*s.start) { 103 | break 104 | } 105 | 106 | itemsFilteredTotal.Inc() 107 | } 108 | } 109 | 110 | out := make(chan Item, StreamFilterBufferSize) 111 | 112 | go func() { 113 | for item := range items { 114 | if item == nil { 115 | continue 116 | } 117 | 118 | if s.finish != nil { 119 | if item.GetTimestamp().After(*s.finish) { 120 | break 121 | } 122 | 123 | if s.start != nil { 124 | itemsFilterProgressFraction.Set( 125 | float64(item.GetTimestamp().Sub(*s.start)) / float64((*s.finish).Sub(*s.start)), 126 | ) 127 | } 128 | } 129 | 130 | out <- item 131 | } 132 | 133 | close(out) 134 | }() 135 | 136 | return out 137 | } 138 | -------------------------------------------------------------------------------- /pkg/pgreplay/suite_test.go: -------------------------------------------------------------------------------- 1 | package pgreplay 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestSuite(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "pkg/pgreplay") 13 | } 14 | -------------------------------------------------------------------------------- /pkg/pgreplay/types.go: -------------------------------------------------------------------------------- 1 | package pgreplay 2 | 3 | import ( 4 | "context" 5 | stdjson "encoding/json" 6 | "fmt" 7 | "regexp" 8 | "strings" 9 | "time" 10 | 11 | pgx "github.com/jackc/pgx/v5" 12 | jsoniter "github.com/json-iterator/go" 13 | ) 14 | 15 | var json = jsoniter.ConfigCompatibleWithStandardLibrary 16 | 17 | type ( 18 | ReplayType int 19 | SessionID string 20 | ) 21 | 22 | type DatabaseConnConfig struct { 23 | Host string 24 | Port uint16 25 | Database string 26 | User string 27 | Password string 28 | } 29 | 30 | type ExtractedLog struct { 31 | Details 32 | ActionLog string 33 | Message string 34 | Parameters string 35 | } 36 | 37 | type LogMessage struct { 38 | actionType string 39 | statement string 40 | regex *regexp.Regexp 41 | } 42 | 43 | func (lm LogMessage) Prefix(parsedFrom string) string { 44 | if parsedFrom == ParsedFromErrLog { 45 | return lm.actionType + lm.statement 46 | } 47 | 48 | return lm.statement 49 | } 50 | 51 | func (lm LogMessage) Match(logline, parsedFrom string) bool { 52 | if parsedFrom == ParsedFromErrLog { 53 | logline = strings.TrimPrefix(logline, lm.actionType) 54 | } 55 | 56 | return lm.regex.MatchString(logline) 57 | } 58 | 59 | func (lm LogMessage) RenderQuery(msg, parsedFrom string) string { 60 | if parsedFrom == ParsedFromCsv { 61 | return msg[len(lm.regex.FindString(msg)):] 62 | } 63 | 64 | return strings.TrimPrefix(msg, lm.Prefix(parsedFrom)) 65 | } 66 | 67 | const ( 68 | ConnectLabel = "Connect" 69 | StatementLabel = "Statement" 70 | BoundExecuteLabel = "BoundExecute" 71 | DisconnectLabel = "Disconnect" 72 | ) 73 | 74 | func ItemMarshalJSON(item Item) ([]byte, error) { 75 | type envelope struct { 76 | Type string `json:"type"` 77 | Item Item `json:"item"` 78 | } 79 | 80 | switch item.(type) { 81 | case Connect, *Connect: 82 | return json.Marshal(envelope{Type: ConnectLabel, Item: item}) 83 | case Statement, *Statement: 84 | return json.Marshal(envelope{Type: StatementLabel, Item: item}) 85 | case BoundExecute, *BoundExecute: 86 | return json.Marshal(envelope{Type: BoundExecuteLabel, Item: item}) 87 | case Disconnect, *Disconnect: 88 | return json.Marshal(envelope{Type: DisconnectLabel, Item: item}) 89 | default: 90 | return nil, nil // it's not important for us to serialize this 91 | } 92 | } 93 | 94 | func ItemUnmarshalJSON(payload []byte) (Item, error) { 95 | envelope := struct { 96 | Type string `json:"type"` 97 | Item stdjson.RawMessage `json:"item"` 98 | }{} 99 | 100 | if err := json.Unmarshal(payload, &envelope); err != nil { 101 | return nil, err 102 | } 103 | 104 | var item Item 105 | 106 | switch envelope.Type { 107 | case ConnectLabel: 108 | item = &Connect{} 109 | case StatementLabel: 110 | item = &Statement{} 111 | case BoundExecuteLabel: 112 | item = &BoundExecute{} 113 | case DisconnectLabel: 114 | item = &Disconnect{} 115 | default: 116 | return nil, fmt.Errorf("did not recognise type: %s", envelope.Type) 117 | } 118 | 119 | return item, json.Unmarshal(envelope.Item, item) 120 | } 121 | 122 | // We support the following types of ReplayItem 123 | var _ Item = &Connect{} 124 | var _ Item = &Disconnect{} 125 | var _ Item = &Statement{} 126 | var _ Item = &BoundExecute{} 127 | 128 | type Item interface { 129 | GetTimestamp() time.Time 130 | GetSessionID() SessionID 131 | GetUser() string 132 | GetDatabase() string 133 | Handle(context.Context, *pgx.Conn) error 134 | } 135 | 136 | type Details struct { 137 | Timestamp time.Time `json:"timestamp"` 138 | SessionID SessionID `json:"session_id"` 139 | User string `json:"user"` 140 | Database string `json:"database"` 141 | } 142 | 143 | func (e Details) GetTimestamp() time.Time { return e.Timestamp } 144 | func (e Details) GetSessionID() SessionID { return e.SessionID } 145 | func (e Details) GetUser() string { return e.User } 146 | func (e Details) GetDatabase() string { return e.Database } 147 | 148 | type Connect struct{ Details } 149 | 150 | func (Connect) Handle(context.Context, *pgx.Conn) error { 151 | return nil // Database will manage opening connections 152 | } 153 | 154 | type Disconnect struct{ Details } 155 | 156 | func (Disconnect) Handle(ctx context.Context, conn *pgx.Conn) error { 157 | return conn.Close(ctx) 158 | } 159 | 160 | type Statement struct { 161 | Details 162 | Query string `json:"query"` 163 | } 164 | 165 | func (s Statement) Handle(ctx context.Context, conn *pgx.Conn) error { 166 | _, err := conn.Exec(ctx, s.Query) 167 | return err 168 | } 169 | 170 | // Execute is parsed and awaiting arguments. It deliberately lacks a Handle method as it 171 | // shouldn't be possible this statement to have been parsed without a following duration 172 | // or detail line that bound it. 173 | type Execute struct { 174 | Details 175 | Query string `json:"query"` 176 | } 177 | 178 | func (e Execute) Bind(parameters []interface{}) BoundExecute { 179 | if parameters == nil { 180 | parameters = make([]interface{}, 0) 181 | } 182 | 183 | return BoundExecute{e, parameters} 184 | } 185 | 186 | // BoundExecute represents an Execute that is now successfully bound with parameters 187 | type BoundExecute struct { 188 | Execute 189 | Parameters []interface{} `json:"parameters"` 190 | } 191 | 192 | func (e BoundExecute) Handle(ctx context.Context, conn *pgx.Conn) error { 193 | _, err := conn.Exec(ctx, e.Query, e.Parameters...) 194 | return err 195 | } 196 | -------------------------------------------------------------------------------- /pkg/pgreplay/types_test.go: -------------------------------------------------------------------------------- 1 | package pgreplay 2 | 3 | import ( 4 | . "github.com/onsi/ginkgo" 5 | . "github.com/onsi/gomega" 6 | ) 7 | 8 | var _ = Describe("Item JSON", func() { 9 | var ( 10 | details = Details{ 11 | Timestamp: time20190225, 12 | SessionID: "5c7404eb.d6bd", 13 | User: "alice", 14 | Database: "pgreplay_test", 15 | } 16 | ) 17 | 18 | Context("Statement", func() { 19 | var item = Statement{details, "select now()"} 20 | 21 | It("Generates JSON", func() { 22 | Expect(ItemMarshalJSON(item)).To( 23 | MatchJSON(` 24 | { 25 | "type": "Statement", 26 | "item": { 27 | "timestamp": "2019-02-25T15:08:27.222Z", 28 | "session_id": "5c7404eb.d6bd", 29 | "user": "alice", 30 | "database": "pgreplay_test", 31 | "query": "select now()" 32 | } 33 | }`), 34 | ) 35 | }) 36 | }) 37 | 38 | Context("BoundExecute", func() { 39 | var item = BoundExecute{Execute{details, "select $1"}, []interface{}{"hello"}} 40 | 41 | It("Generates JSON", func() { 42 | Expect(ItemMarshalJSON(item)).To( 43 | MatchJSON(` 44 | { 45 | "type": "BoundExecute", 46 | "item": { 47 | "timestamp": "2019-02-25T15:08:27.222Z", 48 | "session_id": "5c7404eb.d6bd", 49 | "user": "alice", 50 | "database": "pgreplay_test", 51 | "query": "select $1", 52 | "parameters": ["hello"] 53 | } 54 | }`), 55 | ) 56 | }) 57 | }) 58 | }) 59 | -------------------------------------------------------------------------------- /res/grafana-dashboard-pgreplay-go.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "limit": 100, 11 | "name": "Annotations & Alerts", 12 | "showIn": 0, 13 | "tags": [ 14 | "alerts" 15 | ], 16 | "type": "tags" 17 | } 18 | ] 19 | }, 20 | "description": "Benchmarking dashboard for https://github.com/gocardless/pgreplay-go", 21 | "editable": true, 22 | "gnetId": null, 23 | "graphTooltip": 1, 24 | "id": 919, 25 | "iteration": 1658504579327, 26 | "links": [], 27 | "panels": [ 28 | { 29 | "collapsed": false, 30 | "datasource": null, 31 | "gridPos": { 32 | "h": 1, 33 | "w": 24, 34 | "x": 0, 35 | "y": 0 36 | }, 37 | "id": 28, 38 | "panels": [], 39 | "title": "Benchmarking", 40 | "type": "row" 41 | }, 42 | { 43 | "aliasColors": {}, 44 | "bars": false, 45 | "dashLength": 10, 46 | "dashes": false, 47 | "datasource": "$datasource", 48 | "fieldConfig": { 49 | "defaults": { 50 | "links": [] 51 | }, 52 | "overrides": [] 53 | }, 54 | "fill": 1, 55 | "fillGradient": 0, 56 | "gridPos": { 57 | "h": 6, 58 | "w": 8, 59 | "x": 0, 60 | "y": 1 61 | }, 62 | "hiddenSeries": false, 63 | "id": 30, 64 | "legend": { 65 | "avg": false, 66 | "current": false, 67 | "max": false, 68 | "min": false, 69 | "show": true, 70 | "total": false, 71 | "values": false 72 | }, 73 | "lines": true, 74 | "linewidth": 1, 75 | "links": [], 76 | "nullPointMode": "null", 77 | "options": { 78 | "alertThreshold": true 79 | }, 80 | "percentage": false, 81 | "pluginVersion": "7.5.11", 82 | "pointradius": 5, 83 | "points": false, 84 | "renderer": "flot", 85 | "seriesOverrides": [], 86 | "spaceLength": 10, 87 | "stack": false, 88 | "steppedLine": false, 89 | "targets": [ 90 | { 91 | "exemplar": true, 92 | "expr": "rate(pgreplay_items_processed_total{instance=\"$node\"}[$interval])", 93 | "format": "time_series", 94 | "interval": "", 95 | "intervalFactor": 1, 96 | "legendFormat": "items/s", 97 | "refId": "A" 98 | } 99 | ], 100 | "thresholds": [], 101 | "timeFrom": null, 102 | "timeRegions": [], 103 | "timeShift": null, 104 | "title": "Items Processed/s", 105 | "tooltip": { 106 | "shared": true, 107 | "sort": 0, 108 | "value_type": "individual" 109 | }, 110 | "type": "graph", 111 | "xaxis": { 112 | "buckets": null, 113 | "mode": "time", 114 | "name": null, 115 | "show": true, 116 | "values": [] 117 | }, 118 | "yaxes": [ 119 | { 120 | "$$hashKey": "object:399", 121 | "format": "short", 122 | "label": null, 123 | "logBase": 1, 124 | "max": null, 125 | "min": "0", 126 | "show": true 127 | }, 128 | { 129 | "$$hashKey": "object:400", 130 | "format": "short", 131 | "label": null, 132 | "logBase": 1, 133 | "max": null, 134 | "min": null, 135 | "show": true 136 | } 137 | ], 138 | "yaxis": { 139 | "align": false, 140 | "alignLevel": null 141 | } 142 | }, 143 | { 144 | "aliasColors": {}, 145 | "bars": false, 146 | "dashLength": 10, 147 | "dashes": false, 148 | "datasource": "$datasource", 149 | "fieldConfig": { 150 | "defaults": { 151 | "links": [] 152 | }, 153 | "overrides": [] 154 | }, 155 | "fill": 1, 156 | "fillGradient": 0, 157 | "gridPos": { 158 | "h": 6, 159 | "w": 8, 160 | "x": 8, 161 | "y": 1 162 | }, 163 | "hiddenSeries": false, 164 | "id": 32, 165 | "legend": { 166 | "avg": false, 167 | "current": false, 168 | "max": false, 169 | "min": false, 170 | "show": true, 171 | "total": false, 172 | "values": false 173 | }, 174 | "lines": true, 175 | "linewidth": 1, 176 | "links": [], 177 | "nullPointMode": "null", 178 | "options": { 179 | "alertThreshold": true 180 | }, 181 | "percentage": false, 182 | "pluginVersion": "7.5.11", 183 | "pointradius": 5, 184 | "points": false, 185 | "renderer": "flot", 186 | "seriesOverrides": [ 187 | { 188 | "$$hashKey": "object:654", 189 | "alias": "established/s", 190 | "yaxis": 2 191 | }, 192 | { 193 | "$$hashKey": "object:655", 194 | "alias": "goroutines/s", 195 | "yaxis": 2 196 | } 197 | ], 198 | "spaceLength": 10, 199 | "stack": false, 200 | "steppedLine": false, 201 | "targets": [ 202 | { 203 | "exemplar": true, 204 | "expr": "pgreplay_connections_active{instance=\"$node\"}", 205 | "format": "time_series", 206 | "interval": "", 207 | "intervalFactor": 1, 208 | "legendFormat": "active", 209 | "refId": "A" 210 | }, 211 | { 212 | "exemplar": true, 213 | "expr": "rate(pgreplay_connections_established_total{instance=\"$node\"}[$interval])", 214 | "format": "time_series", 215 | "interval": "", 216 | "intervalFactor": 1, 217 | "legendFormat": "established/s", 218 | "refId": "B" 219 | } 220 | ], 221 | "thresholds": [], 222 | "timeFrom": null, 223 | "timeRegions": [], 224 | "timeShift": null, 225 | "title": "Connections", 226 | "tooltip": { 227 | "shared": true, 228 | "sort": 0, 229 | "value_type": "individual" 230 | }, 231 | "type": "graph", 232 | "xaxis": { 233 | "buckets": null, 234 | "mode": "time", 235 | "name": null, 236 | "show": true, 237 | "values": [] 238 | }, 239 | "yaxes": [ 240 | { 241 | "$$hashKey": "object:668", 242 | "format": "short", 243 | "label": null, 244 | "logBase": 1, 245 | "max": null, 246 | "min": null, 247 | "show": true 248 | }, 249 | { 250 | "$$hashKey": "object:669", 251 | "format": "short", 252 | "label": null, 253 | "logBase": 1, 254 | "max": null, 255 | "min": null, 256 | "show": true 257 | } 258 | ], 259 | "yaxis": { 260 | "align": false, 261 | "alignLevel": null 262 | } 263 | }, 264 | { 265 | "cacheTimeout": null, 266 | "colorBackground": false, 267 | "colorValue": false, 268 | "colors": [ 269 | "#299c46", 270 | "rgba(237, 129, 40, 0.89)", 271 | "#d44a3a" 272 | ], 273 | "datasource": "$datasource", 274 | "fieldConfig": { 275 | "defaults": {}, 276 | "overrides": [] 277 | }, 278 | "format": "s", 279 | "gauge": { 280 | "maxValue": 100, 281 | "minValue": 0, 282 | "show": false, 283 | "thresholdLabels": false, 284 | "thresholdMarkers": true 285 | }, 286 | "gridPos": { 287 | "h": 3, 288 | "w": 8, 289 | "x": 16, 290 | "y": 1 291 | }, 292 | "id": 36, 293 | "interval": null, 294 | "links": [], 295 | "mappingType": 1, 296 | "mappingTypes": [ 297 | { 298 | "name": "value to text", 299 | "value": 1 300 | }, 301 | { 302 | "name": "range to text", 303 | "value": 2 304 | } 305 | ], 306 | "maxDataPoints": 100, 307 | "nullPointMode": "connected", 308 | "nullText": null, 309 | "postfix": "", 310 | "postfixFontSize": "50%", 311 | "prefix": "", 312 | "prefixFontSize": "50%", 313 | "rangeMaps": [ 314 | { 315 | "from": "null", 316 | "text": "N/A", 317 | "to": "null" 318 | } 319 | ], 320 | "sparkline": { 321 | "fillColor": "rgba(31, 118, 189, 0.18)", 322 | "full": false, 323 | "lineColor": "rgb(31, 120, 193)", 324 | "show": false 325 | }, 326 | "tableColumn": "", 327 | "targets": [ 328 | { 329 | "exemplar": true, 330 | "expr": "clamp_min((1.0 - pgreplay_items_filter_progress_fraction{instance=\"$node\"}) / rate(pgreplay_items_filter_progress_fraction{instance=\"$node\"}[$interval]), 0)", 331 | "format": "time_series", 332 | "interval": "", 333 | "intervalFactor": 1, 334 | "legendFormat": "", 335 | "refId": "A" 336 | } 337 | ], 338 | "thresholds": "", 339 | "title": "ETA", 340 | "type": "singlestat", 341 | "valueFontSize": "80%", 342 | "valueMaps": [ 343 | { 344 | "op": "=", 345 | "text": "N/A", 346 | "value": "null" 347 | } 348 | ], 349 | "valueName": "current" 350 | }, 351 | { 352 | "cacheTimeout": null, 353 | "colorBackground": false, 354 | "colorValue": false, 355 | "colors": [ 356 | "#d44a3a", 357 | "rgba(237, 129, 40, 0.89)", 358 | "#299c46" 359 | ], 360 | "datasource": "$datasource", 361 | "fieldConfig": { 362 | "defaults": {}, 363 | "overrides": [] 364 | }, 365 | "format": "percentunit", 366 | "gauge": { 367 | "maxValue": 100, 368 | "minValue": 0, 369 | "show": true, 370 | "thresholdLabels": false, 371 | "thresholdMarkers": true 372 | }, 373 | "gridPos": { 374 | "h": 3, 375 | "w": 8, 376 | "x": 16, 377 | "y": 4 378 | }, 379 | "id": 37, 380 | "interval": null, 381 | "links": [], 382 | "mappingType": 1, 383 | "mappingTypes": [ 384 | { 385 | "name": "value to text", 386 | "value": 1 387 | }, 388 | { 389 | "name": "range to text", 390 | "value": 2 391 | } 392 | ], 393 | "maxDataPoints": 100, 394 | "nullPointMode": "connected", 395 | "nullText": null, 396 | "postfix": "", 397 | "postfixFontSize": "50%", 398 | "prefix": "", 399 | "prefixFontSize": "50%", 400 | "rangeMaps": [ 401 | { 402 | "from": "null", 403 | "text": "N/A", 404 | "to": "null" 405 | } 406 | ], 407 | "sparkline": { 408 | "fillColor": "rgba(31, 118, 189, 0.18)", 409 | "full": false, 410 | "lineColor": "rgb(31, 120, 193)", 411 | "show": false 412 | }, 413 | "tableColumn": "", 414 | "targets": [ 415 | { 416 | "expr": "pgreplay_items_filter_progress_fraction{instance=\"$node\"}", 417 | "format": "time_series", 418 | "intervalFactor": 1, 419 | "refId": "A" 420 | } 421 | ], 422 | "thresholds": "50, 80,90", 423 | "title": "Progress", 424 | "type": "singlestat", 425 | "valueFontSize": "80%", 426 | "valueMaps": [ 427 | { 428 | "op": "=", 429 | "text": "N/A", 430 | "value": "null" 431 | } 432 | ], 433 | "valueName": "current" 434 | }, 435 | { 436 | "aliasColors": {}, 437 | "bars": false, 438 | "dashLength": 10, 439 | "dashes": false, 440 | "datasource": "$datasource", 441 | "fieldConfig": { 442 | "defaults": { 443 | "links": [] 444 | }, 445 | "overrides": [] 446 | }, 447 | "fill": 1, 448 | "fillGradient": 0, 449 | "gridPos": { 450 | "h": 7, 451 | "w": 24, 452 | "x": 0, 453 | "y": 7 454 | }, 455 | "hiddenSeries": false, 456 | "id": 34, 457 | "legend": { 458 | "avg": false, 459 | "current": false, 460 | "max": false, 461 | "min": false, 462 | "show": true, 463 | "total": false, 464 | "values": false 465 | }, 466 | "lines": true, 467 | "linewidth": 1, 468 | "links": [], 469 | "nullPointMode": "null", 470 | "options": { 471 | "alertThreshold": true 472 | }, 473 | "percentage": false, 474 | "pluginVersion": "7.5.11", 475 | "pointradius": 5, 476 | "points": false, 477 | "renderer": "flot", 478 | "seriesOverrides": [ 479 | { 480 | "$$hashKey": "object:480", 481 | "alias": "go_goroutines{instance=\"10.164.0.18:9187\",job=\"postgres-benchmark-postgres-exporter\"}", 482 | "yaxis": 2 483 | }, 484 | { 485 | "$$hashKey": "object:481", 486 | "alias": "goroutines", 487 | "yaxis": 2 488 | } 489 | ], 490 | "spaceLength": 10, 491 | "stack": false, 492 | "steppedLine": false, 493 | "targets": [ 494 | { 495 | "exemplar": true, 496 | "expr": "go_memstats_heap_inuse_bytes{instance=\"$node\"}\nand\npgreplay_connections_active{instance=\"$node\"}", 497 | "format": "time_series", 498 | "interval": "", 499 | "intervalFactor": 1, 500 | "legendFormat": "heap bytes", 501 | "refId": "A" 502 | }, 503 | { 504 | "exemplar": true, 505 | "expr": "go_goroutines{instance=\"$node\"}\nand\npgreplay_connections_active{instance=\"$node\"}", 506 | "format": "time_series", 507 | "interval": "", 508 | "intervalFactor": 1, 509 | "legendFormat": "goroutines", 510 | "refId": "B" 511 | } 512 | ], 513 | "thresholds": [], 514 | "timeFrom": null, 515 | "timeRegions": [], 516 | "timeShift": null, 517 | "title": "Go Runtime", 518 | "tooltip": { 519 | "shared": true, 520 | "sort": 0, 521 | "value_type": "individual" 522 | }, 523 | "type": "graph", 524 | "xaxis": { 525 | "buckets": null, 526 | "mode": "time", 527 | "name": null, 528 | "show": true, 529 | "values": [] 530 | }, 531 | "yaxes": [ 532 | { 533 | "$$hashKey": "object:494", 534 | "format": "decbytes", 535 | "label": null, 536 | "logBase": 1, 537 | "max": null, 538 | "min": null, 539 | "show": true 540 | }, 541 | { 542 | "$$hashKey": "object:495", 543 | "format": "short", 544 | "label": null, 545 | "logBase": 1, 546 | "max": null, 547 | "min": null, 548 | "show": true 549 | } 550 | ], 551 | "yaxis": { 552 | "align": false, 553 | "alignLevel": null 554 | } 555 | } 556 | ], 557 | "refresh": false, 558 | "schemaVersion": 27, 559 | "style": "dark", 560 | "tags": [], 561 | "templating": { 562 | "list": [ 563 | { 564 | "description": null, 565 | "error": null, 566 | "hide": 0, 567 | "includeAll": false, 568 | "label": null, 569 | "multi": false, 570 | "name": "datasource", 571 | "options": [], 572 | "query": "prometheus", 573 | "queryValue": "", 574 | "refresh": 1, 575 | "regex": "", 576 | "skipUrlSync": false, 577 | "type": "datasource" 578 | }, 579 | { 580 | "auto": false, 581 | "auto_count": 30, 582 | "auto_min": "10s", 583 | "current": { 584 | "selected": true, 585 | "text": "1m", 586 | "value": "1m" 587 | }, 588 | "description": null, 589 | "error": null, 590 | "hide": 0, 591 | "label": null, 592 | "name": "interval", 593 | "options": [ 594 | { 595 | "selected": true, 596 | "text": "1m", 597 | "value": "1m" 598 | }, 599 | { 600 | "selected": false, 601 | "text": "5m", 602 | "value": "5m" 603 | }, 604 | { 605 | "selected": false, 606 | "text": "15m", 607 | "value": "15m" 608 | } 609 | ], 610 | "query": "1m,5m,15m", 611 | "refresh": 2, 612 | "skipUrlSync": false, 613 | "type": "interval" 614 | }, 615 | { 616 | "allValue": null, 617 | "datasource": "${datasource}", 618 | "definition": "label_values(up{port=\"9445\"}, instance)", 619 | "description": null, 620 | "error": null, 621 | "hide": 0, 622 | "includeAll": false, 623 | "label": null, 624 | "multi": false, 625 | "name": "node", 626 | "options": [], 627 | "query": { 628 | "query": "label_values(up{port=\"9445\"}, instance)", 629 | "refId": "StandardVariableQuery" 630 | }, 631 | "refresh": 2, 632 | "regex": "", 633 | "skipUrlSync": false, 634 | "sort": 1, 635 | "tagValuesQuery": "", 636 | "tags": [], 637 | "tagsQuery": "", 638 | "type": "query", 639 | "useTags": false 640 | } 641 | ] 642 | }, 643 | "timepicker": { 644 | "refresh_intervals": [ 645 | "15s", 646 | "30s", 647 | "1m", 648 | "5m", 649 | "15m", 650 | "30m", 651 | "1h", 652 | "2h", 653 | "1d" 654 | ], 655 | "time_options": [ 656 | "5m", 657 | "15m", 658 | "1h", 659 | "6h", 660 | "12h", 661 | "24h", 662 | "2d", 663 | "7d", 664 | "30d" 665 | ] 666 | }, 667 | "timezone": "", 668 | "title": "pgreplay-go", 669 | "uid": "0sXjgPR4k", 670 | "version": 5 671 | } -------------------------------------------------------------------------------- /res/grafana.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gocardless/pgreplay-go/9dc847ebd3f8ac91bb07d4425b9b540fda3c9465/res/grafana.jpg --------------------------------------------------------------------------------