├── .github └── workflows │ └── goreleaser.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yaml ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── bench ├── 001_iam.sql ├── 002_transactions.sql ├── 003_mocks.sql ├── README.md ├── results.png └── script.js ├── cmd └── pgo │ ├── pipeline.go │ ├── rest.go │ └── root.go ├── dex-config.yaml ├── docker-compose.pipelines.yaml ├── docker-compose.yaml ├── docs ├── container-images │ ├── apache-age.Containerfile │ ├── pgvector.Containerfile │ ├── timescaledb.Containerfile │ ├── wal-g.Containerfile │ └── wal2json.Containerfile ├── example.pgcat.toml ├── img │ └── pgo.png ├── logical-replication.md ├── nats.conf ├── pipeline-example.docker.yaml ├── postgres-cdc.md ├── rag.md ├── rag.mmd └── rag.mmd.svg ├── examples ├── custom-middleware │ └── main.go ├── file-server │ └── main.go ├── handlers │ ├── health.go │ ├── oidc.go │ └── pgrole.go ├── logical-replication-cdc │ └── main.go ├── pipeline.chain.yaml ├── postgrest-like-oidc-authz │ └── main.go ├── rag │ └── main.go └── webdav │ ├── README.md │ └── webdav.go ├── go.mod ├── go.sum ├── internal └── testutil │ ├── cdc.json │ ├── k8s-svc.json │ ├── pgtest │ └── pgtest.go │ └── testdata.go ├── k8s.yaml ├── main.go ├── pgo-config.yaml ├── pkg ├── config │ ├── config.go │ ├── example.config.yaml │ ├── example.http-peers.yaml │ └── pg-debug-basic.config.yaml ├── httputil │ ├── client.go │ ├── context.go │ ├── middleware │ │ ├── authz.go │ │ ├── basic_auth.go │ │ ├── basic_auth_test.go │ │ ├── cache.go │ │ ├── cache_test.go │ │ ├── cors.go │ │ ├── cors_test.go │ │ ├── file_server.go │ │ ├── logger.go │ │ ├── logger_test.go │ │ ├── middleware.go │ │ ├── postgres.go │ │ ├── proxy.go │ │ ├── request_id.go │ │ ├── request_id_test.go │ │ └── verify_oidc.go │ ├── postgres.go │ ├── router.go │ └── router_test.go ├── metrics │ └── prom.go ├── pglogrepl │ ├── pglogrepl.go │ ├── process_v1.go │ ├── process_v2.go │ ├── stream.go │ ├── stream_test.go │ └── util.go ├── pgx │ ├── conn.go │ ├── conn_test.go │ ├── crud.go │ ├── doc.go │ ├── listen.go │ ├── listen_test.go │ ├── pool.go │ ├── pool_test.go │ ├── role │ │ ├── role.go │ │ └── role_test.go │ └── schema │ │ ├── openapi.go │ │ ├── schema.go │ │ └── schema_test.go ├── pipeline │ ├── cdc │ │ ├── cdc.go │ │ └── cdc_test.go │ ├── connector.go │ ├── connector_test.go │ ├── doc.go │ ├── manager.go │ ├── peer.go │ ├── peer │ │ ├── clickhouse │ │ │ └── peer.go │ │ ├── debug │ │ │ └── peer.go │ │ ├── grpc │ │ │ └── peer.go │ │ ├── http │ │ │ └── peer.go │ │ ├── kafka │ │ │ ├── acl.go │ │ │ ├── config.go │ │ │ ├── doc.go │ │ │ ├── kafka.go │ │ │ ├── peer.go │ │ │ ├── rest.go │ │ │ └── scram.go │ │ ├── mqtt │ │ │ ├── client.go │ │ │ ├── doc.go │ │ │ ├── options.go │ │ │ └── peer.go │ │ ├── nats │ │ │ ├── doc.go │ │ │ └── peer.go │ │ ├── pg │ │ │ └── peer.go │ │ └── plugin_example │ │ │ └── peer.go │ ├── pipeline.go │ ├── process.go │ └── transform │ │ ├── doc.go │ │ ├── extract.go │ │ ├── extract_test.go │ │ ├── filter.go │ │ ├── replace.go │ │ └── transform.go ├── rest │ ├── apache_age.go │ ├── apache_age_test.go │ ├── doc.go │ ├── query.go │ └── server.go ├── util │ ├── cert.go │ ├── cert_test.go │ ├── env.go │ ├── jq.go │ ├── jq_test.go │ └── rand │ │ ├── name.go │ │ └── password.go └── x │ ├── experiments.go │ ├── pgcache │ └── pgcache.go │ ├── pgproxy │ ├── main.go │ └── server.go │ └── rag │ ├── client.go │ ├── client_test.go │ ├── doc.go │ ├── embedding.go │ ├── embedding_test.go │ ├── generate.go │ ├── pgvector.go │ └── pgvector_test.go └── proto ├── cdc.proto └── generated ├── cdc.pb.go └── cdc_grpc.pb.go /.github/workflows/goreleaser.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | branches: 6 | - "main" 7 | tags: 8 | - "v*" 9 | 10 | permissions: 11 | contents: write 12 | packages: write 13 | id-token: write 14 | 15 | jobs: 16 | goreleaser: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Checkout 20 | uses: actions/checkout@v4 21 | with: 22 | fetch-depth: 0 23 | - uses: docker/login-action@v3.3.0 24 | with: 25 | registry: ghcr.io 26 | username: ${{ github.repository_owner }} 27 | password: ${{ secrets.GITHUB_TOKEN }} 28 | - name: Set up Go 29 | uses: actions/setup-go@v5 30 | with: 31 | go-version-file: go.mod 32 | check-latest: true 33 | - name: Run GoReleaser 34 | uses: goreleaser/goreleaser-action@v6 35 | with: 36 | distribution: goreleaser 37 | version: '~> v2' 38 | args: ${{ startsWith(github.ref, 'refs/tags/') && 'release --clean' || 'release --snapshot --clean' }} 39 | env: 40 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | bin/* 8 | Dockerfile.cross 9 | 10 | # Test binary, built with `go test -c` 11 | *.test 12 | 13 | # Output of the go coverage tool, specifically when used with LiteIDE 14 | *.out 15 | 16 | # Go workspace file 17 | go.work 18 | 19 | # Kubernetes Generated files - skip generated files, except for vendored files 20 | !vendor/**/zz_generated.* 21 | 22 | # editor and IDE paraphernalia 23 | .idea 24 | .vscode 25 | *.swp 26 | *.swo 27 | *~ 28 | 29 | # os 30 | .DS_Store 31 | 32 | # local stuff 33 | __* 34 | # goreleaser 35 | dist/ 36 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: 5m 3 | allow-parallel-runners: true 4 | 5 | issues: 6 | # don't skip warning about doc comments 7 | # don't exclude the default set of lint 8 | exclude-use-default: false 9 | # restore some of the defaults 10 | # (fill in the rest as needed) 11 | exclude-rules: 12 | - path: "pkg/x/*" 13 | linters: 14 | - all 15 | - path: _test\.go 16 | linters: 17 | - gocyclo 18 | - errcheck 19 | - dupl 20 | - gosec 21 | - govet 22 | - revive 23 | linters: 24 | disable-all: true 25 | enable: 26 | - dupl 27 | - errcheck 28 | - goconst 29 | - gocyclo 30 | - gofmt 31 | - goimports 32 | - gosimple 33 | - govet 34 | - ineffassign 35 | # - lll 36 | - misspell 37 | - nakedret 38 | - prealloc 39 | - staticcheck 40 | - typecheck 41 | - unconvert 42 | - unparam 43 | - unused 44 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | before: 4 | hooks: 5 | - go mod tidy 6 | - go generate ./... 7 | 8 | builds: 9 | - env: 10 | - CGO_ENABLED=0 11 | goos: 12 | - linux 13 | - windows 14 | - darwin 15 | 16 | changelog: 17 | sort: asc 18 | filters: 19 | exclude: 20 | - "^docs:" 21 | - "^test:" 22 | 23 | release: 24 | footer: >- 25 | 26 | --- 27 | 28 | Released by [GoReleaser](https://github.com/goreleaser/goreleaser). 29 | 30 | kos: 31 | - repositories: 32 | - ghcr.io/edgeflare/pgo 33 | # - edgeflare/pgo 34 | tags: 35 | - "{{.Version}}" 36 | - latest 37 | bare: true 38 | preserve_import_paths: false 39 | platforms: 40 | - linux/amd64 41 | - linux/arm64 42 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guide 2 | 3 | Hello and welcome! We’re excited you want to contribute to pgo. Here’s how you can help: 4 | 5 | ### Reporting Issues 6 | Found a bug or have a suggestion? Open an issue and provide as much detail as possible. 7 | 8 | ### Code Contributions 9 | 1. **Fork the Repo:** Fork the repository to your GitHub account: 10 | ```bash 11 | git clone git@github.com:/pgo.git 12 | ``` 13 | 2. **Clone Your Fork:** Clone it to your local machine: 14 | ```bash 15 | git clone git@github.com:/pgo.git 16 | ``` 17 | 3. **Create a Branch:** 18 | ```bash 19 | git checkout -b feat/feature-name # Use feat, bug, docs, etc. 20 | ``` 21 | 4. **Make Changes:** Make your changes in the new branch. 22 | 5. **Commit Changes:** 23 | ```bash 24 | git commit -m "Description of changes" 25 | ``` 26 | 6. **Push to Your Repo:** 27 | ```bash 28 | git push origin feat/feature-name 29 | ``` 30 | 7. **Open a PR:** Create a pull request from your repository to the `dev` branch of `git@github.com:edgeflare/pgo.git`. Provide a clear description. 31 | 32 | ### Code Style 33 | Follow our coding standards. If unsure, feel free to ask. 34 | 35 | ### Documentation 36 | Update documentation for any new features. 37 | 38 | 39 | Thank you for your contributions! -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.io/golang:1.24 AS builder 2 | ARG TARGETOS 3 | ARG TARGETARCH 4 | 5 | WORKDIR /workspace 6 | 7 | COPY ./go.mod go.mod 8 | COPY ./go.sum go.sum 9 | RUN go mod download 10 | 11 | COPY ./cmd cmd 12 | COPY ./pkg pkg 13 | COPY ./proto proto 14 | COPY ./main.go main.go 15 | 16 | RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o pgo . 17 | 18 | # runtime image 19 | FROM gcr.io/distroless/static:nonroot 20 | WORKDIR / 21 | COPY --from=builder /workspace/pgo . 22 | USER 65532:65532 23 | 24 | ENTRYPOINT ["/pgo"] 25 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: proto build image up down 2 | 3 | # detect container runtime (prefer podman if available) 4 | CONTAINER_RUNTIME := $(shell command -v podman 2> /dev/null || command -v docker 2> /dev/null) 5 | ifeq ($(CONTAINER_RUNTIME),) 6 | $(error No container runtime found. Please install podman or docker) 7 | endif 8 | 9 | CONTAINER_RUNTIME_NAME := $(shell basename $(CONTAINER_RUNTIME)) 10 | COMPOSE_CMD := $(if $(filter podman,$(CONTAINER_RUNTIME_NAME)),podman-compose,docker compose) 11 | 12 | proto: 13 | protoc -I=./proto --go_out=. --go-grpc_out=. ./proto/cdc.proto 14 | 15 | build: 16 | CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o pgo . 17 | 18 | image: 19 | $(CONTAINER_RUNTIME) build -t pgo . 20 | $(COMPOSE_CMD) -f docker-compose.yaml build 21 | 22 | up: 23 | $(COMPOSE_CMD) -f docker-compose.yaml up -d 24 | 25 | down: 26 | $(COMPOSE_CMD) -f docker-compose.yaml down 27 | -------------------------------------------------------------------------------- /bench/001_iam.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA IF NOT EXISTS iam; 2 | 3 | CREATE OR REPLACE FUNCTION iam.generate_dex_sub(p_id text, p_connector_id text) 4 | RETURNS text AS $$ 5 | DECLARE 6 | binary_data bytea; 7 | BEGIN 8 | binary_data := E'\\x0a'::bytea || 9 | set_byte(E'\\x00'::bytea, 0, length(p_id)) || 10 | convert_to(p_id, 'UTF8') || 11 | E'\\x12'::bytea || 12 | set_byte(E'\\x00'::bytea, 0, length(p_connector_id)) || 13 | convert_to(p_connector_id, 'UTF8'); 14 | 15 | -- Base64 encode, make URL-safe, and remove padding 16 | RETURN replace( 17 | replace( 18 | rtrim(encode(binary_data, 'base64'), '='), 19 | '+', '-'), 20 | '/', '_'); 21 | END; 22 | $$ LANGUAGE plpgsql IMMUTABLE; 23 | 24 | -- users table 25 | CREATE TABLE IF NOT EXISTS iam.users ( 26 | id TEXT NOT NULL, 27 | name TEXT, 28 | email TEXT, 29 | preferred_username TEXT, 30 | groups BYTEA, 31 | connector_id TEXT NOT NULL, 32 | sub TEXT PRIMARY KEY GENERATED ALWAYS AS (iam.generate_dex_sub(id, connector_id)) STORED, 33 | created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL, 34 | updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL 35 | ); 36 | 37 | CREATE UNIQUE INDEX users_id_connector_id_idx ON iam.users (id, connector_id); 38 | CREATE INDEX users_email_idx ON iam.users (email); 39 | 40 | CREATE OR REPLACE FUNCTION iam.user_jwt_sub() RETURNS TEXT AS $$ 41 | DECLARE 42 | jwt_claim_sub TEXT; 43 | BEGIN 44 | SELECT (current_setting('request.jwt.claims', true)::json->>'sub')::TEXT INTO jwt_claim_sub; 45 | RETURN jwt_claim_sub; 46 | END; 47 | $$ LANGUAGE plpgsql STABLE; 48 | -------------------------------------------------------------------------------- /bench/002_transactions.sql: -------------------------------------------------------------------------------- 1 | -- benchmark table 2 | CREATE TABLE IF NOT EXISTS transactions ( 3 | id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, 4 | user_id TEXT REFERENCES iam.users(sub) ON DELETE CASCADE, 5 | transaction_date TIMESTAMPTZ NOT NULL, 6 | amount DECIMAL(10,2) NOT NULL, 7 | transaction_type VARCHAR(20) NOT NULL, 8 | category VARCHAR(50) NOT NULL, 9 | merchant_name VARCHAR(100), 10 | payment_method VARCHAR(30) NOT NULL, 11 | status VARCHAR(20) NOT NULL, 12 | created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, 13 | updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP 14 | ); 15 | 16 | -- indexes for better query performance 17 | CREATE INDEX idx_user_id ON transactions(user_id); 18 | CREATE INDEX idx_transaction_date ON transactions(transaction_date); 19 | CREATE INDEX idx_amount ON transactions(amount); 20 | CREATE INDEX idx_status ON transactions(status); 21 | CREATE INDEX idx_category ON transactions(category); 22 | 23 | -- GRANT and Row Level Security (RLS) 24 | GRANT SELECT,INSERT,UPDATE,DELETE ON transactions TO authn; 25 | 26 | -- RLS 27 | ALTER TABLE transactions ENABLE ROW LEVEL SECURITY; 28 | 29 | CREATE POLICY insert_own_tx ON transactions FOR INSERT 30 | WITH CHECK (user_id = iam.user_jwt_sub()); 31 | 32 | CREATE POLICY select_own_tx ON transactions FOR SELECT 33 | USING (user_id = iam.user_jwt_sub()); 34 | 35 | CREATE POLICY update_own_tx ON transactions FOR UPDATE 36 | USING (user_id = iam.user_jwt_sub()) 37 | WITH CHECK (user_id = iam.user_jwt_sub()); 38 | 39 | CREATE POLICY delete_own_tx ON transactions FOR DELETE 40 | USING (user_id = iam.user_jwt_sub()); 41 | 42 | CREATE INDEX IF NOT EXISTS idx_transactions_user_id ON transactions(user_id); 43 | -------------------------------------------------------------------------------- /bench/README.md: -------------------------------------------------------------------------------- 1 | # PGO benchmarks 2 | 3 | We're comparing against the popular https://github.com/PostgREST/postgrest. 4 | 5 | 0. Adjust the IP address of Dex to the LAN IP or equivalent in 6 | - docker-compose.yaml's example-app from which we obtain JWT token issued by the Dex IdP 7 | - dex-config.yaml 8 | - pgo-config.yaml 9 | 10 | 1. Start all containers 11 | 12 | ```sh 13 | docker compose up -d 14 | ``` 15 | 16 | 1. Create JWT secret for PostgREST 17 | 18 | ```sh 19 | curl http://192.168.0.10:5556/dex/keys | jq '.keys[0]' > dex-jwks.json 20 | ``` 21 | 22 | 2. Create needed tables, functions etc from psql console 23 | 24 | ```sh 25 | PGHOST=localhost PGUSER=postgres PGPASSWORD=postgrespw PGDATABASE=main psql 26 | ``` 27 | - execute [001_iam.sql](./001_iam.sql) and [002_transactions.sql](./002_transactions.sql) 28 | 29 | 30 | 4. Once the SQL is executed successfully in the above step, restart all containers to reload schema cache etc 31 | 32 | ```sh 33 | docker compose down 34 | docker compose up -d 35 | ``` 36 | 37 | 5. Visit the example-app at [http://localhost:5555](http://localhost:5555). It should present 2 login options, 38 | each with 1 test user. *Login with Example* doesn't require username/password; use `admin@example.com:password` with email login. 39 | This triggers insertion of 2 rows in public.refresh_token in the `dex` database. PGO pipelines syncs the users in `iam.users` table 40 | in the `main` (your appliaction database). We now can reference `iam.users(sub)` from in table eg in `public.transactions`. 41 | 42 | 6. Insert the [003_mocks.sql](./003_mocks.sql) data into the `transactions` table using psql 43 | 44 | 7. Test the setup 45 | ```sh 46 | export TOKEN=eyJhbGciOiJS.... 47 | ``` 48 | 49 | PostgREST 50 | ```sh 51 | curl "localhost:3000/transactions?select=id,user_id" -H "authorization: Bearer $TOKEN" 52 | ``` 53 | 54 | pgo rest 55 | ```sh 56 | curl "localhost:8001/transactions?select=id,user_id" -H "authorization: Bearer $TOKEN" 57 | ``` 58 | 59 | Notice only the rows satisfiying `user_id == iam.user_jwt_sub()` are returned 60 | 61 | 8. Benchmark with [k6](https://github.com/grafana/k6) 62 | 63 | 64 | pgo rest 65 | 66 | ```sh 67 | export BASE_URL="http://localhost:8001/transactions?select=id,user_id" 68 | export VUS=10000 69 | k6 run bench/script.js 70 | ``` 71 | 72 | PostGREST 73 | 74 | ```sh 75 | export BASE_URL="http://localhost:3000/transactions?select=id,user_id" 76 | export VUS=1000 # can't comfortably handle more 1k+ virutal users 77 | k6 run bench/script.js 78 | ``` 79 | -------------------------------------------------------------------------------- /bench/results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/edgeflare/pgo/2bf5b5d2f7dab9c545cd45387ab1f2115623dcee/bench/results.png -------------------------------------------------------------------------------- /bench/script.js: -------------------------------------------------------------------------------- 1 | import http from 'k6/http'; 2 | import { check } from 'k6'; 3 | 4 | export const options = { 5 | // Virtual Users 6 | vus: __ENV.VUS || 1000, // Adjust 7 | duration: '10s', // duration 8 | 9 | thresholds: { 10 | http_req_failed: ['rate<0.1'], // Allow up to 10% errors 11 | }, 12 | }; 13 | 14 | export default function () { 15 | const url = __ENV.BASE_URL || 'http://localhost:8001/transactions?select=id,user_id' 16 | const res = http.get(url, { 17 | headers: { 18 | 'Authorization': `Bearer ${__ENV.TOKEN}`, 19 | } 20 | }); 21 | check(res, { 'status was 200': (r) => r.status == 200 }); 22 | } 23 | -------------------------------------------------------------------------------- /cmd/pgo/root.go: -------------------------------------------------------------------------------- 1 | package pgo 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/edgeflare/pgo/pkg/config" 8 | "github.com/spf13/cobra" 9 | "github.com/spf13/viper" 10 | ) 11 | 12 | var cfgFile string 13 | var logLevel string 14 | var cfg *config.Config 15 | var rootCmd = &cobra.Command{ 16 | Use: "pgo", 17 | Short: "PGO is a PostgreSQL CDC tool", 18 | Long: `pgo streams data among endpoints aka peers`, 19 | Run: func(cmd *cobra.Command, args []string) { 20 | versionFlag, _ := cmd.Flags().GetBool("version") 21 | if versionFlag { 22 | fmt.Println(config.Version) 23 | return 24 | } 25 | 26 | // If no subcommand is provided, print help 27 | cmd.Help() 28 | }, 29 | } 30 | 31 | func Main() { 32 | if err := rootCmd.Execute(); err != nil { 33 | fmt.Println(err) 34 | os.Exit(1) 35 | } 36 | } 37 | 38 | func init() { 39 | cobra.OnInitialize(initConfig) 40 | rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.config/pgo.yaml)") 41 | rootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "L", "info", "log requests at this level (debug, info, warn, error, fatal, none)") 42 | rootCmd.PersistentFlags().BoolP("version", "v", false, "Print the version number") 43 | 44 | // TODO: below flags should be in rest or pipeline cmd 45 | viper.BindPFlag("postgres.logrepl_conn_string", rootCmd.PersistentFlags().Lookup("postgres.logrepl_conn_string")) 46 | viper.BindPFlag("postgres.tables", rootCmd.PersistentFlags().Lookup("postgres.tables")) 47 | rootCmd.PersistentFlags().String("postgres.logrepl_conn_string", "", "PostgreSQL logical replication connection string") 48 | rootCmd.PersistentFlags().String("postgres.tables", "", "Comma-separated list of tables to replicate") 49 | 50 | // Add the pipeline subcommand 51 | rootCmd.AddCommand(pipelineCmd) 52 | } 53 | 54 | func initConfig() { 55 | var err error 56 | cfg, err = config.Load(cfgFile) 57 | if err != nil { 58 | fmt.Println("Error loading config:", err) 59 | os.Exit(1) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /dex-config.yaml: -------------------------------------------------------------------------------- 1 | issuer: http://192.168.0.10:5556/dex 2 | 3 | storage: 4 | type: postgres 5 | config: 6 | host: 192.168.0.10 7 | port: 5432 8 | database: dex 9 | user: dex 10 | password: dexpw 11 | ssl: 12 | mode: disable 13 | web: 14 | http: 0.0.0.0:5556 15 | allowedOrigins: ['*'] 16 | 17 | staticClients: 18 | - id: public-webui 19 | redirectURIs: 20 | - http://localhost:4200/signin/callback 21 | - http://localhost:4200/signout/callback 22 | name: public-webui 23 | public: true 24 | - id: oauth2-proxy 25 | redirectURIs: 26 | - http://127.0.0.1:5555/callback 27 | - http://localhost:5555/callback 28 | - http://127.0.0.1:4180/oauth2/callback 29 | name: oauth2-proxy 30 | secret: ZXhhbXBsZS1hcHAtc2VjcmV0 31 | trustedPeers: 32 | - public-webui 33 | 34 | connectors: 35 | - type: mockCallback 36 | id: mock 37 | name: Example 38 | #- type: github 39 | # id: github 40 | # name: GitHub 41 | # config: 42 | # clientID: example-client-id 43 | # clientSecret: example-client-secret 44 | # redirectURI: http://192.168.0.10:5556/dex/callback 45 | # - type: google 46 | # id: google 47 | # name: Google 48 | # config: 49 | # issuer: https://accounts.google.com 50 | # # Connector config values starting with a "$" will read from the environment. 51 | # clientID: $GOOGLE_CLIENT_ID 52 | # clientSecret: $GOOGLE_CLIENT_SECRET 53 | # redirectURI: http://127.0.0.1:5556/dex/callback 54 | # hostedDomains: 55 | # - $GOOGLE_HOSTED_DOMAIN 56 | 57 | # Let dex keep a list of passwords which can be used to login to dex. 58 | enablePasswordDB: true 59 | 60 | # A static list of passwords to login the end user. By identifying here, dex 61 | # won't look in its underlying storage for passwords. 62 | # 63 | # If this option isn't chosen users may be added through the gRPC API. 64 | staticPasswords: 65 | - email: "admin@example.com" 66 | # bcrypt hash of the string "password": $(echo password | htpasswd -BinC 10 admin | cut -d: -f2) 67 | hash: "$2a$10$2b2cU8CPhOTaGrs1HRQuAueS7JTT5ZHsHSzYiFPm1leZck7Mc8T4W" 68 | username: "admin" 69 | userID: "08a8684b-db88-4b73-90a9-3cd1661f5466" 70 | -------------------------------------------------------------------------------- /docs/container-images/apache-age.Containerfile: -------------------------------------------------------------------------------- 1 | ARG POSTGRES_VERSION=16.4.0 2 | 3 | FROM docker.io/edgeflare/postgresql:$POSTGRES_VERSION-wal-g-v3.0.3 4 | 5 | ARG AGE_VERSION=1.5.0 6 | 7 | USER root 8 | RUN apt update && apt install -y git build-essential libreadline-dev zlib1g-dev flex bison 9 | 10 | RUN git clone --branch release/PG16/${AGE_VERSION} https://github.com/apache/age.git /tmp/age && \ 11 | cd /tmp/age && \ 12 | make install 13 | 14 | USER 1001 15 | 16 | # TODO: make slimmer image with two stages, or copy build artifacts from apache/age to edgeflare/postgresql -------------------------------------------------------------------------------- /docs/container-images/pgvector.Containerfile: -------------------------------------------------------------------------------- 1 | ARG POSTGRES_VERSION=16.4.0 2 | 3 | FROM docker.io/bitnami/postgresql:$POSTGRES_VERSION AS builder 4 | 5 | ARG PGVECTOR_BRANCH=v0.7.4 6 | 7 | USER root 8 | RUN apt update && apt install -y build-essential git 9 | 10 | WORKDIR /tmp/pgvector 11 | RUN git clone --branch $PGVECTOR_BRANCH https://github.com/pgvector/pgvector.git /tmp/pgvector 12 | RUN make && make install 13 | 14 | # runner image 15 | FROM docker.io/edgeflare/postgresql:$POSTGRES_VERSION-wal-g-v3.0.3 16 | 17 | COPY --from=builder /tmp/pgvector/vector.so /opt/bitnami/postgresql/lib/ 18 | COPY --from=builder /tmp/pgvector/vector.control /opt/bitnami/postgresql/share/extension/ 19 | COPY --from=builder /tmp/pgvector/sql/*.sql /opt/bitnami/postgresql/share/extension/ 20 | 21 | ## alternative using pgvector image 22 | # FROM docker.io/pgvector/pgvector:0.7.4-pg16 AS builder 23 | # FROM docker.io/bitnami/postgresql:$POSTGRES_VERSION 24 | 25 | # COPY --from=builder /usr/lib/postgresql/16/lib/vector.so /opt/bitnami/postgresql/lib/ 26 | # COPY --from=builder /usr/share/postgresql/16/extension/vector* /opt/bitnami/postgresql/share/extension/ 27 | -------------------------------------------------------------------------------- /docs/container-images/timescaledb.Containerfile: -------------------------------------------------------------------------------- 1 | ### extension does not work 2 | 3 | 4 | # # Stage 1: Build TimescaleDB Tools 5 | # ARG POSTGRES_VERSION=16.4.0 6 | # ARG GOLANG_VERSION=1.23.1 7 | 8 | # FROM golang:${GOLANG_VERSION} AS tools 9 | # COPY go.mod go.sum ./ 10 | # RUN apt-get update && apt-get install -y git gcc 11 | 12 | # # Install dependencies 13 | # RUN apt-get update && apt-get install -y git gcc musl-dev 14 | # RUN go install github.com/timescale/timescaledb-tune/cmd/timescaledb-tune@latest && \ 15 | # go install github.com/timescale/timescaledb-parallel-copy/cmd/timescaledb-parallel-copy@latest 16 | 17 | # # Stage 2: Install TimescaleDB 18 | # FROM docker.io/edgeflare/postgresql:${POSTGRES_VERSION}-wal-g-v3.0.3 AS timescaledb-builder 19 | 20 | # # Create the missing directory 21 | # USER root 22 | # RUN mkdir -p /var/lib/apt/lists/partial 23 | 24 | # # Install dependencies 25 | # RUN apt-get update && apt-get install -y build-essential git gcc musl-dev ca-certificates postgresql-server-dev-all postgresql-plpython3 26 | 27 | # COPY --from=tools /go/bin/* /usr/local/bin/ 28 | 29 | # RUN set -ex \ 30 | # && apt-get update \ 31 | # && apt-get install -y postgresql-common \ 32 | # && sh /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y \ 33 | # && apt-get install -y postgresql-16-timescaledb \ 34 | # && apt-get clean \ 35 | # && rm -rf /var/lib/apt/lists/* 36 | 37 | # # Stage 3: Final Image with TimescaleDB 38 | # FROM docker.io/edgeflare/postgresql:${POSTGRES_VERSION}-wal-g-v3.0.3 39 | 40 | # # Copy the installed TimescaleDB files and configurations 41 | # COPY --from=timescaledb-builder /usr/lib/postgresql/16/lib/timescaledb-*.so /usr/local/lib/postgresql/ 42 | # COPY --from=timescaledb-builder /usr/share/postgresql/16/extension/timescaledb--*.sql /usr/local/share/postgresql/extension/ 43 | # COPY --from=timescaledb-builder /usr/share/postgresql/16/extension/timescaledb.control /usr/local/share/postgresql/extension/ 44 | 45 | # # Expose the PostgreSQL port 46 | # EXPOSE 5432 47 | 48 | # USER 1001 49 | 50 | # # Entrypoint and command 51 | # ENTRYPOINT ["/opt/bitnami/scripts/postgresql/entrypoint.sh"] 52 | # CMD ["/opt/bitnami/scripts/postgresql/run.sh"] 53 | -------------------------------------------------------------------------------- /docs/container-images/wal-g.Containerfile: -------------------------------------------------------------------------------- 1 | ARG POSTGRES_VERSION=16.4.0 2 | ARG GOLANG_VERSION=1.23.1 3 | 4 | # build wal-g binary 5 | FROM docker.io/library/golang:${GOLANG_VERSION}-bullseye as build-wal-g 6 | 7 | ARG WALG_VERSION=v3.0.3 8 | 9 | RUN apt update && apt install -y git build-essential 10 | 11 | WORKDIR /workspace 12 | RUN git clone https://github.com/wal-g/wal-g.git /workspace/wal-g 13 | RUN cd /workspace/wal-g && git checkout ${WALG_VERSION} && go mod tidy && go build -o wal-g ./main/pg/... 14 | 15 | # runner image 16 | FROM docker.io/bitnami/postgresql:${POSTGRES_VERSION} 17 | 18 | USER root 19 | COPY --from=build-wal-g /workspace/wal-g/wal-g /usr/local/bin/wal-g 20 | RUN chmod +x /usr/local/bin/wal-g 21 | 22 | USER 1001 23 | ## Optionally, create a directory for wal-g config 24 | # RUN mkdir -p /etc/wal-g 25 | ## add a default wal-g config file here if needed 26 | # COPY wal-g.json /etc/wal-g/wal-g.json 27 | ## Set environment variables for wal-g 28 | #ENV WALG_CONFIG_FILE="/etc/wal-g/wal-g.json" 29 | 30 | # Entrypoint for the container 31 | ENTRYPOINT [ "/opt/bitnami/scripts/postgresql/entrypoint.sh" ] 32 | CMD [ "/opt/bitnami/scripts/postgresql/run.sh" ] 33 | -------------------------------------------------------------------------------- /docs/container-images/wal2json.Containerfile: -------------------------------------------------------------------------------- 1 | ARG POSTGRES_VERSION=16.4.0 2 | 3 | # Use the specified PostgreSQL image as the base image for the builder stage 4 | FROM docker.io/bitnami/postgresql:${POSTGRES_VERSION} AS builder 5 | 6 | ARG WAL2JSON_BRANCH=wal2json_2_6 7 | 8 | USER root 9 | RUN apt-get update && apt-get install -y gcc git make && rm -rf /var/lib/apt/lists/* 10 | RUN git clone https://github.com/eulerto/wal2json.git --branch ${WAL2JSON_BRANCH} 11 | RUN cd wal2json && USE_PGXS=1 make && USE_PGXS=1 make install 12 | 13 | # runner image 14 | FROM docker.io/edgeflare/postgresql:$POSTGRES_VERSION-wal-g-v3.0.3 15 | COPY --from=builder /opt/bitnami/postgresql/lib/wal2json.so /opt/bitnami/postgresql/lib/ 16 | USER 1001 17 | 18 | # set wal_level=logical in postgresql.conf 19 | ENV POSTGRESQL_WAL_LEVEL=logical 20 | -------------------------------------------------------------------------------- /docs/img/pgo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/edgeflare/pgo/2bf5b5d2f7dab9c545cd45387ab1f2115623dcee/docs/img/pgo.png -------------------------------------------------------------------------------- /docs/nats.conf: -------------------------------------------------------------------------------- 1 | # General settings 2 | host: 0.0.0.0 3 | port: 4222 4 | server_name: nats-0 5 | 6 | jetstream { 7 | # storage location, limits and encryption 8 | store_dir: /data/jetstream 9 | max_memory_store: 2G 10 | max_file_store: 8G 11 | } 12 | 13 | # https://docs.nats.io/running-a-nats-service/configuration/websocket/websocket_conf 14 | websocket { 15 | port: 8080 16 | no_tls: true 17 | } 18 | 19 | mqtt { 20 | port: 1883 21 | 22 | # tls { 23 | # cert_file: "/path/to/cert.pem" 24 | # key_file: "/path/to/key.pem" 25 | # } 26 | } -------------------------------------------------------------------------------- /docs/postgres-cdc.md: -------------------------------------------------------------------------------- 1 | # Stream Postgres changes to NATS, MQTT, Kafka, Clickhouse, etc 2 | 3 | 1. Start Postgres, NATS, Kafka, MQTT broker and pgo pipeline as containers 4 | 5 | ```sh 6 | git clone git@github.com:edgeflare/pgo.git 7 | 8 | make image 9 | 10 | # Set KAFKA_CFG_ADVERTISED_LISTENERS env var in docs/docker-compose.yaml to host IP for local access, 11 | # as opposed to from within container network. adjust Kafka brokers IP in docs/pipeline-example.docker.yaml 12 | make up # docker compose up 13 | ``` 14 | 15 | 16 | optionally try pgo locally 17 | 18 | ```sh 19 | go install github.com/edgeflare/pgo@latest # or make build 20 | pgo pipeline --config pkg/config/example.config.yaml 21 | ``` 22 | 23 | 2. Postgres 24 | - As a source: Create a test table, eg `users` in source postgres database 25 | 26 | ```sh 27 | PGUSER=postgres PGPASSWORD=secret PGHOST=localhost PGDATABASE=testdb psql 28 | ``` 29 | 30 | ```sql 31 | CREATE TABLE public.users ( 32 | id INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, 33 | name TEXT 34 | ); 35 | 36 | ALTER TABLE public.users REPLICA IDENTITY FULL; -- instructs postgres to stream full OLD ROW data 37 | ``` 38 | 39 | - As a sink 40 | 41 | ```sh 42 | PGUSER=postgres PGPASSWORD=secret PGHOST=localhost PGDATABASE=testdb PGPORT=5431 psql 43 | ``` 44 | 45 | - Create the same users table in sink database for mirroring. altering replica identity may not be needed in sink 46 | 47 | - Create a second table in sink database which stores **transformed** data 48 | 49 | ```sql 50 | CREATE SCHEMA IF NOT EXISTS another_schema; 51 | 52 | CREATE TABLE IF NOT EXISTS another_schema.transformed_users ( 53 | uuid UUID DEFAULT gen_random_uuid(), -- because we're extracting only `name` field 54 | -- new_id INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, -- to handle UPDATE operations, primary key column type must match in source and sink 55 | new_name TEXT 56 | ); 57 | ``` 58 | 59 | pgo caches the table schemas for simpler parsing of CDC events (rows). To update pgo cache with newly created tables, 60 | either `docker restart docs_pgo_1` or `NOTIFY` it to reload cache by executing on database 61 | 62 | ```sql 63 | NOTIFY pgo, 'reload schema'; 64 | ``` 65 | 66 | 4. Subscribe 67 | 68 | - MQTT: `/any/prefix/schemaName/tableName/operation` topic (testing with mosquitto client) 69 | 70 | ```sh 71 | mosquitto_sub -t pgo/public/users/c # operation: c=create, u=update, d=delete, r=read 72 | ``` 73 | 74 | - Kafka: topic convention is `[prefix].[schema_name].[table_name].[operation]`. use any kafka client eg [`kaf`](https://github.com/birdayz/kaf) 75 | 76 | ```sh 77 | kaf consume pgo.public.users.c --follow # consume messages until program execution 78 | ``` 79 | 80 | - NATS: 81 | 82 | ```sh 83 | nats sub -s nats://localhost:4222 'pgo.public.users.>' # wildcard. includes all nested parts 84 | # nats sub -s nats://localhost:4222 'pgo.public.users.c' # specific 85 | ``` 86 | 87 | 5. `INSERT` (or update etc) into users table 88 | 89 | ```sql 90 | INSERT INTO users (name) VALUES ('alice'); 91 | INSERT INTO users (name) VALUES ('bob'); 92 | ``` 93 | 94 | And notice NATS, MQTT, Kafka, postgres-sink, or debug peer's respective subscriber receiving the message. 95 | It's not Postgres only source. Other peers too can be sources (not all peers fully functional yet). 96 | 97 | Clean up 98 | 99 | ```sh 100 | make down 101 | ``` -------------------------------------------------------------------------------- /docs/rag.md: -------------------------------------------------------------------------------- 1 | # Retrieval-Augmented Generation (RAG) on PostgreSQL with PGVector 2 | 3 | Retrieval-Augmented Generation (RAG) is a technique that combines the strengths of large language models (LLMs) with up-to-date, private, internal information to provide more accurate, contextually relevant responses. It works by: 4 | 5 | 1. Retrieving relevant information from a knowledge base in response to a query. 6 | 2. Augmenting the original query with this retrieved information (context). 7 | 3. Using a language model to generate a response based on both the query and the retrieved context. 8 | 9 | Retrieval-Augmented Generation (RAG) 10 | 11 | ## Key Concepts 12 | 13 | 1. **Embeddings**: Vector representations of texts or other data. 14 | 2. **Vector Database**: A database optimized for storing and querying vector embeddings (e.g., PostgreSQL with pgvector extension). 15 | 3. **(Semantic) Similarity Search**: Finding the most similar vectors to a given query vector. 16 | 4. **RAG (Retrieval-Augmented Generation)**: A technique that enhances language models by retrieving relevant information from a knowledge base. 17 | 18 | ## PGVector Basics 19 | 20 | PGVector is a PostgreSQL extension that adds support for vector operations and similarity search, enabling efficient storage and querying of embeddings. 21 | 22 | ### Data Type 23 | - `vector(n)`: Represents an n-dimensional vector. 24 | 25 | ### Similarity Functions 26 | - `<->`: Euclidean distance 27 | - `<#>`: Negative inner product 28 | - `<=>`: Cosine distance 29 | 30 | ### Creating a Table with Vector Column 31 | 32 | ```sql 33 | CREATE TABLE IF NOT EXISTS embeddings ( 34 | id BIGINT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, 35 | content TEXT, 36 | embedding vector(1536) 37 | ); 38 | ``` 39 | 40 | ### Indexing 41 | 42 | ```sql 43 | -- IVFFlat index (faster build, larger index) 44 | CREATE INDEX ON documents USING ivfflat (embedding vector_l2_ops) WITH (lists = 100); 45 | -- HNSW index (slower build, faster search) 46 | CREATE INDEX ON documents USING hnsw (embedding vector_cosine_ops) WITH (m = 16, ef_construction = 64); 47 | ``` 48 | 49 | ## RAG Implementation Steps 50 | 51 | 1. **Prepare Knowledge Base**: 52 | - Collect and preprocess documents 53 | - Generate embeddings for documents 54 | - Store documents and embeddings in the vector database 55 | 56 | 2. **Query Processing**: 57 | - Generate embedding for the user query 58 | - Perform similarity search to retrieve relevant documents 59 | 60 | 3. **Context Augmentation**: 61 | - Combine retrieved documents with the original query 62 | 63 | 4. **Generation**: 64 | - Feed the augmented context to the language model 65 | - Generate the response 66 | 67 | ## Best Practices 68 | 69 | 1. Choose appropriate embedding dimensions (usually 768, 1536, 3072 for modern models). 70 | 2. Experiment with different indexing methods for optimal performance. 71 | 3. Use batching for efficient embedding generation and database operations. 72 | 4. Implement caching mechanisms to reduce redundant computations. 73 | 5. Regularly update and maintain your knowledge base for accurate retrievals. 74 | 75 | ## Useful PGVector Functions 76 | 77 | - `vector_dims(vector)`: Returns the dimension of a vector 78 | - `vector_norm(vector)`: Calculates the Euclidean norm of a vector 79 | - `vector_add(vector, vector)`: Adds two vectors 80 | - `vector_subtract(vector, vector)`: Subtracts one vector from another 81 | 82 | See implementation [examples/rag/main.go](../examples/rag/main.go). 83 | -------------------------------------------------------------------------------- /docs/rag.mmd: -------------------------------------------------------------------------------- 1 | graph TD 2 | subgraph Knowledge_Base 3 | D[Document_Collection] 4 | C[Document_Chunks] 5 | D --> |1a. Chunk & Process| C 6 | end 7 | 8 | DB[Vector_Database] 9 | Knowledge_Base -->|1b. Generate Embeddings| DB 10 | 11 | Q[User_Query] 12 | E[Query_Embedding] 13 | Q -->|2a. Generate_Embedding| E 14 | E -->|2b. Vector Similarity Search| DB 15 | 16 | CTX[Retrieved_Context] 17 | DB -->|3 Retrieve Relevant Chunks| CTX 18 | 19 | P[Augmented_Prompt] 20 | Q -->|"4a. Combine"| P 21 | CTX -->|"4b. Combine"| P 22 | 23 | LLM[Language_Model] 24 | R[Final_Response] 25 | P -->|5 Send to LLM| LLM 26 | LLM -->|6 Generate Response| R 27 | -------------------------------------------------------------------------------- /examples/custom-middleware/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net/http" 7 | 8 | "github.com/edgeflare/pgo/pkg/httputil" 9 | ) 10 | 11 | func main() { 12 | r := httputil.NewRouter() 13 | 14 | // custom logger middleware 15 | r.Use(customLogger) 16 | 17 | // Group with prefix "/api/v1" 18 | v1 := r.Group("/api/v1") 19 | 20 | // Handle routes in the "/api" group 21 | v1.Handle("GET /users/{user}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 22 | w.Write([]byte(fmt.Sprintf("User endpoint: %s", r.PathValue("user")))) 23 | })) 24 | v1.Handle("POST /products/{product}", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 25 | w.Write([]byte(fmt.Sprintf("Product endpoint: %s", r.PathValue("product")))) 26 | })) 27 | 28 | log.Fatal(r.ListenAndServe(":8080")) 29 | } 30 | 31 | func customLogger(next http.Handler) http.Handler { 32 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 33 | log.Println(r.Method, r.URL.Path) 34 | next.ServeHTTP(w, r) 35 | }) 36 | } 37 | -------------------------------------------------------------------------------- /examples/file-server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "embed" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "net/http" 9 | 10 | mw "github.com/edgeflare/pgo/pkg/httputil/middleware" 11 | ) 12 | 13 | // Embed the static directory 14 | // 15 | //go:embed dist/* 16 | var embeddedFS embed.FS 17 | 18 | var ( 19 | port = flag.Int("port", 8080, "port to listen on") 20 | directory = flag.String("dir", "dist", "directory to serve files from") 21 | spaFallback = flag.Bool("spa", false, "fallback to index.html for not-found files") 22 | useEmbedded = flag.Bool("embed", false, "use embedded static files") 23 | ) 24 | 25 | func main() { 26 | flag.Parse() 27 | 28 | mux := http.NewServeMux() 29 | 30 | var fs *embed.FS 31 | if *useEmbedded { 32 | fs = &embeddedFS 33 | } 34 | 35 | // other mux handlers 36 | // 37 | // static handler should be the last handler 38 | mux.Handle("GET /", mw.Static(*directory, *spaFallback, fs)) 39 | 40 | // // or mount on a `/different/path` other than root `/` 41 | // mux.Handle("GET /static/", http.StripPrefix("/static", mw.Static(*directory, *spaFallback, fs))) 42 | 43 | // Start the server 44 | log.Printf("starting server on :%d", *port) 45 | log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), mux)) 46 | } 47 | 48 | // go run ./examples/file-server/... -port 8081 -dir dist -spa -embed 49 | // go run ./examples/file-server/... -port 8081 -dir examples/file-server/dist -spa 50 | -------------------------------------------------------------------------------- /examples/handlers/health.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/edgeflare/pgo/pkg/httputil" 7 | ) 8 | 9 | // HealthHandler returns the request ID as a plain text response with a 200 OK status code. 10 | func HealthHandler(w http.ResponseWriter, r *http.Request) { 11 | requestID := r.Context().Value(httputil.RequestIDCtxKey).(string) 12 | if requestID == "" { 13 | http.Error(w, "Request ID not found", http.StatusInternalServerError) 14 | return 15 | } 16 | 17 | httputil.Text(w, http.StatusOK, requestID) 18 | } 19 | -------------------------------------------------------------------------------- /examples/handlers/oidc.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/edgeflare/pgo/pkg/httputil" 7 | // "github.com/zitadel/oidc/v3/pkg/oidc" 8 | ) 9 | 10 | type AuthzResponse struct { 11 | Allowed bool `json:"allowed"` 12 | } 13 | 14 | // GetMyAccountHandler retrieves the user information from the OIDC context and responds with the user details. 15 | // 16 | // If the user is not found in the context or an error occurs, an HTTP 401 Unauthorized error is returned. 17 | func GetMyAccountHandler(w http.ResponseWriter, r *http.Request) { 18 | var user map[string]any 19 | if user, ok := httputil.OIDCUser(r); !ok || user == nil { 20 | http.Error(w, "User not found in context", http.StatusUnauthorized) 21 | return 22 | } 23 | httputil.JSON(w, http.StatusOK, user) 24 | } 25 | 26 | // GetSimpleAuthzHandler performs an authorization check based on a requested /endpoint/{claim}/{value} path 27 | // eg `GET /endpoint/editor/true` will check if the user has the claim `editor` with the value `true` 28 | // The response is an `AuthzResponse` object indicating whether the user is authorized (Allowed: true) or not (Allowed: false) 29 | func GetSimpleAuthzHandler(w http.ResponseWriter, r *http.Request) { 30 | user, ok := httputil.OIDCUser(r) 31 | if !ok { 32 | httputil.Error(w, http.StatusUnauthorized, "Unauthorized") 33 | return 34 | } 35 | 36 | requestedClaim := r.PathValue("claim") 37 | requestedValue := r.PathValue("value") 38 | if value, ok := user[requestedClaim].(string); !ok || value != requestedValue { 39 | httputil.JSON(w, http.StatusOK, AuthzResponse{Allowed: false}) 40 | return 41 | } 42 | 43 | httputil.JSON(w, http.StatusOK, AuthzResponse{Allowed: true}) 44 | } 45 | -------------------------------------------------------------------------------- /examples/handlers/pgrole.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/edgeflare/pgo/pkg/httputil" 7 | "github.com/jackc/pgx/v5/pgxpool" 8 | ) 9 | 10 | func GetMyPgRoleHandler() http.HandlerFunc { 11 | return func(w http.ResponseWriter, r *http.Request) { 12 | // retrive the role from the request context, set by the middleware 13 | ctxRole := r.Context().Value(httputil.OIDCRoleClaimCtxKey) 14 | 15 | // retrieve the connection from request context 16 | conn, ok := r.Context().Value(httputil.PgConnCtxKey).(*pgxpool.Conn) 17 | if !ok || conn == nil { 18 | http.Error(w, "Failed to get connection from context", http.StatusInternalServerError) 19 | return 20 | } 21 | defer conn.Release() 22 | 23 | // query the current role using the connection 24 | var queryRole string 25 | err := conn.Conn().QueryRow(r.Context(), "SELECT current_role").Scan(&queryRole) 26 | if err != nil { 27 | http.Error(w, "Failed to query current role", http.StatusInternalServerError) 28 | return 29 | } 30 | 31 | // construct both roles as a json response 32 | roles := map[string]string{ 33 | "ctx_role": ctxRole.(string), 34 | "query_role": queryRole, 35 | } 36 | httputil.JSON(w, http.StatusOK, roles) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /examples/logical-replication-cdc/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "cmp" 5 | "context" 6 | "fmt" 7 | "log" 8 | "os" 9 | "os/signal" 10 | "syscall" 11 | "time" 12 | 13 | "github.com/edgeflare/pgo/pkg/pglogrepl" 14 | "github.com/edgeflare/pgo/pkg/pipeline/cdc" 15 | "github.com/jackc/pgx/v5/pgconn" 16 | ) 17 | 18 | func main() { 19 | ctx, cancel := context.WithCancel(context.Background()) 20 | defer cancel() 21 | 22 | // handle interrupt signals (e.g., Ctrl+C) 23 | sigChan := make(chan os.Signal, 1) 24 | signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) 25 | go func() { 26 | <-sigChan 27 | cancel() 28 | }() 29 | 30 | conn, err := pgconn.Connect(context.Background(), cmp.Or(os.Getenv("PGO_PGLOGREPL_CONN_STRING"), 31 | "postgres://postgres:secret@localhost:5432/testdb?replication=database")) 32 | if err != nil { 33 | log.Fatal("connect failed:", err) 34 | } 35 | defer conn.Close(context.Background()) 36 | 37 | // configuration for logical replication. all options optional 38 | // cfg := pglogrepl.DefaultConfig() 39 | cfg := &pglogrepl.Config{ 40 | Publication: "pgo_pub", 41 | Slot: "pgo_slot", 42 | Plugin: "pgoutput", 43 | // tables to watch/replicate 44 | Tables: []string{ 45 | // "table_name", // (usually public) default_schema.table_name 46 | // "schema_name.example_table", // specific schema.table 47 | // "schema_name.*", // all tables in specified schema 48 | "*", 49 | // or "*.*" for all tables in all non-system schemas 50 | }, 51 | Ops: []pglogrepl.Op{ 52 | pglogrepl.OpInsert, 53 | pglogrepl.OpUpdate, 54 | pglogrepl.OpDelete, 55 | }, 56 | StandbyUpdateInterval: 10 * time.Second, 57 | BufferSize: 1000, // go channel size 58 | // not functional yet. to capture old row data for UPDATE operations, manually execute 59 | // ALTER TABLE schema_name.table_name REPLICA IDENTITY FULL; 60 | // ReplicaIdentity: map[string]pglogrepl.ReplicaIdentity{"table_name": pglogrepl.ReplicaIdentityFull}, 61 | } 62 | 63 | // start streaming changes 64 | events, err := pglogrepl.Stream(ctx, conn, cfg) 65 | if err != nil { 66 | log.Fatalf("Failed to start streaming: %v", err) 67 | } 68 | 69 | // process incoming CDC events 70 | for event := range events { 71 | switch event.Payload.Op { 72 | case cdc.OpCreate: 73 | fmt.Printf("Insert on %s: %v\n", event.Payload.Source.Table, event.Payload.After) 74 | case cdc.OpUpdate: 75 | fmt.Printf("Update on %s: Before=%v After=%v\n", event.Payload.Source.Table, event.Payload.Before, event.Payload.After) 76 | case cdc.OpDelete: 77 | fmt.Printf("Delete on %s: Before%v\n", event.Payload.Source.Table, event.Payload.Before) 78 | } 79 | } 80 | 81 | // check if the context was canceled 82 | if ctx.Err() != nil { 83 | log.Println("Streaming stopped:", ctx.Err()) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /examples/pipeline.chain.yaml: -------------------------------------------------------------------------------- 1 | peers: 2 | - name: example-pg-source 3 | connector: postgres 4 | config: 5 | connString: "host=localhost port=5432 user=postgres password=secret dbname=testdb replication=database" 6 | replication: 7 | tables: ["*"] 8 | 9 | - name: nats-default 10 | connector: nats 11 | config: 12 | servers: ["nats://localhost:4222"] 13 | subjectPrefix: "pgo" 14 | 15 | - name: debug # logs CDC events to stdout 16 | connector: debug 17 | 18 | - name: mqtt-default 19 | connector: mqtt 20 | config: 21 | servers: ["tcp://localhost:1883"] 22 | 23 | # Postgres CDC is streamed to NATS from which MQTT the CDC 24 | pipelines: 25 | - name: pg-nats 26 | sources: 27 | - name: example-pg-source 28 | sinks: 29 | - name: nats-default 30 | - name: debug 31 | 32 | - name: nats-mqtt 33 | sources: 34 | - name: nats-default 35 | sinks: 36 | - name: mqtt-default 37 | # - name: debug # apparently it gets in a strange loop; keeps printing empty CDC 38 | -------------------------------------------------------------------------------- /examples/postgrest-like-oidc-authz/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "cmp" 5 | "context" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "net/http" 10 | "os" 11 | "os/signal" 12 | "syscall" 13 | "time" 14 | 15 | "github.com/edgeflare/pgo/pkg/httputil" 16 | mw "github.com/edgeflare/pgo/pkg/httputil/middleware" 17 | "github.com/edgeflare/pgo/pkg/pgx" 18 | ) 19 | 20 | func main() { 21 | port := flag.Int("port", 8080, "port to run the server on") 22 | flag.Parse() 23 | 24 | r := httputil.NewRouter() 25 | 26 | // Group with prefix "/api/v1" 27 | apiv1 := r.Group("/api/v1") 28 | 29 | // optional middleware with default options 30 | apiv1.Use(mw.RequestID, mw.LoggerWithOptions(nil), mw.CORSWithOptions(nil)) 31 | 32 | // OIDC middleware for authentication 33 | oidcConfig := mw.OIDCProviderConfig{ 34 | ClientID: os.Getenv("PGO_OIDC_CLIENT_ID"), 35 | ClientSecret: os.Getenv("PGO_OIDC_CLIENT_SECRET"), 36 | Issuer: os.Getenv("PGO_OIDC_ISSUER"), 37 | } 38 | apiv1.Use(mw.VerifyOIDCToken(oidcConfig)) 39 | 40 | pgxPoolMgr := pgx.NewPoolManager() 41 | err := pgxPoolMgr.Add(context.Background(), pgx.Pool{Name: "default", ConnString: os.Getenv("PGO_POSTGRES_CONN_STRING")}, true) 42 | if err != nil { 43 | log.Fatal(err) 44 | } 45 | 46 | pgxPool, err := pgxPoolMgr.Active() 47 | if err != nil { 48 | log.Fatal(err) 49 | } 50 | 51 | // Use Postgres middleware to attach a pgxpool.Conn to the request context for authorized users 52 | pgmw := mw.Postgres(pgxPool, mw.WithOIDCAuthz( 53 | oidcConfig, 54 | cmp.Or(os.Getenv("PGO_POSTGRES_OIDC_ROLE_CLAIM_KEY"), ".policy.pgrole")), 55 | ) 56 | apiv1.Use(pgmw) 57 | 58 | // Below `GET /api/v1/mypgrole` queries for, and responds with, 59 | // session_user, current_user using the pgxpool.Conn attached by the Postgres middleware 60 | apiv1.Handle("GET /mypgrole", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 61 | user, conn, err := httputil.ConnWithRole(r) 62 | if err != nil { 63 | httputil.Error(w, http.StatusInternalServerError, err.Error()) 64 | } 65 | defer conn.Release() 66 | 67 | var session_user, current_user string 68 | pgErr := conn.QueryRow(r.Context(), "SELECT session_user").Scan(&session_user) 69 | if pgErr != nil { 70 | httputil.Error(w, http.StatusInternalServerError, pgErr.Error()) 71 | } 72 | pgErr = conn.QueryRow(r.Context(), "SELECT current_user").Scan(¤t_user) 73 | if pgErr != nil { 74 | httputil.Error(w, http.StatusInternalServerError, pgErr.Error()) 75 | } 76 | 77 | role := map[string]string{ 78 | // role with which initial connection to database is established 79 | "session_user": session_user, 80 | // role with which application query is performed to process this particular request 81 | "current_user": current_user, 82 | "user_sub": user["sub"].(string), 83 | } 84 | 85 | httputil.JSON(w, http.StatusOK, role) 86 | })) 87 | 88 | // Run server in a goroutine 89 | go func() { 90 | if err := r.ListenAndServe(fmt.Sprintf(":%d", *port)); err != nil && err != http.ErrServerClosed { 91 | log.Fatalf("Server error: %v", err) 92 | } 93 | }() 94 | 95 | fmt.Printf("Server is running on port %d\n", *port) 96 | 97 | // Set up signal handling 98 | stop := make(chan os.Signal, 1) 99 | signal.Notify(stop, os.Interrupt, syscall.SIGTERM) 100 | 101 | // Wait for SIGINT or SIGTERM 102 | <-stop 103 | 104 | fmt.Println("Shutting down server...") 105 | 106 | // Create a deadline for the shutdown 107 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 108 | defer cancel() 109 | 110 | // Attempt graceful shutdown 111 | if err := r.Shutdown(ctx); err != nil { 112 | fmt.Printf("server forced to shutdown: %s", err) 113 | } 114 | fmt.Println("Server gracefully stopped") 115 | } 116 | -------------------------------------------------------------------------------- /examples/rag/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // See [docs/rag.md](../../docs/rag.md) for more information. 4 | 5 | import ( 6 | "context" 7 | "fmt" 8 | "log" 9 | 10 | "github.com/edgeflare/pgo/pkg/util" 11 | "github.com/edgeflare/pgo/pkg/x/rag" 12 | "github.com/jackc/pgx/v5" 13 | ) 14 | 15 | func main() { 16 | ctx := context.Background() 17 | 18 | conn, err := pgx.Connect(ctx, util.GetEnvOrDefault("PGO_POSTGRES_CONN_STRING", "postgres://postgres:secret@localhost:5432/postgres")) 19 | if err != nil { 20 | log.Fatalf("Failed to connect to database: %v", err) 21 | } 22 | defer conn.Close(ctx) 23 | 24 | // Create a new RAG client 25 | client, err := rag.NewClient(conn, rag.DefaultConfig()) 26 | client.Config.TableName = "lms.courses" 27 | // TODO: fix primary key data type from primary key column in contentSelectQuery 28 | 29 | if err != nil { 30 | log.Fatalf("Failed to create RAG client: %v", err) 31 | } 32 | 33 | err = client.CreateEmbedding(ctx, "SELECT id, CONCAT('title:', title, ', summary:', summary) AS content FROM lms.courses") 34 | // err = client.CreateEmbedding(ctx, "") // CreateEmbedding constructs content by concatenating colname:value of other columns for each row 35 | // err = client.CreateEmbedding(ctx) // Assumes the table has a column named `content` that contains the content for which embedding will be created 36 | if err != nil { 37 | log.Fatalf("Failed to create embeddings: %v", err) 38 | } 39 | 40 | fmt.Println("Embeddings have been successfully created.") 41 | 42 | // retrieval example 43 | input := "example input text" 44 | limit := 2 45 | 46 | results, err := client.Retrieve(ctx, input, limit) 47 | if err != nil { 48 | log.Fatalf("Failed to retrieve content: %v", err) 49 | } 50 | 51 | // Print the retrieved results 52 | for _, r := range results { 53 | fmt.Printf("ID: %v\nContent: %s\nEmbedding: %v\n", r.PK, r.Content, r.Embedding.Slice()[0]) 54 | } 55 | 56 | prompt := "count the courses. just give me the number." 57 | // this is requesting infromation from internal data 58 | // typically LLM models don't have access to your data 59 | // unless you exposed publicly for models to access and be trained on 60 | // give it a shot with Generate function. it will likely spit out gibberish ie hallucinate 61 | response, err := client.Generate(ctx, prompt) 62 | if err != nil { 63 | log.Fatalf("Failed to generate content: %v", err) 64 | } 65 | fmt.Println(string(response)) 66 | 67 | // Now augment the prompt with the retrieved data 68 | // number of retrieved rows (relevant contents) from embeddings table to augment the prompt with 69 | retrievalLimit := 3 70 | // this is the query to retrieve the relevant contents from the embeddings table 71 | // retrievalInput is optional. if not provided, the prompt is used as the input to retrieve the relevant contents from the embeddings table 72 | retrievalInput := "python, machine learning, data science" 73 | responseWithRetrieval, err := client.GenerateWithRetrieval(ctx, prompt, retrievalLimit, retrievalInput) 74 | if err != nil { 75 | log.Fatalf("Failed to generate content: %v", err) 76 | } 77 | fmt.Println(string(responseWithRetrieval)) 78 | } 79 | -------------------------------------------------------------------------------- /examples/webdav/README.md: -------------------------------------------------------------------------------- 1 | # Go WebDAV Server 2 | 3 | This is a simple Go WebDAV server implementation using the `golang.org/x/net/webdav` package. The `edgeflare/pgo/middleware` package is used to provide middleware for request ID, CORS, and logging. 4 | 5 | ## Features 6 | 7 | * Serves files from the `./webdav-dir` directory. 8 | * Supports basic WebDAV operations (see below). 9 | * Includes middleware for request ID generation, CORS handling, and logging. 10 | 11 | ## Common WebDAV Methods 12 | 13 | | Method | Description | Example | 14 | |---|---|---| 15 | | `PROPFIND` | Retrieves properties of a resource or collection. | `curl -X PROPFIND -H "Depth: 1" http://localhost:8080/webdav/` | 16 | | `GET` | Retrieves the content of a file. | `curl http://localhost:8080/webdav/filename.txt` | 17 | | `PUT` | Uploads or modifies a file. | `echo 'helloworld' > newfile.txt && curl -T newfile.txt http://localhost:8080/webdav/newfile.txt` | 18 | | `MKCOL` | Creates a new collection (directory). | `curl -X MKCOL http://localhost:8080/webdav/newdirectory/` | 19 | | `DELETE` | Deletes a file or collection. | `curl -X DELETE http://localhost:8080/webdav/file-to-delete.txt` | 20 | | `MOVE` | Moves or renames a file or collection. | `curl -X MOVE -H "Destination: http://localhost:8080/webdav/newname.txt" http://localhost:8080/webdav/oldname.txt` | 21 | | `COPY` | Copies a file or collection. | `curl -X COPY -H "Destination: http://localhost:8080/webdav/copy-of-file.txt" http://localhost:8080/webdav/original-file.txt` | 22 | | `LOCK` | Obtains a lock on a resource. | `curl -X LOCK -H "Timeout: Infinite" http://localhost:8080/webdav/file-to-lock.txt` | 23 | | `UNLOCK` | Releases a lock on a resource. | `curl -X UNLOCK -H "Lock-Token: " http://localhost:8080/webdav/file-to-unlock.txt` | 24 | 25 | **Note:** Replace placeholders like `` with actual values. 26 | 27 | ## Running the Server 28 | 29 | 1. Make sure you have Go installed. 30 | 2. Install the required dependencies: `go get golang.org/x/net/webdav github.com/edgeflare/pgo/middleware` 31 | 3. Create a directory named `webdav-dir` to store your files. 32 | 4. Run the server: `go run main.go` 33 | 5. The server will start listening on `http://localhost:8080`. 34 | 35 | ## Additional Notes 36 | 37 | * This server uses a simple in-memory lock system. For production use, consider a more persistent lock manager. 38 | * Refer to the `golang.org/x/net/webdav` documentation for more advanced configuration and features. 39 | -------------------------------------------------------------------------------- /examples/webdav/webdav.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | 7 | mw "github.com/edgeflare/pgo/pkg/httputil/middleware" 8 | "golang.org/x/net/webdav" 9 | ) 10 | 11 | func main() { 12 | webdavHandler := &webdav.Handler{ 13 | Prefix: "/webdav/", 14 | FileSystem: webdav.Dir("./webdav-dir"), 15 | LockSystem: webdav.NewMemLS(), 16 | Logger: func(r *http.Request, err error) { 17 | if err != nil { 18 | log.Printf("WebDAV error: %s", err) 19 | } 20 | }, 21 | } 22 | 23 | // Apply middlewares 24 | handler := mw.Add(webdavHandler, 25 | mw.RequestID, 26 | mw.CORSWithOptions(nil), 27 | mw.LoggerWithOptions(nil), 28 | ) 29 | 30 | mux := http.NewServeMux() 31 | mux.Handle("/webdav/", handler) 32 | 33 | // Start the server using the ServeMux 34 | log.Println("Starting WebDAV server on :8080") 35 | http.ListenAndServe(":8080", mux) 36 | } 37 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/edgeflare/pgo 2 | 3 | go 1.24.1 4 | 5 | require ( 6 | github.com/ClickHouse/clickhouse-go/v2 v2.33.0 7 | github.com/IBM/sarama v1.45.1 8 | github.com/apache/age/drivers/golang v0.0.0-20240927175111-c75d9e477e2f 9 | github.com/cenkalti/backoff/v4 v4.3.0 10 | github.com/coreos/go-oidc/v3 v3.13.0 11 | github.com/eclipse/paho.mqtt.golang v1.5.0 12 | github.com/google/uuid v1.6.0 13 | github.com/jackc/pglogrepl v0.0.0-20250315193731-29dcedd74728 14 | github.com/jackc/pgx/v5 v5.7.2 15 | github.com/mitchellh/mapstructure v1.5.0 16 | github.com/nats-io/nats.go v1.39.1 17 | github.com/pganalyze/pg_query_go/v5 v5.1.0 18 | github.com/pgvector/pgvector-go v0.3.0 19 | github.com/prometheus/client_golang v1.21.1 20 | github.com/spf13/cobra v1.9.1 21 | github.com/spf13/viper v1.20.0 22 | github.com/stretchr/testify v1.10.0 23 | github.com/xdg-go/scram v1.1.2 24 | go.uber.org/zap v1.27.0 25 | golang.org/x/net v0.37.0 26 | google.golang.org/grpc v1.71.0 27 | google.golang.org/protobuf v1.36.5 28 | ) 29 | 30 | require ( 31 | github.com/ClickHouse/ch-go v0.65.1 // indirect 32 | github.com/andybalholm/brotli v1.1.1 // indirect 33 | github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1 // indirect 34 | github.com/beorn7/perks v1.0.1 // indirect 35 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 36 | github.com/davecgh/go-spew v1.1.1 // indirect 37 | github.com/eapache/go-resiliency v1.7.0 // indirect 38 | github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect 39 | github.com/eapache/queue v1.1.0 // indirect 40 | github.com/fsnotify/fsnotify v1.8.0 // indirect 41 | github.com/go-faster/city v1.0.1 // indirect 42 | github.com/go-faster/errors v0.7.1 // indirect 43 | github.com/go-jose/go-jose/v4 v4.0.5 // indirect 44 | github.com/go-viper/mapstructure/v2 v2.2.1 // indirect 45 | github.com/golang/snappy v0.0.4 // indirect 46 | github.com/gorilla/websocket v1.5.3 // indirect 47 | github.com/hashicorp/errwrap v1.0.0 // indirect 48 | github.com/hashicorp/go-multierror v1.1.1 // indirect 49 | github.com/hashicorp/go-uuid v1.0.3 // indirect 50 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 51 | github.com/jackc/pgio v1.0.0 // indirect 52 | github.com/jackc/pgpassfile v1.0.0 // indirect 53 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect 54 | github.com/jackc/puddle/v2 v2.2.2 // indirect 55 | github.com/jcmturner/aescts/v2 v2.0.0 // indirect 56 | github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect 57 | github.com/jcmturner/gofork v1.7.6 // indirect 58 | github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect 59 | github.com/jcmturner/rpc/v2 v2.0.3 // indirect 60 | github.com/klauspost/compress v1.17.11 // indirect 61 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 62 | github.com/nats-io/nkeys v0.4.9 // indirect 63 | github.com/nats-io/nuid v1.0.1 // indirect 64 | github.com/paulmach/orb v0.11.1 // indirect 65 | github.com/pelletier/go-toml/v2 v2.2.3 // indirect 66 | github.com/pierrec/lz4/v4 v4.1.22 // indirect 67 | github.com/pkg/errors v0.9.1 // indirect 68 | github.com/pmezard/go-difflib v1.0.0 // indirect 69 | github.com/prometheus/client_model v0.6.1 // indirect 70 | github.com/prometheus/common v0.62.0 // indirect 71 | github.com/prometheus/procfs v0.15.1 // indirect 72 | github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect 73 | github.com/sagikazarmark/locafero v0.7.0 // indirect 74 | github.com/segmentio/asm v1.2.0 // indirect 75 | github.com/shopspring/decimal v1.4.0 // indirect 76 | github.com/sourcegraph/conc v0.3.0 // indirect 77 | github.com/spf13/afero v1.12.0 // indirect 78 | github.com/spf13/cast v1.7.1 // indirect 79 | github.com/spf13/pflag v1.0.6 // indirect 80 | github.com/subosito/gotenv v1.6.0 // indirect 81 | github.com/x448/float16 v0.8.4 // indirect 82 | github.com/xdg-go/pbkdf2 v1.0.0 // indirect 83 | github.com/xdg-go/stringprep v1.0.4 // indirect 84 | go.opentelemetry.io/otel v1.35.0 // indirect 85 | go.opentelemetry.io/otel/trace v1.35.0 // indirect 86 | go.uber.org/multierr v1.11.0 // indirect 87 | golang.org/x/crypto v0.36.0 // indirect 88 | golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 // indirect 89 | golang.org/x/oauth2 v0.28.0 // indirect 90 | golang.org/x/sync v0.12.0 // indirect 91 | golang.org/x/sys v0.31.0 // indirect 92 | golang.org/x/text v0.23.0 // indirect 93 | google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect 94 | gopkg.in/yaml.v3 v3.0.1 // indirect 95 | ) 96 | -------------------------------------------------------------------------------- /internal/testutil/k8s-svc.json: -------------------------------------------------------------------------------- 1 | { 2 | "kind": "Service", 3 | "apiVersion": "v1", 4 | "metadata": { 5 | "name": "nginx", 6 | "creationTimestamp": null, 7 | "labels": { 8 | "app": "nginx" 9 | } 10 | }, 11 | "spec": { 12 | "ports": [ 13 | { 14 | "name": "http", 15 | "protocol": "TCP", 16 | "port": 8080, 17 | "targetPort": 8080 18 | }, 19 | { 20 | "name": "https", 21 | "protocol": "TCP", 22 | "port": 8443, 23 | "targetPort": 8443 24 | } 25 | ], 26 | "selector": { 27 | "app": "nginx" 28 | }, 29 | "type": "ClusterIP" 30 | }, 31 | "status": { 32 | "loadBalancer": {} 33 | } 34 | } -------------------------------------------------------------------------------- /internal/testutil/pgtest/pgtest.go: -------------------------------------------------------------------------------- 1 | package pgtest 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/jackc/pgx/v5" 10 | "github.com/jackc/pgx/v5/pgconn" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | // TestDB encapsulates test database functionality 15 | type TestDB struct { 16 | Config *pgx.ConnConfig 17 | } 18 | 19 | // Connect creates a new database connection for testing 20 | func Connect(ctx context.Context, t testing.TB) *pgx.Conn { 21 | config, err := pgx.ParseConfig(os.Getenv("TEST_DATABASE")) 22 | require.NoError(t, err) 23 | 24 | config.OnNotice = func(_ *pgconn.PgConn, n *pgconn.Notice) { 25 | t.Logf("PostgreSQL %s: %s", n.Severity, n.Message) 26 | } 27 | 28 | conn, err := pgx.ConnectConfig(ctx, config) 29 | require.NoError(t, err) 30 | 31 | t.Cleanup(func() { 32 | Close(t, conn) 33 | }) 34 | 35 | return conn 36 | } 37 | 38 | // Close safely closes a database connection 39 | func Close(t testing.TB, conn *pgx.Conn) { 40 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 41 | defer cancel() 42 | require.NoError(t, conn.Close(ctx)) 43 | } 44 | 45 | // WithConn provides a database connection to a test function and handles cleanup 46 | func WithConn(t testing.TB, fn func(*pgx.Conn)) { 47 | ctx := context.Background() 48 | conn := Connect(ctx, t) 49 | defer Close(t, conn) 50 | fn(conn) 51 | } 52 | 53 | // ParseConfig returns a test connection config with logging 54 | func ParseConfig(t testing.TB) *pgx.ConnConfig { 55 | config, err := pgx.ParseConfig(os.Getenv("TEST_DATABASE")) 56 | require.NoError(t, err) 57 | 58 | config.OnNotice = func(_ *pgconn.PgConn, n *pgconn.Notice) { 59 | t.Logf("PostgreSQL %s: %s", n.Severity, n.Message) 60 | } 61 | 62 | return config 63 | } 64 | -------------------------------------------------------------------------------- /internal/testutil/testdata.go: -------------------------------------------------------------------------------- 1 | package testutil 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "path/filepath" 7 | "runtime" 8 | ) 9 | 10 | // LoadJSON reads and unmarshals a JSON file. If target is provided, it attempts to unmarshal the JSON into the target struct. 11 | func LoadJSON(filename string, target ...any) (map[string]any, error) { 12 | var result map[string]any 13 | 14 | _, currentFile, _, _ := runtime.Caller(0) 15 | dir := filepath.Dir(currentFile) 16 | 17 | data, err := os.ReadFile(filepath.Join(dir, filename)) 18 | if err != nil { 19 | return nil, err 20 | } 21 | 22 | err = json.Unmarshal(data, &result) 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | if len(target) > 0 && target[0] != nil { 28 | err = json.Unmarshal(data, target[0]) 29 | if err != nil { 30 | return nil, err 31 | } 32 | } 33 | 34 | return result, nil 35 | } 36 | -------------------------------------------------------------------------------- /k8s.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | app.kubernetes.io/name: pgo 6 | name: pgo 7 | spec: 8 | selector: 9 | matchLabels: 10 | app.kubernetes.io/name: pgo 11 | template: 12 | metadata: 13 | labels: 14 | app.kubernetes.io/name: pgo 15 | spec: 16 | containers: 17 | - args: 18 | - rest 19 | - --config 20 | - /rest/config.yaml 21 | env: 22 | - name: PGO_POSTGRES_ANON_ROLE 23 | value: anon 24 | image: ghcr.io/edgeflare/pgo 25 | imagePullPolicy: Always 26 | name: pgo-rest 27 | ports: 28 | - containerPort: 8080 29 | protocol: TCP 30 | resources: {} 31 | volumeMounts: 32 | - mountPath: /rest 33 | name: rest-config 34 | # - image: ghcr.io/edgeflare/pgo 35 | # name: pgo-pipeline 36 | # resources: {} 37 | volumes: 38 | - name: rest-config 39 | secret: 40 | defaultMode: 420 41 | secretName: pgo-rest 42 | --- 43 | apiVersion: v1 44 | kind: Service 45 | metadata: 46 | labels: 47 | app.kubernetes.io/name: pgo 48 | name: pgo 49 | spec: 50 | ports: 51 | - name: "8080" 52 | port: 80 53 | protocol: TCP 54 | targetPort: 8080 55 | selector: 56 | app.kubernetes.io/name: pgo 57 | type: ClusterIP 58 | --- 59 | apiVersion: v1 60 | stringData: 61 | config.yaml: | 62 | rest: 63 | listenAddr: ":8080" 64 | pg: 65 | connString: "host=localhost port=5432 user=pgo password=pgopw dbname=testdb" 66 | oidc: 67 | issuer: https://iam.example.org 68 | clientID: example-client-id 69 | clientSecret: example-client-secret 70 | roleClaimKey: .policy.pgrole 71 | basicAuth: 72 | admin: adminpw 73 | user1: user1pw 74 | anonRole: anon 75 | kind: Secret 76 | metadata: 77 | name: pgo-rest 78 | --- 79 | apiVersion: gateway.networking.k8s.io/v1 80 | kind: HTTPRoute 81 | metadata: 82 | labels: 83 | app.kubernetes.io/name: pgo 84 | name: pgo 85 | spec: 86 | hostnames: 87 | - api.example.org 88 | - pgo.example.org 89 | parentRefs: 90 | - group: gateway.networking.k8s.io 91 | kind: Gateway 92 | name: envoy-default 93 | namespace: envoy-gateway-system 94 | sectionName: https 95 | - group: gateway.networking.k8s.io 96 | kind: Gateway 97 | name: envoy-default 98 | namespace: envoy-gateway-system 99 | sectionName: http 100 | - group: gateway.networking.k8s.io 101 | kind: Gateway 102 | name: istio-default 103 | namespace: istio-system 104 | sectionName: https 105 | - group: gateway.networking.k8s.io 106 | kind: Gateway 107 | name: istio-default 108 | namespace: istio-system 109 | sectionName: http 110 | rules: 111 | - backendRefs: 112 | - group: "" 113 | kind: Service 114 | name: pgo 115 | port: 80 116 | weight: 1 117 | matches: 118 | - path: 119 | type: PathPrefix 120 | value: / 121 | --- 122 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/edgeflare/pgo/cmd/pgo" 4 | 5 | func main() { 6 | pgo.Main() 7 | } 8 | -------------------------------------------------------------------------------- /pgo-config.yaml: -------------------------------------------------------------------------------- 1 | rest: 2 | listenAddr: ":8001" 3 | pg: 4 | connString: "host=db-postgresql port=5432 user=pgo password=pgopw dbname=main sslmode=disable" # container 5 | # connString: "host=localhost port=5432 user=pgo password=pgopw dbname=main sslmode=prefer" # local 6 | oidc: 7 | issuer: http://192.168.0.10:5556/dex 8 | skipTLSVerify: true # for testing only 9 | clientID: oauth2-proxy 10 | clientSecret: ZXhhbXBsZS1hcHAtc2VjcmV0 11 | roleClaimKey: .policy.pgrole 12 | basicAuth: {} 13 | anonRole: anon 14 | 15 | pipeline: 16 | peers: 17 | - name: db-postgresql-dex 18 | connector: postgres 19 | config: 20 | connString: "host=db-postgresql port=5432 user=postgres password=postgrespw dbname=dex sslmode=disable replication=database" 21 | replication: 22 | tables: ["public.refresh_token"] 23 | - name: db-postgresql-main 24 | connector: postgres 25 | config: 26 | connString: "host=db-postgresql port=5432 user=postgres password=postgrespw dbname=main sslmode=prefer" 27 | - name: debug # logs CDC events to stdout 28 | connector: debug 29 | pipelines: 30 | - name: sync-users-from-dex-to-main 31 | sources: 32 | - name: db-postgresql-dex 33 | sinks: 34 | - name: debug 35 | - name: db-postgresql-main 36 | transformations: 37 | - type: filter 38 | config: 39 | operations: ["c"] # c=create/insert, u=update, d=delete, r=read/select 40 | - type: extract 41 | config: 42 | fields: ["claims_user_id", "claims_username", "claims_email", "claims_preferred_username", "claims_groups", "connector_id"] 43 | - type: replace 44 | config: 45 | schemas: 46 | public: iam 47 | tables: 48 | refresh_token: users 49 | columns: 50 | claims_user_id: id 51 | claims_email: email 52 | claims_username: name 53 | claims_preferred_username: preferred_username 54 | claims_groups: groups 55 | --- 56 | -------------------------------------------------------------------------------- /pkg/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/edgeflare/pgo/pkg/pipeline" 9 | "github.com/spf13/viper" 10 | ) 11 | 12 | // Config holds application-wide configuration 13 | type Config struct { 14 | REST RESTConfig `mapstructure:"rest"` 15 | Pipeline pipeline.Config `mapstructure:"pipeline"` 16 | } 17 | 18 | type RESTConfig struct { 19 | PG PGConfig `mapstructure:"pg"` 20 | ListenAddr string `mapstructure:"listenAddr"` 21 | BaseURL string `mapstructure:"baseURL"` 22 | OIDC OIDCConfig `mapstructure:"oidc"` 23 | BasicAuth map[string]string `mapstructure:"basicAuth"` 24 | AnonRole string `mapstructure:"anonRole"` 25 | Omitempty bool `mapstructure:"omitempty"` 26 | } 27 | 28 | type PGConfig struct { 29 | ConnString string `mapstructure:"connString"` 30 | } 31 | 32 | type OIDCConfig struct { 33 | ClientID string `mapstructure:"clientID"` 34 | ClientSecret string `mapstructure:"clientSecret"` 35 | Issuer string `mapstructure:"issuer"` 36 | SkipTLSVerify bool `mapstructure:"skipTLSVerify"` 37 | RoleClaimKey string `mapstructure:"roleClaimKey"` 38 | } 39 | 40 | // SetDefaults applies default values to viper 41 | func SetDefaults(v *viper.Viper) { 42 | // REST defaults 43 | v.SetDefault("rest.listenAddr", ":8080") 44 | v.SetDefault("rest.anonRole", "anon") 45 | v.SetDefault("rest.oidc.roleClaimKey", ".policy.pgrole") 46 | v.SetDefault("rest.omitempty", false) 47 | } 48 | 49 | // Load reads config from file or environment 50 | func Load(cfgFile string) (*Config, error) { 51 | v := viper.New() 52 | 53 | SetDefaults(v) 54 | 55 | // Try to load config file 56 | if cfgFile != "" { 57 | v.SetConfigFile(cfgFile) 58 | if err := v.ReadInConfig(); err != nil { 59 | return nil, fmt.Errorf("error reading config file %s: %w", cfgFile, err) 60 | } 61 | fmt.Println("Using config file:", v.ConfigFileUsed()) 62 | } else { 63 | // Look for default config locations 64 | v.SetConfigName("pgo") 65 | v.SetConfigType("yaml") 66 | if home, err := os.UserHomeDir(); err == nil { 67 | v.AddConfigPath(filepath.Join(home, ".config")) 68 | } 69 | v.AddConfigPath(".") 70 | 71 | // Try to read but don't fail if not found 72 | if err := v.ReadInConfig(); err == nil { 73 | fmt.Println("Using config file:", v.ConfigFileUsed()) 74 | } else if _, ok := err.(viper.ConfigFileNotFoundError); !ok { 75 | return nil, fmt.Errorf("error reading default config: %w", err) 76 | } 77 | } 78 | 79 | // Override with environment variables 80 | v.AutomaticEnv() 81 | v.SetEnvPrefix("PGO") 82 | 83 | // CLI flags can override via viper.BindPFlag() elsewhere 84 | 85 | // Build the config 86 | var cfg Config 87 | if err := v.Unmarshal(&cfg); err != nil { 88 | return nil, fmt.Errorf("unable to decode config: %w", err) 89 | } 90 | 91 | // Handle special cases 92 | if cfg.REST.AnonRole == "" { 93 | cfg.REST.AnonRole = "anon" 94 | } 95 | 96 | return &cfg, nil 97 | } 98 | 99 | // Version is the current version of the application 100 | var Version = `v0.0.1-experimental` // should be overridden by build process, ldflags 101 | -------------------------------------------------------------------------------- /pkg/config/example.http-peers.yaml: -------------------------------------------------------------------------------- 1 | # Basic API Key auth 2 | - name: webhook-notifications 3 | connector: http 4 | config: 5 | endpoints: 6 | - url: "https://api.example.com/webhook" 7 | method: "POST" 8 | headers: 9 | X-Custom-Header: "value" 10 | auth: 11 | type: "apikey" 12 | apiKey: "your-api-key" 13 | apiKeyName: "X-API-Key" # optional, defaults to X-API-Key 14 | 15 | # OAuth2/Bearer token 16 | - name: oauth2-webhook 17 | connector: http 18 | config: 19 | endpoints: 20 | - url: "https://api.company.com/webhook" 21 | auth: 22 | type: "oauth2" 23 | clientId: "client-id" 24 | clientSecret: "client-secret" 25 | tokenUrl: "https://auth.company.com/token" 26 | scopes: "webhook.write" 27 | 28 | # Cloudflare API 29 | - name: cloudflare-webhook 30 | connector: http 31 | config: 32 | endpoints: 33 | - url: "https://api.cloudflare.com/client/v4/zones/{zone-id}/webhooks" 34 | auth: 35 | type: "cloudflare" 36 | cloudflareToken: "your-api-token" 37 | # Or use API key + email 38 | # cloudflareKey: "your-api-key" 39 | # cloudflareEmail: "your-email@example.com" 40 | 41 | # GCP Service Account 42 | - name: gcp-webhook 43 | connector: http 44 | config: 45 | endpoints: 46 | - url: "https://us-central1-project.cloudfunctions.net/webhook" 47 | auth: 48 | type: "gcp_service_account" 49 | serviceAccountFile: "/path/to/service-account.json" 50 | -------------------------------------------------------------------------------- /pkg/config/pg-debug-basic.config.yaml: -------------------------------------------------------------------------------- 1 | peers: 2 | - name: example-pg-source 3 | connector: postgres 4 | config: 5 | connString: "host=localhost port=5432 user=postgres password=secret dbname=testdb replication=database" 6 | replication: 7 | tables: ["*"] 8 | - name: debug # logs CDC events to stdout 9 | connector: debug 10 | 11 | pipelines: 12 | - name: log-pg-cdc 13 | sources: 14 | - name: example-pg-source 15 | sinks: 16 | - name: debug -------------------------------------------------------------------------------- /pkg/httputil/context.go: -------------------------------------------------------------------------------- 1 | package httputil 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | ) 7 | 8 | type ContextKey string 9 | 10 | const ( 11 | RequestIDCtxKey ContextKey = "RequestID" 12 | LogEntryCtxKey ContextKey = "LogEntry" 13 | OIDCUserCtxKey ContextKey = "OIDCUser" 14 | BasicAuthCtxKey ContextKey = "BasicAuth" 15 | PgConnCtxKey ContextKey = "PgConn" 16 | OIDCRoleClaimCtxKey ContextKey = "OIDCRoleClaim" 17 | ) 18 | 19 | func OIDCUser(r *http.Request) (map[string]any, bool) { 20 | claims, ok := r.Context().Value(OIDCUserCtxKey).(map[string]any) 21 | if !ok || claims == nil { 22 | return nil, false 23 | } 24 | return claims, true 25 | } 26 | 27 | // migrate from zitadel/oidc to coreos/go-oidc 28 | // func OIDCUser(r *http.Request) (*oidc.IntrospectionResponse, bool) { 29 | // user, ok := r.Context().Value(OIDCUserCtxKey).(*oidc.IntrospectionResponse) 30 | // if !ok || user == nil { 31 | // return nil, false 32 | // } 33 | // return user, true 34 | // } 35 | 36 | // BasicAuthUser retrieves the authenticated username from the context. 37 | func BasicAuthUser(r *http.Request) (string, bool) { 38 | user, ok := r.Context().Value(BasicAuthCtxKey).(string) 39 | return user, ok 40 | } 41 | 42 | // BindOrError decodes the JSON body of an HTTP request, r, into the given destination object, dst. 43 | // If decoding fails, it responds with a 400 Bad Request error. 44 | func BindOrError(r *http.Request, w http.ResponseWriter, dst interface{}) error { 45 | if err := json.NewDecoder(r.Body).Decode(dst); err != nil { 46 | Error(w, http.StatusBadRequest, err.Error()) 47 | return err 48 | } 49 | return nil 50 | } 51 | 52 | // JSON writes a JSON response with the given status code and data. 53 | func JSON(w http.ResponseWriter, statusCode int, data interface{}) { 54 | w.Header().Set("Content-Type", "application/json") 55 | w.WriteHeader(statusCode) 56 | if err := json.NewEncoder(w).Encode(data); err != nil { 57 | http.Error(w, "Failed to encode response", http.StatusInternalServerError) 58 | } 59 | } 60 | 61 | // Text writes a plain text response with the given status code and text content. 62 | func Text(w http.ResponseWriter, statusCode int, text string) { 63 | w.Header().Set("Content-Type", "text/plain") 64 | w.WriteHeader(statusCode) 65 | if _, err := w.Write([]byte(text)); err != nil { 66 | http.Error(w, "Failed to write response", http.StatusInternalServerError) 67 | } 68 | } 69 | 70 | // HTML writes an HTML response with the given status code and HTML content. 71 | func HTML(w http.ResponseWriter, statusCode int, html string) { 72 | w.Header().Set("Content-Type", "text/html") 73 | w.WriteHeader(statusCode) 74 | if _, err := w.Write([]byte(html)); err != nil { 75 | http.Error(w, "Failed to write response", http.StatusInternalServerError) 76 | } 77 | } 78 | 79 | // Blob writes a binary response with the given status code and data. 80 | func Blob(w http.ResponseWriter, statusCode int, data []byte, contentType string) { 81 | w.Header().Set("Content-Type", contentType) 82 | w.WriteHeader(statusCode) 83 | if _, err := w.Write(data); err != nil { 84 | http.Error(w, "Failed to write response", http.StatusInternalServerError) 85 | } 86 | } 87 | 88 | // ErrorResponse represents a structured error response. 89 | type ErrorResponse struct { 90 | Message string `json:"message"` 91 | Code int `json:"code"` 92 | } 93 | 94 | // Error sends a JSON response with an error code and message. 95 | func Error(w http.ResponseWriter, statusCode int, message string) { 96 | w.Header().Set("Content-Type", "application/json") 97 | w.WriteHeader(statusCode) 98 | errorResponse := ErrorResponse{ 99 | Code: statusCode, 100 | Message: message, 101 | } 102 | if err := json.NewEncoder(w).Encode(errorResponse); err != nil { 103 | // Fallback if JSON encoding fails 104 | http.Error(w, "Failed to encode error response", http.StatusInternalServerError) 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /pkg/httputil/middleware/authz.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/edgeflare/pgo/pkg/httputil" 7 | "github.com/edgeflare/pgo/pkg/util" 8 | ) 9 | 10 | // AuthzResponse contains authorization result. 11 | type AuthzResponse struct { 12 | Role string `json:"role"` 13 | Allowed bool `json:"allowed"` 14 | } 15 | 16 | // AuthzFunc evaluates context and returns authorization status. 17 | type AuthzFunc func(ctx context.Context) (AuthzResponse, error) 18 | 19 | // WithOIDCAuthz extracts role from OIDC token and adds to context. 20 | func WithOIDCAuthz(oidcCfg OIDCProviderConfig, roleClaimKey string) AuthzFunc { 21 | oidcInitOnce.Do(func() { 22 | if oidcProvider == nil { 23 | oidcProvider = initOIDCProvider(oidcCfg) 24 | } 25 | }) 26 | return func(ctx context.Context) (AuthzResponse, error) { 27 | user, ok := ctx.Value(httputil.OIDCUserCtxKey).(map[string]any) 28 | if !ok { 29 | return AuthzResponse{Allowed: false}, nil 30 | } 31 | pgrole, err := util.Jq(user, roleClaimKey) 32 | if err != nil { 33 | return AuthzResponse{Allowed: false}, nil 34 | } 35 | role, ok := pgrole.(string) 36 | if !ok { 37 | return AuthzResponse{Allowed: false}, nil 38 | } 39 | _ = context.WithValue(ctx, httputil.OIDCRoleClaimCtxKey, pgrole) 40 | return AuthzResponse{Role: role, Allowed: true}, nil 41 | } 42 | } 43 | 44 | // WithBasicAuthz creates auth function for Basic Auth. 45 | func WithBasicAuthz() AuthzFunc { 46 | return func(ctx context.Context) (AuthzResponse, error) { 47 | user, ok := ctx.Value(httputil.BasicAuthCtxKey).(string) 48 | if !ok { 49 | return AuthzResponse{Allowed: false}, nil 50 | } 51 | _ = context.WithValue(ctx, httputil.OIDCRoleClaimCtxKey, user) 52 | return AuthzResponse{Role: user, Allowed: true}, nil 53 | } 54 | } 55 | 56 | // WithAnonAuthz creates auth function using specified role. 57 | func WithAnonAuthz(anonRole string) AuthzFunc { 58 | return func(ctx context.Context) (AuthzResponse, error) { 59 | _ = context.WithValue(ctx, httputil.OIDCRoleClaimCtxKey, anonRole) 60 | return AuthzResponse{Role: anonRole, Allowed: true}, nil 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /pkg/httputil/middleware/basic_auth.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "context" 5 | "encoding/base64" 6 | "net/http" 7 | "strings" 8 | 9 | "github.com/edgeflare/pgo/pkg/httputil" 10 | ) 11 | 12 | // BasicAuthConfig holds the username-password pairs for basic authentication. 13 | type BasicAuthConfig struct { 14 | Credentials map[string]string 15 | } 16 | 17 | // NewBasicAuthCreds creates a new instance of BasicAuthConfig with multiple username/password pairs. 18 | func BasicAuthCreds(credentials map[string]string) *BasicAuthConfig { 19 | return &BasicAuthConfig{ 20 | Credentials: credentials, 21 | } 22 | } 23 | 24 | // VerifyBasicAuth is a middleware function for basic authentication. 25 | // By default, it sends a 401 Unauthorized response if credentials are missing or invalid. 26 | // If send401Unauthorized is false, it allows requests without valid Basic Auth credentials 27 | // to continue without interference. 28 | func VerifyBasicAuth(config *BasicAuthConfig, send401Unauthorized ...bool) func(http.Handler) http.Handler { 29 | send401 := true // Default behavior: Send 401 on failure 30 | if len(send401Unauthorized) > 0 { 31 | send401 = send401Unauthorized[0] 32 | } 33 | 34 | return func(next http.Handler) http.Handler { 35 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 36 | authHeader := r.Header.Get("Authorization") 37 | if authHeader == "" { 38 | if send401 { 39 | w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`) 40 | http.Error(w, "Authorization header missing", http.StatusUnauthorized) 41 | return 42 | } 43 | // No Authorization header and send401Unauthorized is false, 44 | // so let other middleware/handlers handle it 45 | next.ServeHTTP(w, r) 46 | return 47 | } 48 | 49 | // Check if the authorization header is in the correct format 50 | if !strings.HasPrefix(authHeader, "Basic ") { 51 | if send401 { 52 | http.Error(w, "Invalid authorization format", http.StatusUnauthorized) 53 | return 54 | } 55 | // Other authorization scheme present and send401Unauthorized is false 56 | next.ServeHTTP(w, r) 57 | return 58 | } 59 | 60 | // Decode the base64 encoded credentials 61 | encodedCredentials := strings.TrimPrefix(authHeader, "Basic ") 62 | credentials, err := base64.StdEncoding.DecodeString(encodedCredentials) 63 | if err != nil { 64 | if send401 { 65 | http.Error(w, "Invalid base64 encoding", http.StatusUnauthorized) 66 | return 67 | } 68 | next.ServeHTTP(w, r) 69 | return 70 | } 71 | 72 | // Split the credentials into username and password 73 | creds := strings.SplitN(string(credentials), ":", 2) 74 | if len(creds) != 2 { 75 | if send401 { 76 | http.Error(w, "Invalid credentials format", http.StatusUnauthorized) 77 | return 78 | } 79 | next.ServeHTTP(w, r) 80 | return 81 | } 82 | 83 | username, password := creds[0], creds[1] 84 | 85 | // Verify the credentials 86 | if validPassword, ok := config.Credentials[username]; !ok || validPassword != password { 87 | if send401 { 88 | w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`) 89 | http.Error(w, "Invalid credentials", http.StatusUnauthorized) 90 | return 91 | } 92 | next.ServeHTTP(w, r) 93 | return 94 | } 95 | 96 | // Store authenticated user in context 97 | ctx := context.WithValue(r.Context(), httputil.BasicAuthCtxKey, username) 98 | next.ServeHTTP(w, r.WithContext(ctx)) 99 | }) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /pkg/httputil/middleware/basic_auth_test.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "encoding/base64" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | 9 | "github.com/edgeflare/pgo/pkg/httputil" 10 | ) 11 | 12 | func TestVerifyBasicAuth(t *testing.T) { 13 | tests := []struct { 14 | config *BasicAuthConfig 15 | name string 16 | authHeader string 17 | expectedBody string 18 | expectedUser string 19 | expectedStatus int 20 | }{ 21 | { 22 | name: "missing authorization header", 23 | config: BasicAuthCreds(map[string]string{"user": "pass"}), 24 | authHeader: "", 25 | expectedStatus: http.StatusUnauthorized, 26 | expectedBody: "Authorization header missing\n", 27 | }, 28 | { 29 | name: "invalid authorization format", 30 | config: BasicAuthCreds(map[string]string{"user": "pass"}), 31 | authHeader: "Bearer some-token", 32 | expectedStatus: http.StatusUnauthorized, 33 | expectedBody: "Invalid authorization format\n", 34 | }, 35 | { 36 | name: "invalid base64 encoding", 37 | config: BasicAuthCreds(map[string]string{"user": "pass"}), 38 | authHeader: "Basic invalid-base64", 39 | expectedStatus: http.StatusUnauthorized, 40 | expectedBody: "Invalid base64 encoding\n", 41 | }, 42 | { 43 | name: "invalid credentials format", 44 | config: BasicAuthCreds(map[string]string{"user": "pass"}), 45 | authHeader: "Basic " + base64.StdEncoding.EncodeToString([]byte("userpass")), 46 | expectedStatus: http.StatusUnauthorized, 47 | expectedBody: "Invalid credentials format\n", 48 | }, 49 | { 50 | name: "invalid credentials", 51 | config: BasicAuthCreds(map[string]string{"user": "pass"}), 52 | authHeader: "Basic " + base64.StdEncoding.EncodeToString([]byte("user:wrongpass")), 53 | expectedStatus: http.StatusUnauthorized, 54 | expectedBody: "Invalid credentials\n", 55 | }, 56 | { 57 | name: "valid credentials", 58 | config: BasicAuthCreds(map[string]string{"user": "pass"}), 59 | authHeader: "Basic " + base64.StdEncoding.EncodeToString([]byte("user:pass")), 60 | expectedStatus: http.StatusOK, 61 | expectedBody: "OK", 62 | expectedUser: "user", 63 | }, 64 | } 65 | 66 | for _, tt := range tests { 67 | t.Run(tt.name, func(t *testing.T) { 68 | req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil) 69 | if tt.authHeader != "" { 70 | req.Header.Set("Authorization", tt.authHeader) 71 | } 72 | rr := httptest.NewRecorder() 73 | 74 | handler := VerifyBasicAuth(tt.config)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 75 | user, ok := r.Context().Value(httputil.BasicAuthCtxKey).(string) 76 | if !ok || user != tt.expectedUser { 77 | http.Error(w, "User not found in context", http.StatusInternalServerError) 78 | return 79 | } 80 | w.WriteHeader(http.StatusOK) 81 | w.Write([]byte("OK")) 82 | })) 83 | 84 | handler.ServeHTTP(rr, req) 85 | 86 | if status := rr.Code; status != tt.expectedStatus { 87 | t.Errorf("status code: expected %v, got %v", tt.expectedStatus, status) 88 | } 89 | 90 | if body := rr.Body.String(); body != tt.expectedBody { 91 | t.Errorf("body: expected %v, got %v", tt.expectedBody, body) 92 | } 93 | }) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /pkg/httputil/middleware/cache.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | // Cache is a simple in-memory cache with expiration 9 | type Cache struct { 10 | items map[string]cacheItem 11 | sync.RWMutex 12 | } 13 | 14 | // cacheItem holds cached data along with its expiration 15 | type cacheItem struct { 16 | value interface{} 17 | expiration time.Time 18 | } 19 | 20 | // NewCache creates a new Cache 21 | func NewCache() *Cache { 22 | return &Cache{ 23 | items: make(map[string]cacheItem), 24 | } 25 | } 26 | 27 | // Set adds an item to the cache with a specified expiration duration 28 | func (c *Cache) Set(key string, value interface{}, duration time.Duration) { 29 | c.Lock() 30 | defer c.Unlock() 31 | c.items[key] = cacheItem{ 32 | value: value, 33 | expiration: time.Now().Add(duration), 34 | } 35 | } 36 | 37 | // Get retrieves an item from the cache 38 | func (c *Cache) Get(key string) (interface{}, bool) { 39 | c.RLock() 40 | defer c.RUnlock() 41 | item, found := c.items[key] 42 | if !found { 43 | return nil, false 44 | } 45 | if time.Now().After(item.expiration) { 46 | delete(c.items, key) 47 | return nil, false 48 | } 49 | return item.value, true 50 | } 51 | 52 | // CleanupExpired removes expired items from the cache 53 | func (c *Cache) CleanupExpired() { 54 | c.Lock() 55 | defer c.Unlock() 56 | for key, item := range c.items { 57 | if time.Now().After(item.expiration) { 58 | delete(c.items, key) 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /pkg/httputil/middleware/cache_test.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestCache_SetAndGet(t *testing.T) { 11 | cache := NewCache() 12 | 13 | // Test setting and getting a value 14 | cache.Set("key1", "value1", 1*time.Minute) 15 | value, found := cache.Get("key1") 16 | if !found { 17 | t.Error("Expected to find key1") 18 | } 19 | if value != "value1" { 20 | t.Errorf("Expected value1, got %v", value) 21 | } 22 | 23 | // Test getting a non-existent key 24 | _, found = cache.Get("nonexistent") 25 | if found { 26 | t.Error("Expected not to find nonexistent key") 27 | } 28 | } 29 | 30 | func TestCache_Expiration(t *testing.T) { 31 | cache := NewCache() 32 | 33 | // Set a key with a short expiration 34 | cache.Set("short", "value", 10*time.Millisecond) 35 | 36 | // Wait for expiration 37 | time.Sleep(20 * time.Millisecond) 38 | 39 | // Try to get the expired key 40 | _, found := cache.Get("short") 41 | if found { 42 | t.Error("Expected short to be expired") 43 | } 44 | } 45 | 46 | func TestCache_CleanupExpired(t *testing.T) { 47 | cache := NewCache() 48 | 49 | // Set some keys with different expiration times 50 | cache.Set("expired1", "value1", 10*time.Millisecond) 51 | cache.Set("expired2", "value2", 10*time.Millisecond) 52 | cache.Set("valid", "value3", 1*time.Minute) 53 | 54 | // Wait for some keys to expire 55 | time.Sleep(20 * time.Millisecond) 56 | 57 | // Run cleanup 58 | cache.CleanupExpired() 59 | 60 | // Check if expired keys were removed and valid key remains 61 | _, found1 := cache.Get("expired1") 62 | _, found2 := cache.Get("expired2") 63 | _, found3 := cache.Get("valid") 64 | 65 | if found1 || found2 { 66 | t.Error("Expected expired keys to be removed") 67 | } 68 | if !found3 { 69 | t.Error("Expected valid key to remain") 70 | } 71 | } 72 | 73 | func TestCache_Concurrency(t *testing.T) { 74 | cache := NewCache() 75 | var wg sync.WaitGroup 76 | concurrency := 100 77 | 78 | // Concurrent writes 79 | for i := 0; i < concurrency; i++ { 80 | wg.Add(1) 81 | go func(i int) { 82 | defer wg.Done() 83 | key := fmt.Sprintf("key%d", i) 84 | cache.Set(key, i, 1*time.Minute) 85 | }(i) 86 | } 87 | 88 | // Concurrent reads 89 | for i := 0; i < concurrency; i++ { 90 | wg.Add(1) 91 | go func(i int) { 92 | defer wg.Done() 93 | key := fmt.Sprintf("key%d", i) 94 | _, _ = cache.Get(key) 95 | }(i) 96 | } 97 | 98 | wg.Wait() 99 | 100 | // Verify all keys are present 101 | for i := 0; i < concurrency; i++ { 102 | key := fmt.Sprintf("key%d", i) 103 | _, found := cache.Get(key) 104 | if !found { 105 | t.Errorf("Expected to find key: %s", key) 106 | } 107 | } 108 | } 109 | 110 | func BenchmarkCache_Set(b *testing.B) { 111 | cache := NewCache() 112 | for i := 0; i < b.N; i++ { 113 | cache.Set(fmt.Sprintf("key%d", i), i, 1*time.Minute) 114 | } 115 | } 116 | 117 | func BenchmarkCache_Get(b *testing.B) { 118 | cache := NewCache() 119 | for i := 0; i < 1000; i++ { 120 | cache.Set(fmt.Sprintf("key%d", i), i, 1*time.Minute) 121 | } 122 | 123 | b.ResetTimer() 124 | for i := 0; i < b.N; i++ { 125 | cache.Get(fmt.Sprintf("key%d", i%1000)) 126 | } 127 | } 128 | 129 | func BenchmarkCache_SetParallel(b *testing.B) { 130 | cache := NewCache() 131 | b.RunParallel(func(pb *testing.PB) { 132 | i := 0 133 | for pb.Next() { 134 | cache.Set(fmt.Sprintf("key%d", i), i, 1*time.Minute) 135 | i++ 136 | } 137 | }) 138 | } 139 | 140 | func BenchmarkCache_GetParallel(b *testing.B) { 141 | cache := NewCache() 142 | for i := 0; i < 1000; i++ { 143 | cache.Set(fmt.Sprintf("key%d", i), i, 1*time.Minute) 144 | } 145 | 146 | b.ResetTimer() 147 | b.RunParallel(func(pb *testing.PB) { 148 | i := 0 149 | for pb.Next() { 150 | cache.Get(fmt.Sprintf("key%d", i%1000)) 151 | i++ 152 | } 153 | }) 154 | } 155 | -------------------------------------------------------------------------------- /pkg/httputil/middleware/cors.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | ) 7 | 8 | // CORSOptions defines configuration for CORS. 9 | type CORSOptions struct { 10 | AllowedOrigins []string 11 | AllowedMethods []string 12 | AllowedHeaders []string 13 | AllowCredentials bool 14 | } 15 | 16 | // defaultCORSOptions returns the default CORS options. 17 | func defaultCORSOptions() *CORSOptions { 18 | return &CORSOptions{ 19 | AllowedOrigins: []string{"*"}, 20 | AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, 21 | AllowedHeaders: []string{"Content-Type", "Content-Length", "Accept-Encoding", "X-CSRF-Token", "Authorization", "accept", "origin", "Cache-Control", "X-Requested-With"}, 22 | AllowCredentials: true, 23 | } 24 | } 25 | 26 | // CORSWithOptions creates a CORS middleware with the provided configuration. 27 | // If options is nil, it will use the default CORS settings. 28 | // If options is an empty struct (CORSOptions{}), it will create a middleware with no CORS headers. 29 | func CORSWithOptions(options *CORSOptions) func(http.Handler) http.Handler { 30 | if options == nil { 31 | options = defaultCORSOptions() 32 | } 33 | 34 | return func(next http.Handler) http.Handler { 35 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 36 | if len(options.AllowedOrigins) > 0 { 37 | w.Header().Set("Access-Control-Allow-Origin", strings.Join(options.AllowedOrigins, ",")) 38 | } 39 | if len(options.AllowedMethods) > 0 { 40 | w.Header().Set("Access-Control-Allow-Methods", strings.Join(options.AllowedMethods, ",")) 41 | } 42 | if len(options.AllowedHeaders) > 0 { 43 | w.Header().Set("Access-Control-Allow-Headers", strings.Join(options.AllowedHeaders, ",")) 44 | } 45 | if options.AllowCredentials { 46 | w.Header().Set("Access-Control-Allow-Credentials", "true") 47 | } 48 | 49 | // Handle preflight request 50 | if r.Method == http.MethodOptions { 51 | w.WriteHeader(http.StatusNoContent) 52 | return 53 | } 54 | 55 | next.ServeHTTP(w, r) 56 | }) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /pkg/httputil/middleware/cors_test.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "testing" 7 | ) 8 | 9 | func TestCORSWithOptions(t *testing.T) { 10 | tests := []struct { 11 | options *CORSOptions 12 | expectedHeaders map[string]string 13 | name string 14 | method string 15 | expectedStatus int 16 | }{ 17 | { 18 | name: "default options", 19 | method: http.MethodGet, 20 | options: defaultCORSOptions(), 21 | expectedHeaders: map[string]string{ 22 | "Access-Control-Allow-Origin": "*", 23 | "Access-Control-Allow-Methods": "GET,POST,PUT,DELETE,OPTIONS", 24 | "Access-Control-Allow-Headers": "Content-Type,Content-Length,Accept-Encoding,X-CSRF-Token,Authorization,accept,origin,Cache-Control,X-Requested-With", 25 | "Access-Control-Allow-Credentials": "true", 26 | }, 27 | expectedStatus: http.StatusOK, 28 | }, 29 | { 30 | name: "custom options", 31 | method: http.MethodGet, 32 | options: &CORSOptions{ 33 | AllowedOrigins: []string{"http://example.com"}, 34 | AllowedMethods: []string{"GET", "POST"}, 35 | AllowedHeaders: []string{"Content-Type"}, 36 | AllowCredentials: false, 37 | }, 38 | expectedHeaders: map[string]string{ 39 | "Access-Control-Allow-Origin": "http://example.com", 40 | "Access-Control-Allow-Methods": "GET,POST", 41 | "Access-Control-Allow-Headers": "Content-Type", 42 | }, 43 | expectedStatus: http.StatusOK, 44 | }, 45 | { 46 | name: "empty options", 47 | method: http.MethodGet, 48 | options: &CORSOptions{}, 49 | expectedHeaders: map[string]string{ 50 | // No CORS headers should be set 51 | }, 52 | expectedStatus: http.StatusOK, 53 | }, 54 | { 55 | name: "preflight request", 56 | method: http.MethodOptions, 57 | options: defaultCORSOptions(), 58 | expectedHeaders: map[string]string{ 59 | "Access-Control-Allow-Origin": "*", 60 | "Access-Control-Allow-Methods": "GET,POST,PUT,DELETE,OPTIONS", 61 | "Access-Control-Allow-Headers": "Content-Type,Content-Length,Accept-Encoding,X-CSRF-Token,Authorization,accept,origin,Cache-Control,X-Requested-With", 62 | "Access-Control-Allow-Credentials": "true", 63 | }, 64 | expectedStatus: http.StatusNoContent, 65 | }, 66 | } 67 | 68 | for _, tt := range tests { 69 | t.Run(tt.name, func(t *testing.T) { 70 | req, _ := http.NewRequest(tt.method, "http://example.com", nil) 71 | rr := httptest.NewRecorder() 72 | 73 | handler := CORSWithOptions(tt.options)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 74 | w.WriteHeader(http.StatusOK) 75 | })) 76 | 77 | handler.ServeHTTP(rr, req) 78 | 79 | for header, expectedValue := range tt.expectedHeaders { 80 | if value := rr.Header().Get(header); value != expectedValue { 81 | t.Errorf("header %s: expected %v, got %v", header, expectedValue, value) 82 | } 83 | } 84 | 85 | if status := rr.Code; status != tt.expectedStatus { 86 | t.Errorf("status code: expected %v, got %v", tt.expectedStatus, status) 87 | } 88 | }) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /pkg/httputil/middleware/file_server.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "bytes" 5 | "embed" 6 | "io/fs" 7 | "mime" 8 | "net/http" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | 13 | "log" 14 | ) 15 | 16 | // Static returns an http.Handler that serves static files. 17 | // If the embeddedFS arg is not nil, it uses serveEmbedded; else, serveLocal 18 | // 19 | // Example usage: 20 | // 21 | // mux := http.NewServeMux() 22 | // mux.Handle("GET /", middleware.Static("dist", true, embeddedFS)) 23 | // 24 | // This will serve files from the "dist" directory of the embedded file system and use "index.html" 25 | // as a fallback for routes not directly mapped to a file. 26 | func Static(directory string, spaFallback bool, embeddedFS *embed.FS) http.Handler { 27 | if embeddedFS != nil { 28 | return serveEmbedded(*embeddedFS, directory, spaFallback) 29 | } 30 | return serveLocal(directory, spaFallback) 31 | } 32 | 33 | func serveEmbedded(embeddedFS embed.FS, baseDir string, spaFallback bool) http.Handler { 34 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 35 | filePath := filepath.Join(baseDir, r.URL.Path) 36 | if !isValidFilePath(filePath) { 37 | http.Error(w, "Forbidden", http.StatusForbidden) 38 | return 39 | } 40 | 41 | fileData, fileInfo, err := readFileFromFS(embeddedFS, filePath) 42 | if err != nil && spaFallback { 43 | filePath = filepath.Join(baseDir, "index.html") 44 | fileData, fileInfo, err = readFileFromFS(embeddedFS, filePath) 45 | } 46 | 47 | if err != nil { 48 | http.NotFound(w, r) 49 | return 50 | } 51 | 52 | setContentType(w, filePath) 53 | http.ServeContent(w, r, filePath, fileInfo.ModTime(), bytes.NewReader(fileData)) 54 | }) 55 | } 56 | 57 | // serveLocal returns an http.Handler that serves static files from the given directory in the host machine/container 58 | func serveLocal(directory string, spaFallback bool) http.Handler { 59 | absDir, err := filepath.Abs(directory) 60 | if err != nil { 61 | log.Fatalf("Failed to resolve absolute path for %s: %v", directory, err) 62 | } 63 | 64 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 65 | path := strings.TrimPrefix(r.URL.Path, "/") 66 | cleanPath := filepath.Clean(path) 67 | 68 | requestedPath := filepath.Join(absDir, cleanPath) 69 | if !isSubPath(absDir, requestedPath) { 70 | http.Error(w, "Forbidden", http.StatusForbidden) 71 | return 72 | } 73 | 74 | fileInfo, err := os.Stat(requestedPath) 75 | if err != nil && spaFallback { 76 | indexPath := filepath.Join(absDir, "index.html") 77 | requestedPath = indexPath 78 | fileInfo, err = os.Stat(indexPath) 79 | } 80 | 81 | if err != nil { 82 | http.NotFound(w, r) 83 | return 84 | } 85 | 86 | if fileInfo.IsDir() && !spaFallback { 87 | http.Error(w, "Directory listing not allowed", http.StatusForbidden) 88 | return 89 | } 90 | 91 | setContentType(w, requestedPath) 92 | http.ServeFile(w, r, requestedPath) 93 | }) 94 | } 95 | 96 | // isValidFilePath checks if the file path is valid and does not contain any directory traversal attempts. 97 | func isValidFilePath(filePath string) bool { 98 | cleaned := filepath.Clean(filePath) 99 | return !strings.HasPrefix(cleaned, ".") 100 | } 101 | 102 | // readFileFromFS reads a file from the embedded filesystem and returns its data and FileInfo. 103 | func readFileFromFS(embeddedFS embed.FS, filePath string) ([]byte, fs.FileInfo, error) { 104 | fileData, err := fs.ReadFile(embeddedFS, filePath) 105 | if err != nil { 106 | return nil, nil, err 107 | } 108 | 109 | fileInfo, err := fs.Stat(embeddedFS, filePath) 110 | if err != nil { 111 | return nil, nil, err 112 | } 113 | 114 | return fileData, fileInfo, nil 115 | } 116 | 117 | // isSubPath checks if a path is a subdirectory of the base directory. 118 | func isSubPath(baseDir, path string) bool { 119 | rel, err := filepath.Rel(baseDir, path) 120 | return err == nil && !strings.HasPrefix(rel, "..") 121 | } 122 | 123 | // setContentType sets the Content-Type header based on the file extension. 124 | func setContentType(w http.ResponseWriter, filePath string) { 125 | if ext := filepath.Ext(filePath); ext != "" { 126 | if ct := mime.TypeByExtension(ext); ct != "" { 127 | w.Header().Set("Content-Type", ct) 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /pkg/httputil/middleware/logger.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "time" 8 | 9 | "github.com/edgeflare/pgo/pkg/httputil" 10 | "github.com/google/uuid" 11 | "go.uber.org/zap" 12 | ) 13 | 14 | // ResponseRecorder is a wrapper for http.ResponseWriter to capture status codes and durations. 15 | type ResponseRecorder struct { 16 | start time.Time 17 | http.ResponseWriter 18 | StatusCode int 19 | } 20 | 21 | func NewResponseRecorder(w http.ResponseWriter) *ResponseRecorder { 22 | return &ResponseRecorder{ 23 | ResponseWriter: w, 24 | StatusCode: http.StatusOK, 25 | start: time.Now(), 26 | } 27 | } 28 | 29 | func (rr *ResponseRecorder) WriteHeader(statusCode int) { 30 | rr.StatusCode = statusCode 31 | rr.ResponseWriter.WriteHeader(statusCode) 32 | } 33 | 34 | func (rr *ResponseRecorder) Write(b []byte) (int, error) { 35 | return rr.ResponseWriter.Write(b) 36 | } 37 | 38 | // Retrieve log metadata from context 39 | func GetLogEntryMetadata(ctx context.Context) map[string]interface{} { 40 | if metadata, ok := ctx.Value(httputil.LogEntryCtxKey).(map[string]interface{}); ok { 41 | return metadata 42 | } 43 | return nil 44 | } 45 | 46 | // LoggerOptions defines configuration for the logger middleware. 47 | type LoggerOptions struct { 48 | Logger *zap.Logger 49 | Format func(reqID string, rec *ResponseRecorder, r *http.Request, latency time.Duration) []zap.Field 50 | } 51 | 52 | var defaultLogger *zap.Logger 53 | 54 | func init() { 55 | var err error 56 | defaultLogger, err = zap.NewProduction() 57 | if err != nil { 58 | panic(err) 59 | } 60 | defer defaultLogger.Sync() 61 | } 62 | 63 | func LoggerWithOptions(options *LoggerOptions) func(http.Handler) http.Handler { 64 | if options == nil { 65 | options = &LoggerOptions{Logger: defaultLogger} 66 | } 67 | 68 | if options.Format == nil { 69 | options.Format = func(reqID string, rec *ResponseRecorder, r *http.Request, latency time.Duration) []zap.Field { 70 | return []zap.Field{ 71 | zap.String("req_id", reqID), 72 | zap.Int("status", rec.StatusCode), 73 | zap.String("method", r.Method), 74 | zap.String("host", r.Host), 75 | zap.String("url", r.URL.String()), 76 | zap.String("remote_addr", r.RemoteAddr), 77 | zap.String("user_agent", r.UserAgent()), 78 | zap.Duration("latency", latency), 79 | } 80 | } 81 | } 82 | 83 | return func(next http.Handler) http.Handler { 84 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 85 | start := time.Now() 86 | if _, ok := r.Context().Value(httputil.LogEntryCtxKey).(*zap.Logger); !ok { 87 | reqID, ok := r.Context().Value(httputil.RequestIDCtxKey).(string) 88 | if !ok { 89 | reqID = uuid.Nil.String() 90 | } 91 | 92 | rec := NewResponseRecorder(w) 93 | // try to minimize the data passed via context 94 | ctx := context.WithValue(r.Context(), httputil.LogEntryCtxKey, options.Logger) 95 | r = r.WithContext(ctx) 96 | 97 | next.ServeHTTP(rec, r) 98 | 99 | latency := time.Since(start) 100 | 101 | pgRole, ok := r.Context().Value(httputil.OIDCRoleClaimCtxKey).(string) 102 | if !ok { 103 | fmt.Println("pg_role isn't set. to fix add logger after all middleware.AuthzFunc{}") 104 | pgRole = "unknown" 105 | } 106 | 107 | fields := options.Format(reqID, rec, r, latency) 108 | fields = append(fields, zap.String("pg_role", pgRole)) 109 | options.Logger.Info("response", fields...) 110 | } else { 111 | next.ServeHTTP(w, r) 112 | } 113 | }) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /pkg/httputil/middleware/logger_test.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | "time" 9 | 10 | "github.com/edgeflare/pgo/pkg/httputil" 11 | "github.com/google/uuid" 12 | "github.com/stretchr/testify/assert" 13 | "github.com/stretchr/testify/require" 14 | "go.uber.org/zap" 15 | "go.uber.org/zap/zaptest/observer" 16 | ) 17 | 18 | func newTestLogger() (*zap.Logger, *observer.ObservedLogs) { 19 | core, logs := observer.New(zap.InfoLevel) 20 | logger := zap.New(core) 21 | return logger, logs 22 | } 23 | 24 | func TestGetLogEntryMetadata(t *testing.T) { 25 | ctx := context.WithValue(context.Background(), httputil.LogEntryCtxKey, map[string]interface{}{"foo": "bar"}) 26 | metadata := GetLogEntryMetadata(ctx) 27 | require.NotNil(t, metadata) 28 | assert.Equal(t, "bar", metadata["foo"]) 29 | 30 | ctx = context.Background() 31 | metadata = GetLogEntryMetadata(ctx) 32 | assert.Nil(t, metadata) 33 | } 34 | 35 | func TestLoggerWithOptions(t *testing.T) { 36 | logger, logs := newTestLogger() 37 | options := &LoggerOptions{ 38 | Logger: logger, 39 | Format: func(reqID string, rec *ResponseRecorder, r *http.Request, latency time.Duration) []zap.Field { 40 | return []zap.Field{ 41 | zap.String("test", "log"), 42 | } 43 | }, 44 | } 45 | middleware := LoggerWithOptions(options) 46 | 47 | handler := middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 48 | w.WriteHeader(http.StatusOK) 49 | })) 50 | 51 | req := httptest.NewRequest(http.MethodGet, "http://example.com/foo", nil) 52 | rr := httptest.NewRecorder() 53 | 54 | handler.ServeHTTP(rr, req) 55 | 56 | assert.Equal(t, http.StatusOK, rr.Code) 57 | assert.Equal(t, 1, logs.Len()) 58 | assert.Equal(t, "response", logs.All()[0].Message) 59 | assert.Equal(t, "log", logs.All()[0].ContextMap()["test"]) 60 | } 61 | 62 | func TestLoggerWithDefaultOptions(t *testing.T) { 63 | logger, logs := newTestLogger() 64 | defaultLogger = logger 65 | middleware := LoggerWithOptions(nil) 66 | 67 | handler := middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 68 | w.WriteHeader(http.StatusOK) 69 | })) 70 | 71 | req := httptest.NewRequest(http.MethodGet, "http://example.com/foo", nil) 72 | rr := httptest.NewRecorder() 73 | 74 | handler.ServeHTTP(rr, req) 75 | 76 | assert.Equal(t, http.StatusOK, rr.Code) 77 | assert.Equal(t, 1, logs.Len()) 78 | assert.Equal(t, "response", logs.All()[0].Message) 79 | assert.Equal(t, "GET", logs.All()[0].ContextMap()["method"]) 80 | } 81 | 82 | func TestLoggerWithoutRequestID(t *testing.T) { 83 | logger, logs := newTestLogger() 84 | options := &LoggerOptions{ 85 | Logger: logger, 86 | } 87 | middleware := LoggerWithOptions(options) 88 | 89 | handler := middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 90 | w.WriteHeader(http.StatusOK) 91 | })) 92 | 93 | req := httptest.NewRequest(http.MethodGet, "http://example.com/foo", nil) 94 | rr := httptest.NewRecorder() 95 | 96 | handler.ServeHTTP(rr, req) 97 | 98 | assert.Equal(t, http.StatusOK, rr.Code) 99 | assert.Equal(t, 1, logs.Len()) 100 | assert.Equal(t, "response", logs.All()[0].Message) 101 | assert.Equal(t, uuid.Nil.String(), logs.All()[0].ContextMap()["req_id"]) 102 | } 103 | 104 | func TestLoggerWithRequestID(t *testing.T) { 105 | logger, logs := newTestLogger() 106 | options := &LoggerOptions{ 107 | Logger: logger, 108 | } 109 | middleware := LoggerWithOptions(options) 110 | 111 | handler := middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 112 | w.WriteHeader(http.StatusOK) 113 | })) 114 | 115 | req := httptest.NewRequest(http.MethodGet, "http://example.com/foo", nil) 116 | reqID := uuid.New().String() 117 | ctx := context.WithValue(req.Context(), httputil.RequestIDCtxKey, reqID) 118 | req = req.WithContext(ctx) 119 | rr := httptest.NewRecorder() 120 | 121 | handler.ServeHTTP(rr, req) 122 | 123 | assert.Equal(t, http.StatusOK, rr.Code) 124 | assert.Equal(t, 1, logs.Len()) 125 | assert.Equal(t, "response", logs.All()[0].Message) 126 | assert.Equal(t, reqID, logs.All()[0].ContextMap()["req_id"]) 127 | } 128 | -------------------------------------------------------------------------------- /pkg/httputil/middleware/middleware.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/edgeflare/pgo/pkg/httputil" 7 | ) 8 | 9 | // Add applies one or more middleware functions to a handler in the order they were provided. 10 | // The first middleware in the list will be the outermost wrapper (executed first). 11 | func Add(h http.Handler, middlewares ...httputil.Middleware) http.Handler { 12 | // Apply middlewares in reverse order so that the first middleware 13 | // in the list is the outermost wrapper (executed first) 14 | for i := len(middlewares) - 1; i >= 0; i-- { 15 | h = middlewares[i](h) 16 | } 17 | return h 18 | } 19 | -------------------------------------------------------------------------------- /pkg/httputil/middleware/postgres.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | 7 | "github.com/edgeflare/pgo/pkg/httputil" 8 | "github.com/jackc/pgx/v5/pgxpool" 9 | ) 10 | 11 | // Postgres middleware attaches a connection from pool to the request context if the http request user is authorized. 12 | func Postgres(pool *pgxpool.Pool, authorizers ...AuthzFunc) func(http.Handler) http.Handler { 13 | return func(next http.Handler) http.Handler { 14 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 15 | ctx := r.Context() 16 | 17 | for _, authorize := range authorizers { 18 | authzResponse, err := authorize(ctx) 19 | if err != nil { 20 | http.Error(w, "Authorization error", http.StatusInternalServerError) 21 | return 22 | } 23 | if authzResponse.Allowed { 24 | ctx = context.WithValue(ctx, httputil.OIDCRoleClaimCtxKey, authzResponse.Role) 25 | break 26 | } 27 | } 28 | 29 | if pgRole, ok := ctx.Value(httputil.OIDCRoleClaimCtxKey).(string); ok { 30 | // Acquire a connection from the default pool 31 | conn, err := pool.Acquire(r.Context()) 32 | if err != nil { 33 | http.Error(w, "Internal Server Error", http.StatusInternalServerError) 34 | return 35 | } 36 | // caller should 37 | // defer conn.Release() 38 | 39 | // set the connection in the context 40 | ctx = context.WithValue(ctx, httputil.PgConnCtxKey, conn) 41 | ctx = context.WithValue(ctx, httputil.OIDCRoleClaimCtxKey, pgRole) 42 | r = r.WithContext(ctx) 43 | next.ServeHTTP(w, r) 44 | } else { 45 | http.Error(w, "Unauthorized", http.StatusUnauthorized) 46 | } 47 | }) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /pkg/httputil/middleware/proxy.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "crypto/tls" 5 | "net/http" 6 | "net/http/httputil" 7 | "net/url" 8 | "strings" 9 | ) 10 | 11 | // ProxyOptions holds the options for the proxy middleware 12 | type ProxyOptions struct { 13 | TLSConfig *tls.Config 14 | TrimPrefix string 15 | ForwardedHost string 16 | } 17 | 18 | // Proxy creates a reverse proxy handler based on the given target and options 19 | func Proxy(target string, opts ProxyOptions) http.HandlerFunc { 20 | // Parse the target URL 21 | targetURL, err := url.Parse(target) 22 | if err != nil { 23 | return func(w http.ResponseWriter, _ *http.Request) { 24 | http.Error(w, "invalid target URL", http.StatusInternalServerError) 25 | } 26 | } 27 | 28 | // Set default options 29 | if opts.ForwardedHost == "" { 30 | opts.ForwardedHost = targetURL.Host 31 | } 32 | if opts.TLSConfig == nil { 33 | opts.TLSConfig = &tls.Config{ 34 | InsecureSkipVerify: true, 35 | ServerName: targetURL.Hostname(), 36 | } 37 | } 38 | 39 | proxy := httputil.NewSingleHostReverseProxy(targetURL) 40 | 41 | // Configure the Director function 42 | proxy.Director = func(req *http.Request) { 43 | // Preserve the original request context 44 | req.URL.Scheme = targetURL.Scheme 45 | req.URL.Host = targetURL.Host 46 | 47 | // Update the Host header to the target host 48 | req.Host = targetURL.Host 49 | 50 | // Trim prefix if provided 51 | if opts.TrimPrefix != "" { 52 | req.URL.Path = strings.TrimPrefix(req.URL.Path, opts.TrimPrefix) 53 | } 54 | 55 | // Set the X-Forwarded-Host header if provided 56 | if opts.ForwardedHost != "" { 57 | req.Header.Set("X-Forwarded-Host", opts.ForwardedHost) 58 | req.Host = opts.ForwardedHost 59 | } 60 | 61 | } 62 | 63 | // Configure the proxy transport 64 | proxy.Transport = &http.Transport{ 65 | TLSClientConfig: opts.TLSConfig, 66 | } 67 | 68 | return func(w http.ResponseWriter, r *http.Request) { 69 | proxy.ServeHTTP(w, r) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /pkg/httputil/middleware/request_id.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | 7 | "github.com/edgeflare/pgo/pkg/httputil" 8 | "github.com/google/uuid" 9 | ) 10 | 11 | const RequestIDHeader = "X-Request-Id" 12 | 13 | // RequestID middleware generates a unique request ID and tracks request duration. 14 | func RequestID(next http.Handler) http.Handler { 15 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 16 | // Check if request ID is already set in the context 17 | reqID, ok := r.Context().Value(httputil.RequestIDCtxKey).(string) 18 | if !ok || reqID == "" { 19 | reqID = uuid.New().String() 20 | } 21 | 22 | ctx := r.Context() 23 | // Not sure whether storing the request ID in the context is useful 24 | // currently used by the logger middleware, but it can read from the request header set by this middleware 25 | ctx = context.WithValue(ctx, httputil.RequestIDCtxKey, reqID) 26 | w.Header().Set(RequestIDHeader, reqID) 27 | 28 | next.ServeHTTP(w, r.WithContext(ctx)) 29 | }) 30 | } 31 | -------------------------------------------------------------------------------- /pkg/httputil/middleware/request_id_test.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | 9 | "github.com/edgeflare/pgo/pkg/httputil" 10 | "github.com/google/uuid" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestRequestID(t *testing.T) { 15 | t.Run("should generate a new request ID if none exists", func(t *testing.T) { 16 | handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 17 | reqID := r.Context().Value(httputil.RequestIDCtxKey).(string) 18 | _, err := uuid.Parse(reqID) 19 | assert.NoError(t, err, "Request ID should be a valid UUID") 20 | }) 21 | 22 | reqIDMiddleware := RequestID(handler) 23 | 24 | req := httptest.NewRequest("GET", "http://example.com/foo", nil) 25 | w := httptest.NewRecorder() 26 | 27 | reqIDMiddleware.ServeHTTP(w, req) 28 | 29 | resp := w.Result() 30 | reqID := resp.Header.Get(RequestIDHeader) 31 | _, err := uuid.Parse(reqID) 32 | assert.NoError(t, err, "Response header X-Request-Id should be a valid UUID") 33 | }) 34 | 35 | t.Run("should preserve existing request ID", func(t *testing.T) { 36 | existingReqID := uuid.New().String() 37 | 38 | handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 39 | reqID := r.Context().Value(httputil.RequestIDCtxKey).(string) 40 | assert.Equal(t, existingReqID, reqID, "Request ID should match the existing ID") 41 | }) 42 | 43 | reqIDMiddleware := RequestID(handler) 44 | 45 | // Create a request with a pre-set context containing a request ID 46 | ctx := context.WithValue(context.Background(), httputil.RequestIDCtxKey, existingReqID) 47 | req := httptest.NewRequest("GET", "http://example.com/foo", nil).WithContext(ctx) 48 | w := httptest.NewRecorder() 49 | 50 | reqIDMiddleware.ServeHTTP(w, req) 51 | 52 | resp := w.Result() 53 | reqID := resp.Header.Get(RequestIDHeader) 54 | assert.Equal(t, existingReqID, reqID, "Response header X-Request-Id should match the existing ID") 55 | }) 56 | 57 | t.Run("should handle multiple requests independently", func(t *testing.T) { 58 | handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 59 | reqID := r.Context().Value(httputil.RequestIDCtxKey).(string) 60 | w.Write([]byte(reqID)) 61 | }) 62 | 63 | reqIDMiddleware := RequestID(handler) 64 | 65 | // Test first request 66 | req1 := httptest.NewRequest("GET", "http://example.com/foo1", nil) 67 | w1 := httptest.NewRecorder() 68 | reqIDMiddleware.ServeHTTP(w1, req1) 69 | _ = w1.Result() 70 | body1 := w1.Body.String() 71 | 72 | // Test second request 73 | req2 := httptest.NewRequest("GET", "http://example.com/foo2", nil) 74 | w2 := httptest.NewRecorder() 75 | reqIDMiddleware.ServeHTTP(w2, req2) 76 | _ = w2.Result() 77 | body2 := w2.Body.String() 78 | 79 | assert.NotEqual(t, body1, body2, "Request IDs should be different for different requests") 80 | 81 | // Additional validation for request IDs in responses 82 | _, err1 := uuid.Parse(body1) 83 | _, err2 := uuid.Parse(body2) 84 | assert.NoError(t, err1, "Response body for first request should be a valid UUID") 85 | assert.NoError(t, err2, "Response body for second request should be a valid UUID") 86 | }) 87 | } 88 | -------------------------------------------------------------------------------- /pkg/metrics/prom.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "cmp" 5 | "context" 6 | "log" 7 | "net/http" 8 | "sync" 9 | "time" 10 | 11 | "github.com/prometheus/client_golang/prometheus" 12 | "github.com/prometheus/client_golang/prometheus/promauto" 13 | "github.com/prometheus/client_golang/prometheus/promhttp" 14 | ) 15 | 16 | var ( 17 | TransformationErrors = promauto.NewCounterVec( 18 | prometheus.CounterOpts{ 19 | Name: "pgo_transformation_errors_total", 20 | Help: "Total number of transformation errors by type and pipeline", 21 | }, 22 | []string{"error_type", "pipeline", "source", "sink"}, 23 | ) 24 | 25 | PublishErrors = promauto.NewCounterVec( 26 | prometheus.CounterOpts{ 27 | Name: "pgo_publish_errors_total", 28 | Help: "Total number of publish errors by sink", 29 | }, 30 | []string{"sink"}, 31 | ) 32 | 33 | ProcessedEvents = promauto.NewCounterVec( 34 | prometheus.CounterOpts{ 35 | Name: "pgo_processed_events_total", 36 | Help: "Total number of processed events by pipeline", 37 | }, 38 | []string{"pipeline", "source", "sink"}, 39 | ) 40 | 41 | EventProcessingDuration = promauto.NewHistogramVec( 42 | prometheus.HistogramOpts{ 43 | Name: "pgo_event_processing_duration_seconds", 44 | Help: "Duration of event processing", 45 | Buckets: prometheus.DefBuckets, 46 | }, 47 | []string{"pipeline", "source", "sink"}, 48 | ) 49 | ) 50 | 51 | type PromServerOpts struct { 52 | Addr string 53 | Path string // Path for metrics endpoint, defaults to "/metrics" 54 | ShutdownTimeout time.Duration // Timeout for server shutdown, defaults to 5 seconds 55 | ReadHeaderTimeout time.Duration // Timeout for reading request headers, defaults to 3 seconds 56 | } 57 | 58 | func defaultPrometheusServerOptions() PromServerOpts { 59 | return PromServerOpts{ 60 | Addr: ":9100", 61 | Path: "/metrics", 62 | ShutdownTimeout: 5 * time.Second, 63 | ReadHeaderTimeout: 3 * time.Second, 64 | } 65 | } 66 | 67 | // StartPrometheusServer starts a Prometheus metrics server with the given options 68 | // The server gracefully shutdown when the provided context is canceled 69 | func StartPrometheusServer(ctx context.Context, wg *sync.WaitGroup, opts *PromServerOpts) { 70 | // merge with defaults 71 | effectiveOpts := defaultPrometheusServerOptions() 72 | if opts != nil { 73 | effectiveOpts.Addr = cmp.Or(opts.Addr, effectiveOpts.Addr) 74 | effectiveOpts.Path = cmp.Or(opts.Path, effectiveOpts.Path) 75 | effectiveOpts.ShutdownTimeout = cmp.Or(opts.ShutdownTimeout, effectiveOpts.ShutdownTimeout) 76 | effectiveOpts.ReadHeaderTimeout = cmp.Or(opts.ReadHeaderTimeout, effectiveOpts.ReadHeaderTimeout) 77 | } 78 | 79 | mux := http.NewServeMux() 80 | mux.Handle(effectiveOpts.Path, promhttp.Handler()) 81 | server := &http.Server{ 82 | Addr: effectiveOpts.Addr, 83 | Handler: mux, 84 | ReadHeaderTimeout: effectiveOpts.ReadHeaderTimeout, 85 | } 86 | 87 | serverClosed := make(chan struct{}) 88 | 89 | // Increment wait group 90 | wg.Add(1) 91 | 92 | // Start server 93 | go func() { 94 | defer wg.Done() 95 | log.Printf("Starting Prometheus metrics server on %s", effectiveOpts.Addr) 96 | if err := server.ListenAndServe(); err != http.ErrServerClosed { 97 | log.Printf("Metrics server error: %v", err) 98 | } 99 | close(serverClosed) 100 | }() 101 | 102 | // Monitor context cancellation in a separate goroutine 103 | go func() { 104 | <-ctx.Done() 105 | 106 | // Create a timeout context for shutdown 107 | shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), effectiveOpts.ShutdownTimeout) 108 | defer shutdownCancel() 109 | 110 | // Attempt graceful shutdown 111 | if err := server.Shutdown(shutdownCtx); err != nil { 112 | log.Printf("Error shutting down metrics server: %v", err) 113 | } 114 | 115 | // Wait for server to close or timeout 116 | select { 117 | case <-serverClosed: 118 | log.Println("Metrics server shutdown complete") 119 | case <-shutdownCtx.Done(): 120 | log.Println("Metrics server shutdown timed out") 121 | } 122 | }() 123 | } 124 | -------------------------------------------------------------------------------- /pkg/pglogrepl/pglogrepl.go: -------------------------------------------------------------------------------- 1 | // Package pglogrepl provides Debezium-compatible change data capture (CDC) events from PostgreSQL write-ahead logs (WAL). 2 | // It uses github.com/jackc/pglogrepl to read WAL, and then formats the changes into standardized CDC messages. 3 | package pglogrepl 4 | 5 | import ( 6 | "cmp" 7 | "fmt" 8 | "time" 9 | ) 10 | 11 | // Op represents a type of database operation to be replicated. 12 | type Op string 13 | 14 | const ( 15 | OpInsert Op = "insert" 16 | OpUpdate Op = "update" 17 | OpDelete Op = "delete" 18 | OpTruncate Op = "truncate" 19 | 20 | defaultStandbyUpdateInterval = 10 * time.Second 21 | defaultBufferSize = 1000 22 | defaultPublication = "pgo_pub" 23 | defaultSlot = "pgo_slot" 24 | defaultPlugin = "pgoutput" 25 | ) 26 | 27 | // Config holds replication configuration. 28 | type Config struct { 29 | Publication string `json:"publication"` 30 | Slot string `json:"slot"` 31 | Plugin string `json:"plugin"` 32 | // Tables to add to publication. Example: 33 | // ["table_wo_schema", "specific_schema.example_table", "another_schema.*"] 34 | // ["*"] or ["*.*"] for all tables in all schemas 35 | Tables []string `json:"tables"` 36 | Ops []Op `json:"ops"` 37 | PartitionRoot bool `json:"partitionRoot"` 38 | StandbyUpdateInterval time.Duration `json:"standbyUpdateInterval"` 39 | // ReplicaIdentity configures how much old row data is captured for each table. 40 | // not functional yet. manually execute sql to alter DEFAULT (streams primary key columns) 41 | ReplicaIdentity map[string]ReplicaIdentity `json:"relreplident"` 42 | BufferSize int `json:"bufferSize"` 43 | } 44 | 45 | // ReplicaIdentity specifies what row data Postgres streams during UPDATE/DELETE operations: 46 | // - Default (d): streams primary key columns 47 | // - None (n): streams no old row data 48 | // - Full (f): streams all columns 49 | // - Index (i): streams columns in specified index 50 | // 51 | // Set with: ALTER TABLE table_name REPLICA IDENTITY [DEFAULT|NOTHING|FULL|USING INDEX name] 52 | // 53 | // Query with: SELECT relreplident FROM pg_class WHERE oid = 'schema.table'::regclass; 54 | type ReplicaIdentity string 55 | 56 | const ( 57 | // ReplicaIdentityDefault streams only primary key columns 58 | ReplicaIdentityDefault ReplicaIdentity = "d" 59 | 60 | // ReplicaIdentityNothing streams no old row data 61 | ReplicaIdentityNothing ReplicaIdentity = "n" 62 | 63 | // ReplicaIdentityFull streams all columns of old rows 64 | ReplicaIdentityFull ReplicaIdentity = "f" 65 | 66 | // ReplicaIdentityIndex streams columns from a specified unique index 67 | ReplicaIdentityIndex ReplicaIdentity = "i" 68 | ) 69 | 70 | func DefaultConfig() *Config { 71 | return &Config{ 72 | Publication: defaultPublication, 73 | Slot: defaultSlot, 74 | Plugin: defaultPlugin, 75 | StandbyUpdateInterval: defaultStandbyUpdateInterval, 76 | Ops: []Op{OpInsert, OpUpdate, OpDelete, OpTruncate}, 77 | BufferSize: defaultBufferSize, 78 | } 79 | } 80 | 81 | func validateConfig(cfg *Config) error { 82 | if cfg == nil { 83 | return fmt.Errorf("config cannot be nil") 84 | } 85 | for _, op := range cfg.Ops { 86 | switch op { 87 | case OpInsert, OpUpdate, OpDelete, OpTruncate: 88 | default: 89 | return fmt.Errorf("invalid operation: %s", op) 90 | } 91 | } 92 | if cfg.StandbyUpdateInterval < time.Second { 93 | return fmt.Errorf("standby update interval must be at least 1 second") 94 | } 95 | return nil 96 | } 97 | 98 | func mergeWithDefaults(cfg *Config) *Config { 99 | def := DefaultConfig() 100 | if cfg == nil { 101 | return def 102 | } 103 | 104 | if len(cfg.Ops) == 0 { 105 | cfg.Ops = def.Ops 106 | } 107 | 108 | cfg.Publication = cmp.Or(cfg.Publication, def.Publication) 109 | cfg.Slot = cmp.Or(cfg.Slot, def.Slot) 110 | cfg.Plugin = cmp.Or(cfg.Plugin, def.Plugin) 111 | cfg.StandbyUpdateInterval = cmp.Or(cfg.StandbyUpdateInterval, def.StandbyUpdateInterval) 112 | cfg.BufferSize = cmp.Or(cfg.BufferSize, def.BufferSize) 113 | 114 | return cfg 115 | } 116 | -------------------------------------------------------------------------------- /pkg/pglogrepl/process_v1.go: -------------------------------------------------------------------------------- 1 | package pglogrepl 2 | 3 | // import ( 4 | // "log" 5 | 6 | // "github.com/edgeflare/pgo/pkg/pipeline/cdc" 7 | // "github.com/jackc/pglogrepl" 8 | // "github.com/jackc/pgx/v5/pgtype" 9 | // ) 10 | 11 | // // processV1 processes a logical replication message (wal2json) from PostgreSQL WAL data. 12 | // // Prefer pgoutput (which is default) 13 | // // TODO: improve if deemed useful 14 | // func processV1(walData []byte, relations map[uint32]*pglogrepl.RelationMessage, typeMap *pgtype.Map) ([]cdc.Event, error) { 15 | // logicalMsg, err := pglogrepl.Parse(walData) 16 | // if err != nil { 17 | // log.Fatalf("Parse logical replication message: %s", err) 18 | // } 19 | // log.Printf("Receive a logical replication message: %s", logicalMsg.Type()) 20 | // switch logicalMsg := logicalMsg.(type) { 21 | // case *pglogrepl.RelationMessage: 22 | // relations[logicalMsg.RelationID] = logicalMsg 23 | 24 | // case *pglogrepl.BeginMessage: 25 | // // Indicates the beginning of a group of changes in a transaction. This is only sent for committed transactions. 26 | 27 | // case *pglogrepl.CommitMessage: 28 | 29 | // case *pglogrepl.InsertMessage: 30 | // rel, ok := relations[logicalMsg.RelationID] 31 | // if !ok { 32 | // log.Fatalf("unknown relation ID %d", logicalMsg.RelationID) 33 | // } 34 | // values := map[string]interface{}{} 35 | // for idx, col := range logicalMsg.Tuple.Columns { 36 | // colName := rel.Columns[idx].Name 37 | // switch col.DataType { 38 | // case 'n': // null 39 | // values[colName] = nil 40 | // case 'u': // unchanged toast 41 | // // This TOAST value was not changed. TOAST values are not stored in the tuple 42 | // case 't': // text 43 | // val, err := decodeTextColumnData(typeMap, col.Data, rel.Columns[idx].DataType) 44 | // if err != nil { 45 | // log.Fatalln("error decoding column data:", err) 46 | // } 47 | // values[colName] = val 48 | // } 49 | // } 50 | // log.Printf("INSERT INTO %s.%s: %v", rel.Namespace, rel.RelationName, values) 51 | 52 | // case *pglogrepl.UpdateMessage: 53 | // // ... 54 | // case *pglogrepl.DeleteMessage: 55 | // // ... 56 | // case *pglogrepl.TruncateMessage: 57 | // // ... 58 | 59 | // case *pglogrepl.TypeMessage: 60 | // case *pglogrepl.OriginMessage: 61 | 62 | // case *pglogrepl.LogicalDecodingMessage: 63 | // log.Printf("Logical decoding message: %q, %q", logicalMsg.Prefix, logicalMsg.Content) 64 | 65 | // case *pglogrepl.StreamStartMessageV2: 66 | // log.Printf("Stream start message: xid %d, first segment? %d", logicalMsg.Xid, logicalMsg.FirstSegment) 67 | // case *pglogrepl.StreamStopMessageV2: 68 | // log.Printf("Stream stop message") 69 | // case *pglogrepl.StreamCommitMessageV2: 70 | // log.Printf("Stream commit message: xid %d", logicalMsg.Xid) 71 | // case *pglogrepl.StreamAbortMessageV2: 72 | // log.Printf("Stream abort message: xid %d", logicalMsg.Xid) 73 | // default: 74 | // log.Printf("Unknown message type in pgoutput stream: %T", logicalMsg) 75 | // } 76 | // return []cdc.Event{}, nil 77 | // } 78 | -------------------------------------------------------------------------------- /pkg/pglogrepl/util.go: -------------------------------------------------------------------------------- 1 | package pglogrepl 2 | 3 | import ( 4 | "github.com/jackc/pglogrepl" 5 | "github.com/jackc/pgx/v5/pgtype" 6 | "go.uber.org/zap" 7 | ) 8 | 9 | // decodeColumn decodes a single column from a PostgreSQL logical replication message. 10 | func decodeColumn(col *pglogrepl.TupleDataColumn, typeMap *pgtype.Map, dataType uint32) interface{} { 11 | switch col.DataType { 12 | case 'n': 13 | return nil 14 | case 'u': 15 | return nil // or some placeholder for unchanged toast 16 | case 't': 17 | val, err := decodeTextColumnData(typeMap, col.Data, dataType) 18 | if err != nil { 19 | zap.L().Error("error decoding column data", zap.Error(err)) 20 | return nil 21 | } 22 | return val 23 | default: 24 | zap.L().Warn("unknown column data type", zap.Any("dataType", col.DataType)) 25 | return nil 26 | } 27 | } 28 | 29 | // decodeTextColumnData decodes the binary data of a column into its corresponding Go type. 30 | // It uses the provided type map to determine the appropriate codec for decoding. 31 | func decodeTextColumnData(mi *pgtype.Map, data []byte, dataType uint32) (interface{}, error) { 32 | if dt, ok := mi.TypeForOID(dataType); ok { 33 | return dt.Codec.DecodeValue(mi, dataType, pgtype.TextFormatCode, data) 34 | } 35 | return string(data), nil 36 | } 37 | -------------------------------------------------------------------------------- /pkg/pgx/conn.go: -------------------------------------------------------------------------------- 1 | package pgx 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/jackc/pgx/v5" 7 | "github.com/jackc/pgx/v5/pgconn" 8 | ) 9 | 10 | // Conn defines a common interface for interacting with PostgreSQL connections. 11 | // This interface abstracts away the underlying connection type (e.g., pgx.Conn, 12 | // pgxpool.Conn) allowing for easier use within frameworks and libraries that 13 | // need to work with both single connections and connection pools. 14 | type Conn interface { 15 | // Exec executes a SQL statement in the context of the given context 'ctx'. 16 | // It returns a CommandTag containing details about the executed statement, 17 | // or an error if there was an issue during execution. 18 | Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) 19 | // Query executes a SQL query in the context of the given context 'ctx'. 20 | // It returns a Rows object that can be used to iterate over the results 21 | // of the query, or an error if there was an issue during execution. 22 | Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) 23 | // QueryRow executes a query that is expected to return at most one row. 24 | // It returns a Row object that can be used to retrieve the single row, 25 | // or an error if there was an issue during execution. 26 | QueryRow(ctx context.Context, sql string, args ...any) pgx.Row 27 | // Begin starts a transaction. Unlike database/sql, the context only affects the begin command. 28 | // i.e. there is no auto-rollback on context cancellation. 29 | Begin(ctx context.Context) (pgx.Tx, error) 30 | // BeginTx starts a transaction with txOptions determining the transaction mode. Unlike database/sql, 31 | // the context only affects the begin command. i.e. there is no auto-rollback on context cancellation. 32 | BeginTx(ctx context.Context, txOptions pgx.TxOptions) (pgx.Tx, error) 33 | } 34 | -------------------------------------------------------------------------------- /pkg/pgx/conn_test.go: -------------------------------------------------------------------------------- 1 | package pgx 2 | 3 | import ( 4 | "github.com/jackc/pgx/v5" 5 | "github.com/jackc/pgx/v5/pgxpool" 6 | ) 7 | 8 | // Compile-time interface compliance checks 9 | var ( 10 | _ Conn = (*pgx.Conn)(nil) 11 | _ Conn = (*pgxpool.Pool)(nil) 12 | ) 13 | -------------------------------------------------------------------------------- /pkg/pgx/crud.go: -------------------------------------------------------------------------------- 1 | package pgx 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "strings" 8 | 9 | "github.com/jackc/pgx/v5" 10 | ) 11 | 12 | type queryBuilder struct { 13 | schema string 14 | table string 15 | columns []string 16 | values []any 17 | nextIndex int 18 | } 19 | 20 | func newQueryBuilder(tableName string, schema ...string) *queryBuilder { 21 | schemaName := "public" 22 | if len(schema) > 0 && schema[0] != "" { 23 | schemaName = schema[0] 24 | } 25 | return &queryBuilder{ 26 | schema: schemaName, 27 | table: tableName, 28 | nextIndex: 1, 29 | } 30 | } 31 | 32 | func (qb *queryBuilder) addValue(column string, value any) { 33 | qb.columns = append(qb.columns, column) 34 | qb.values = append(qb.values, value) 35 | } 36 | 37 | func (qb *queryBuilder) placeholder() string { 38 | placeholder := fmt.Sprintf("$%d", qb.nextIndex) 39 | qb.nextIndex++ 40 | return placeholder 41 | } 42 | 43 | func (qb *queryBuilder) tableIdentifier() string { 44 | return pgx.Identifier{qb.schema, qb.table}.Sanitize() 45 | } 46 | 47 | // InsertRow inserts a new record into the specified table using the provided data. 48 | func InsertRow(ctx context.Context, conn Conn, tableName string, data any, schema ...string) error { 49 | qb := newQueryBuilder(tableName, schema...) 50 | 51 | dataMap, ok := data.(map[string]any) 52 | if !ok { 53 | return fmt.Errorf("data is not in expected format map[string]any") 54 | } 55 | 56 | var columns, placeholders []string 57 | for key, value := range dataMap { 58 | columns = append(columns, pgx.Identifier{key}.Sanitize()) 59 | placeholders = append(placeholders, qb.placeholder()) 60 | qb.addValue("", value) 61 | } 62 | 63 | query := fmt.Sprintf( 64 | "INSERT INTO %s (%s) VALUES (%s)", 65 | qb.tableIdentifier(), 66 | strings.Join(columns, ", "), 67 | strings.Join(placeholders, ", "), 68 | ) 69 | 70 | _, err := conn.Exec(ctx, query, qb.values...) 71 | if err != nil { 72 | return fmt.Errorf("failed to insert record: %w", err) 73 | } 74 | return nil 75 | } 76 | 77 | // UpdateRow updates an existing record in the specified table using the provided data. 78 | func UpdateRow(ctx context.Context, conn Conn, tableName string, data any, where map[string]any, schema ...string) error { 79 | qb := newQueryBuilder(tableName, schema...) 80 | 81 | var setClauses, whereClauses []string 82 | 83 | dataMap, ok := data.(map[string]any) 84 | if !ok { 85 | return fmt.Errorf("data is not in expected format map[string]any") 86 | } 87 | 88 | // Build SET clause 89 | for key, value := range dataMap { 90 | setClauses = append(setClauses, fmt.Sprintf("%s = %s", 91 | pgx.Identifier{key}.Sanitize(), 92 | qb.placeholder())) 93 | qb.addValue("", value) 94 | } 95 | 96 | // Build WHERE clause 97 | for key, value := range where { 98 | whereClauses = append(whereClauses, fmt.Sprintf("%s = %s", 99 | pgx.Identifier{key}.Sanitize(), 100 | qb.placeholder())) 101 | qb.addValue("", value) 102 | } 103 | 104 | if len(whereClauses) == 0 { 105 | return fmt.Errorf("no WHERE conditions provided") 106 | } 107 | 108 | query := fmt.Sprintf( 109 | "UPDATE %s SET %s WHERE %s", 110 | qb.tableIdentifier(), 111 | strings.Join(setClauses, ", "), 112 | strings.Join(whereClauses, " AND "), 113 | ) 114 | 115 | result, err := conn.Exec(ctx, query, qb.values...) 116 | if err != nil { 117 | return fmt.Errorf("failed to update record: %w", err) 118 | } 119 | 120 | if result.RowsAffected() == 0 { 121 | return fmt.Errorf("no rows were updated") 122 | } 123 | 124 | return nil 125 | } 126 | 127 | // convenience functions for JSON input 128 | func InsertRowJSON(ctx context.Context, conn Conn, tableName string, jsonData []byte, schema ...string) error { 129 | data, err := parseJSON(jsonData) 130 | if err != nil { 131 | return err 132 | } 133 | return InsertRow(ctx, conn, tableName, data, schema...) 134 | } 135 | 136 | func UpdateRowJSON(ctx context.Context, conn Conn, tableName string, jsonData []byte, where map[string]any, schema ...string) error { 137 | data, err := parseJSON(jsonData) 138 | if err != nil { 139 | return err 140 | } 141 | return UpdateRow(ctx, conn, tableName, data, where, schema...) 142 | } 143 | 144 | func parseJSON(data []byte) (map[string]any, error) { 145 | var result map[string]any 146 | if err := json.Unmarshal(data, &result); err != nil { 147 | return nil, fmt.Errorf("failed to parse JSON data: %w", err) 148 | } 149 | return result, nil 150 | } 151 | -------------------------------------------------------------------------------- /pkg/pgx/doc.go: -------------------------------------------------------------------------------- 1 | // Wrapper utils around github.com/jackc/pgx 2 | package pgx 3 | -------------------------------------------------------------------------------- /pkg/pgx/listen.go: -------------------------------------------------------------------------------- 1 | package pgx 2 | 3 | /* 4 | remove if not used 5 | import ( 6 | "context" 7 | "fmt" 8 | 9 | "github.com/jackc/pgx/v5" 10 | "github.com/jackc/pgx/v5/pgconn" 11 | "github.com/jackc/pgx/v5/pgxpool" 12 | ) 13 | 14 | // Listen listens (`LISTEN channel_name;`) on a given channel for notifications (triggered by `NOTIFY channel_name, 'payload_string';`) 15 | // and returns channels of notifications and errors. It runs in a goroutine and listens until the context is canceled. 16 | func Listen(ctx context.Context, conn Conn, channelName string) (<-chan *pgconn.Notification, <-chan error) { 17 | notifications := make(chan *pgconn.Notification) 18 | errors := make(chan error) 19 | 20 | var waitForNotification func(context.Context) (*pgconn.Notification, error) 21 | 22 | if poolConn, ok := conn.(*pgxpool.Conn); ok { 23 | waitForNotification = poolConn.Conn().WaitForNotification 24 | } else if pgxConn, ok := conn.(*pgx.Conn); ok { 25 | waitForNotification = pgxConn.WaitForNotification 26 | } else { 27 | errors <- fmt.Errorf("connection must be *pgxpool.Conn or *pgx.Conn") 28 | close(notifications) 29 | close(errors) 30 | return notifications, errors 31 | } 32 | 33 | if _, err := conn.Exec(ctx, "LISTEN "+channelName); err != nil { 34 | errors <- fmt.Errorf("error listening to channel: %w", err) 35 | close(notifications) 36 | close(errors) 37 | return notifications, errors 38 | } 39 | 40 | go func() { 41 | defer close(notifications) 42 | defer close(errors) 43 | 44 | for { 45 | select { 46 | case <-ctx.Done(): 47 | errors <- ctx.Err() 48 | return 49 | default: 50 | notification, err := waitForNotification(ctx) 51 | if err != nil { 52 | errors <- err 53 | continue 54 | } 55 | if notification != nil { 56 | notifications <- notification 57 | } 58 | } 59 | } 60 | }() 61 | 62 | return notifications, errors 63 | } 64 | */ 65 | -------------------------------------------------------------------------------- /pkg/pgx/listen_test.go: -------------------------------------------------------------------------------- 1 | package pgx 2 | 3 | /* 4 | import ( 5 | "context" 6 | "testing" 7 | "time" 8 | 9 | "github.com/edgeflare/pgo/internal/testutil/pgtest" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestListen(t *testing.T) { 14 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 15 | defer cancel() 16 | 17 | listenConn := pgtest.Connect(ctx, t) 18 | defer pgtest.Close(t, listenConn) 19 | 20 | notifyConn := pgtest.Connect(ctx, t) 21 | defer pgtest.Close(t, notifyConn) 22 | 23 | channelName := "test_channel" 24 | 25 | // Start listening 26 | notifications, errors := Listen(ctx, listenConn, channelName) 27 | 28 | // Send notification after a short delay 29 | go func() { 30 | time.Sleep(100 * time.Millisecond) // Ensure listener is ready 31 | _, err := notifyConn.Exec(ctx, "NOTIFY "+channelName+", 'test_message'") 32 | require.NoError(t, err) 33 | }() 34 | 35 | select { 36 | case notification := <-notifications: 37 | require.NotNil(t, notification) 38 | require.Equal(t, "test_message", notification.Payload) 39 | case err := <-errors: 40 | t.Fatalf("Unexpected error: %v", err) 41 | case <-time.After(1 * time.Second): 42 | t.Fatal("Timeout waiting for notification") 43 | } 44 | } 45 | */ 46 | -------------------------------------------------------------------------------- /pkg/pgx/schema/schema_test.go: -------------------------------------------------------------------------------- 1 | package schema 2 | 3 | import ( 4 | "cmp" 5 | "context" 6 | "os" 7 | "testing" 8 | "time" 9 | 10 | "github.com/jackc/pgx/v5/pgxpool" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func TestSchemaWatch(t *testing.T) { 15 | ctx := context.Background() 16 | connString := cmp.Or(os.Getenv("TEST_DATABASE"), "postgres://postgres:secret@localhost:5432/testdb") 17 | 18 | // Connect to create a test table 19 | pool, err := pgxpool.New(ctx, connString) 20 | require.NoError(t, err) 21 | defer pool.Close() 22 | 23 | // Create a test table to trigger a schema change 24 | _, err = pool.Exec(ctx, ` 25 | CREATE TABLE IF NOT EXISTS test_watch ( 26 | id SERIAL PRIMARY KEY, 27 | name TEXT 28 | ) 29 | `) 30 | require.NoError(t, err) 31 | defer pool.Exec(ctx, "DROP TABLE IF EXISTS test_watch") 32 | 33 | // Initialize cache 34 | cache, err := NewCache(connString) 35 | require.NoError(t, err) 36 | defer cache.Close() 37 | 38 | err = cache.Init(ctx) 39 | require.NoError(t, err) 40 | 41 | // Create a done channel to stop watching after our test 42 | done := make(chan bool) 43 | go func() { 44 | // Watch for changes 45 | for tables := range cache.Watch() { 46 | // Verify we get table data 47 | require.NotEmpty(t, tables) 48 | done <- true 49 | return 50 | } 51 | }() 52 | 53 | // Notify schema change 54 | _, err = pool.Exec(ctx, "NOTIFY "+reloadChannel+", '"+reloadPayload+"'") 55 | require.NoError(t, err) 56 | 57 | // Wait for the watch to pick up changes 58 | select { 59 | case <-done: 60 | // Test passed 61 | case <-time.After(5 * time.Second): 62 | t.Fatal("timeout waiting for schema change notification") 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /pkg/pipeline/cdc/cdc.go: -------------------------------------------------------------------------------- 1 | package cdc 2 | 3 | // Operation represents the type of change that occurred 4 | type Operation string 5 | 6 | const ( 7 | OpCreate Operation = "c" 8 | OpUpdate Operation = "u" 9 | OpDelete Operation = "d" 10 | OpRead Operation = "r" 11 | OpTruncate Operation = "t" 12 | ) 13 | 14 | // Source contains metadata about where a change originated 15 | type Source struct { 16 | Version string `json:"version"` 17 | Connector string `json:"connector"` 18 | Name string `json:"name"` 19 | TsMs int64 `json:"ts_ms"` 20 | Snapshot bool `json:"snapshot"` 21 | Db string `json:"db"` 22 | Sequence string `json:"sequence"` 23 | Schema string `json:"schema"` 24 | Table string `json:"table"` 25 | TxID int64 `json:"txId"` 26 | Lsn int64 `json:"lsn"` 27 | Xmin *int64 `json:"xmin,omitempty"` 28 | } 29 | 30 | // Transaction contains metadata about the transaction this change belongs to 31 | type Transaction struct { 32 | ID string `json:"id"` 33 | TotalOrder int64 `json:"total_order"` 34 | DataCollectionOrder int64 `json:"data_collection_order"` 35 | } 36 | 37 | // Payload represents the actual change data 38 | type Payload struct { 39 | Before interface{} `json:"before"` 40 | After interface{} `json:"after"` 41 | Source Source `json:"source"` 42 | Op Operation `json:"op"` 43 | TsMs int64 `json:"ts_ms"` 44 | Transaction *Transaction `json:"transaction,omitempty"` 45 | } 46 | 47 | // Field represents a schema field definition 48 | type Field struct { 49 | Field string `json:"field"` 50 | Type string `json:"type"` 51 | Optional bool `json:"optional"` 52 | Name string `json:"name,omitempty"` 53 | Fields []Field `json:"fields,omitempty"` 54 | } 55 | 56 | // Schema represents the schema definition for a change event 57 | type Schema struct { 58 | Type string `json:"type"` 59 | Optional bool `json:"optional"` 60 | Name string `json:"name"` 61 | Fields []Field `json:"fields"` 62 | } 63 | 64 | // Event represents a complete change data capture event 65 | type Event struct { 66 | Schema Schema `json:"schema"` 67 | Payload Payload `json:"payload"` 68 | } 69 | 70 | // SourceBuilder helps construct Source objects with reasonable defaults 71 | type SourceBuilder struct { 72 | source Source 73 | } 74 | 75 | func NewSourceBuilder(connector, name string) *SourceBuilder { 76 | return &SourceBuilder{ 77 | source: Source{ 78 | Version: "1.0", 79 | Connector: connector, 80 | Name: name, 81 | Snapshot: false, 82 | Sequence: "[0,0]", 83 | }, 84 | } 85 | } 86 | 87 | func (b *SourceBuilder) WithSchema(schema string) *SourceBuilder { 88 | b.source.Schema = schema 89 | return b 90 | } 91 | 92 | func (b *SourceBuilder) WithTable(table string) *SourceBuilder { 93 | b.source.Table = table 94 | return b 95 | } 96 | 97 | func (b *SourceBuilder) WithDatabase(db string) *SourceBuilder { 98 | b.source.Db = db 99 | return b 100 | } 101 | 102 | func (b *SourceBuilder) WithTimestamp(ts int64) *SourceBuilder { 103 | b.source.TsMs = ts 104 | return b 105 | } 106 | 107 | func (b *SourceBuilder) WithTransaction(txID int64, lsn int64) *SourceBuilder { 108 | b.source.TxID = txID 109 | b.source.Lsn = lsn 110 | return b 111 | } 112 | 113 | func (b *SourceBuilder) Build() Source { 114 | return b.source 115 | } 116 | 117 | // EventBuilder helps construct complete CDC events 118 | type EventBuilder struct { 119 | event Event 120 | } 121 | 122 | func NewEventBuilder() *EventBuilder { 123 | return &EventBuilder{ 124 | event: Event{ 125 | Schema: Schema{ 126 | Type: "struct", 127 | Optional: false, 128 | }, 129 | }, 130 | } 131 | } 132 | 133 | func (b *EventBuilder) WithSource(source Source) *EventBuilder { 134 | b.event.Payload.Source = source 135 | return b 136 | } 137 | 138 | func (b *EventBuilder) WithOperation(op Operation) *EventBuilder { 139 | b.event.Payload.Op = op 140 | return b 141 | } 142 | 143 | func (b *EventBuilder) WithBefore(before interface{}) *EventBuilder { 144 | b.event.Payload.Before = before 145 | return b 146 | } 147 | 148 | func (b *EventBuilder) WithAfter(after interface{}) *EventBuilder { 149 | b.event.Payload.After = after 150 | return b 151 | } 152 | 153 | func (b *EventBuilder) WithTimestamp(ts int64) *EventBuilder { 154 | b.event.Payload.TsMs = ts 155 | return b 156 | } 157 | 158 | func (b *EventBuilder) WithTransaction(tx *Transaction) *EventBuilder { 159 | b.event.Payload.Transaction = tx 160 | return b 161 | } 162 | 163 | func (b *EventBuilder) Build() Event { 164 | return b.event 165 | } 166 | -------------------------------------------------------------------------------- /pkg/pipeline/cdc/cdc_test.go: -------------------------------------------------------------------------------- 1 | package cdc 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/edgeflare/pgo/internal/testutil" 7 | ) 8 | 9 | // TestDebeziumConformanceCDC tests the conformance of the CDC struct to the Debezium CDC format. 10 | func TestDebeziumConformanceCDC(t *testing.T) { 11 | var event Event 12 | _, err := testutil.LoadJSON("cdc.json", &event) 13 | if err != nil { 14 | t.Fatalf("Failed to load test data: %v", err) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /pkg/pipeline/connector.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | 7 | "github.com/edgeflare/pgo/pkg/pipeline/cdc" 8 | ) 9 | 10 | type ConnectorType int 11 | 12 | const ( 13 | ConnectorTypeUnknown ConnectorType = iota 14 | ConnectorTypePub // Sink / consumer-only 15 | ConnectorTypeSub // Source / producer-only 16 | ConnectorTypePubSub // Source and sink 17 | ) 18 | 19 | var ( 20 | ErrConnectorTypeMismatch = errors.New("connector type mismatch") 21 | ) 22 | 23 | // A Connector represents a data pipeline component. 24 | type Connector interface { 25 | // Connect initializes the connector with the provided configuration. 26 | // The config parameter is a raw JSON message containing connector-specific settings. 27 | // Additional arguments can be passed via the args parameter. 28 | Connect(config json.RawMessage, args ...any) error 29 | 30 | // Pub sends the given CDC event to the connector's destination. 31 | // It returns an error if the publish operation fails. 32 | Pub(event cdc.Event, args ...any) error 33 | 34 | // Sub provides a channel for consumingCDC events. 35 | Sub(args ...any) (<-chan cdc.Event, error) 36 | 37 | // Type returns the type of the connector (SUB, PUB, or PUBSUB) 38 | Type() ConnectorType 39 | 40 | Disconnect() error 41 | } 42 | 43 | // Predefined connectors 44 | const ( 45 | ConnectorClickHouse = "clickhouse" 46 | ConnectorDebug = "debug" 47 | ConnectorHTTP = "http" 48 | ConnectorKafka = "kafka" 49 | ConnectorMQTT = "mqtt" 50 | ConnectorNATS = "nats" 51 | ConnectorGRPC = "grpc" 52 | ConnectorPostgres = "postgres" 53 | ) 54 | 55 | // RegisterConnector adds a new connector to the registry. 56 | // The name parameter is used as a key to identify the connector type. 57 | func RegisterConnector(name string, c Connector) { 58 | connectors[name] = c 59 | } 60 | -------------------------------------------------------------------------------- /pkg/pipeline/connector_test.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/edgeflare/pgo/pkg/pipeline/cdc" 8 | ) 9 | 10 | func TestNewManager(t *testing.T) { 11 | // Create a new manager 12 | manager := NewManager() 13 | 14 | // Test connectors 15 | t.Run("Test Connectors", func(t *testing.T) { 16 | for _, c := range connectors { 17 | t.Run(fmt.Sprintf("Connector: %T", c), func(t *testing.T) { 18 | if err := c.Connect(nil); err != nil { 19 | t.Errorf("Failed to initialize connector: %v", err) 20 | } 21 | 22 | msg := cdc.Event{ 23 | // TODO PostgresCDC --> CDC 24 | // Table: "test", 25 | // Data: map[string]interface{}{"hello": "world"}, 26 | // Operation: logrepl.OperationInsert, 27 | } 28 | if err := c.Pub(msg); err != nil { 29 | t.Errorf("Failed to publish message: %v", err) 30 | } 31 | }) 32 | } 33 | }) 34 | 35 | // Test plugin registration 36 | t.Run("Register Plugin", func(t *testing.T) { 37 | // go build -buildmode=plugin -o /tmp/example-plugin.so ./pkg/pipeline/peer/plugin_example/... 38 | err := manager.RegisterConnectorPlugin("/tmp/example-plugin.so", "example") 39 | if err != nil { 40 | t.Fatalf("Failed to load plugin: %v", err) 41 | } 42 | }) 43 | } 44 | -------------------------------------------------------------------------------- /pkg/pipeline/doc.go: -------------------------------------------------------------------------------- 1 | // Package pipeline provides a framework for managing data pipelines 2 | // from/to PostgreSQL to/from various `Peer`s (ie data source/destination). 3 | // 4 | // Supported peer types include ClickHouse, HTTP endpoints, Kafka, 5 | // MQTT, and gRPC, with extensibility through Go plugins. 6 | // 7 | // It defines a `Connector` interface that all `Peer` types must implement. 8 | package pipeline 9 | -------------------------------------------------------------------------------- /pkg/pipeline/peer.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | // Peer is a data source/destination with an associated connector (ie NATS, Kafka, MQTT, ClickHouse, etc). 4 | type Peer struct { 5 | Name string `mapstructure:"name"` 6 | ConnectorName string `mapstructure:"connector"` 7 | // Config contains the connection config of underlying library 8 | // eg github.com/IBM/sarama.Config, github.com/eclipse/paho.mqtt.golang.ClientOptions etc 9 | Config map[string]any `mapstructure:"config"` 10 | // Extra arguments for Connect, Pub, Sub methods 11 | Args []any 12 | } 13 | 14 | func (p *Peer) Connector() Connector { 15 | return connectors[p.ConnectorName] 16 | } 17 | 18 | // // need to check back for plugins / extensions 19 | // func (p *Peer) Name() string { 20 | // return p.name 21 | // } 22 | -------------------------------------------------------------------------------- /pkg/pipeline/peer/clickhouse/peer.go: -------------------------------------------------------------------------------- 1 | package clickhouse 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | 9 | "github.com/ClickHouse/clickhouse-go/v2" 10 | "github.com/ClickHouse/clickhouse-go/v2/lib/driver" 11 | "github.com/edgeflare/pgo/pkg/pipeline" 12 | "github.com/edgeflare/pgo/pkg/pipeline/cdc" 13 | "github.com/edgeflare/pgo/pkg/util" 14 | ) 15 | 16 | type PeerClickHouse struct { 17 | conn driver.Conn 18 | config *clickhouse.Options 19 | } 20 | 21 | func (p *PeerClickHouse) Connect(config json.RawMessage, args ...any) error { 22 | p.config = &clickhouse.Options{} 23 | 24 | if config != nil { 25 | if err := json.Unmarshal(config, p.config); err != nil { 26 | return fmt.Errorf("failed to parse ClickHouse config: %w", err) 27 | } 28 | } 29 | 30 | // Set values from environment variables or use defaults 31 | if len(p.config.Addr) == 0 { 32 | p.config.Addr = []string{util.GetEnvOrDefault("PGO_CLICKHOUSE_ADDR", "localhost:9000")} 33 | } 34 | if p.config.Auth.Database == "" { 35 | p.config.Auth.Database = util.GetEnvOrDefault("PGO_CLICKHOUSE_AUTH_DATABASE", "default") 36 | } 37 | if p.config.Auth.Username == "" { 38 | p.config.Auth.Username = util.GetEnvOrDefault("PGO_CLICKHOUSE_AUTH_USERNAME", "default") 39 | } 40 | if p.config.Auth.Password == "" { 41 | p.config.Auth.Password = util.GetEnvOrDefault("PGO_CLICKHOUSE_AUTH_PASSWORD", "") 42 | } 43 | 44 | // Create a new ClickHouse connection 45 | conn, err := clickhouse.Open(p.config) 46 | if err != nil { 47 | return fmt.Errorf("failed to connect to ClickHouse: %w", err) 48 | } 49 | 50 | // Test the connection 51 | if err := conn.Ping(context.Background()); err != nil { 52 | return fmt.Errorf("failed to ping ClickHouse: %w", err) 53 | } 54 | 55 | p.conn = conn 56 | return nil 57 | } 58 | 59 | func (p *PeerClickHouse) Pub(event cdc.Event, args ...any) error { 60 | // TODO: FIX 61 | // sql := fmt.Sprintf(` 62 | // INSERT INTO %s.%s ( 63 | // operation, 64 | // table_name, 65 | // schema_name, 66 | // timestamp, 67 | // data 68 | // ) VALUES (?, ?, ?, ?, ?) 69 | // `, p.config.Auth.Database, event.Table) 70 | 71 | // // Convert the data to JSON 72 | // dataJSON, err := json.Marshal(event.Data) 73 | // if err != nil { 74 | // return fmt.Errorf("failed to marshal event data: %w", err) 75 | // } 76 | 77 | // // Execute the INSERT statement 78 | // err = p.conn.Exec(context.Background(), sql, 79 | // event.Operation, 80 | // event.Table, 81 | // event.Schema, 82 | // event.Timestamp, 83 | // dataJSON, 84 | // ) 85 | // if err != nil { 86 | // return fmt.Errorf("failed to insert data into ClickHouse: %w", err) 87 | // } 88 | 89 | log.Printf("%v: Published event for table %v.%v", pipeline.ConnectorClickHouse, event.Schema, event.Payload.Source.Table) 90 | return nil 91 | } 92 | 93 | func (p *PeerClickHouse) Sub(args ...any) (<-chan cdc.Event, error) { 94 | // TODO: Implement Sub 95 | return nil, pipeline.ErrConnectorTypeMismatch 96 | } 97 | 98 | func (p *PeerClickHouse) Type() pipeline.ConnectorType { 99 | return pipeline.ConnectorTypePub 100 | } 101 | 102 | func (p *PeerClickHouse) Disconnect() error { 103 | if p.conn != nil { 104 | return p.conn.Close() 105 | } 106 | return nil 107 | } 108 | 109 | func init() { 110 | pipeline.RegisterConnector(pipeline.ConnectorClickHouse, &PeerClickHouse{}) 111 | } 112 | -------------------------------------------------------------------------------- /pkg/pipeline/peer/debug/peer.go: -------------------------------------------------------------------------------- 1 | package debug 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | 7 | "github.com/edgeflare/pgo/pkg/pipeline" 8 | "github.com/edgeflare/pgo/pkg/pipeline/cdc" 9 | ) 10 | 11 | // PeerDebug is a debug peer that logs the data to the console 12 | type PeerDebug struct{} 13 | 14 | func (p *PeerDebug) Pub(event cdc.Event, _ ...any) error { 15 | // TODO: should take a log formatting arg 16 | log.Printf("%s %+v", pipeline.ConnectorDebug, event) 17 | return nil 18 | } 19 | 20 | func (p *PeerDebug) Connect(_ json.RawMessage, _ ...any) error { 21 | return nil 22 | } 23 | 24 | func (p *PeerDebug) Sub(_ ...any) (<-chan cdc.Event, error) { 25 | return nil, pipeline.ErrConnectorTypeMismatch 26 | } 27 | 28 | func (p *PeerDebug) Type() pipeline.ConnectorType { 29 | return pipeline.ConnectorTypePub 30 | } 31 | 32 | func (p *PeerDebug) Disconnect() error { 33 | return nil 34 | } 35 | 36 | func init() { 37 | pipeline.RegisterConnector(pipeline.ConnectorDebug, &PeerDebug{}) 38 | } 39 | -------------------------------------------------------------------------------- /pkg/pipeline/peer/kafka/acl.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/IBM/sarama" 7 | "go.uber.org/zap" 8 | ) 9 | 10 | // Client handles produce, consume and ACL-related operations 11 | type Client struct { 12 | config *Config 13 | logger *zap.Logger 14 | } 15 | 16 | // NewClient creates a new ACLManager 17 | func NewClient(config *Config, logger *zap.Logger) *Client { 18 | return &Client{ 19 | config: config, 20 | logger: logger, 21 | } 22 | } 23 | 24 | // newClusterAdmin creates a new sarama.ClusterAdmin 25 | func (c *Client) newClusterAdmin() (sarama.ClusterAdmin, error) { 26 | saramaConfig, err := c.config.ToSaramaConfig() 27 | if err != nil { 28 | return nil, fmt.Errorf("failed to create sarama config: %w", err) 29 | } 30 | 31 | admin, err := sarama.NewClusterAdmin(c.config.GetBrokers(), saramaConfig) 32 | if err != nil { 33 | return nil, fmt.Errorf("failed to create cluster admin: %w", err) 34 | } 35 | 36 | return admin, nil 37 | } 38 | 39 | // CreateACL creates a new ACL 40 | func (c *Client) CreateACL(resource sarama.Resource, acl sarama.Acl) error { 41 | admin, err := c.newClusterAdmin() 42 | if err != nil { 43 | return fmt.Errorf("failed to create cluster admin: %w", err) 44 | } 45 | defer admin.Close() 46 | 47 | err = admin.CreateACL(resource, acl) 48 | if err != nil { 49 | return fmt.Errorf("failed to create ACL: %w", err) 50 | } 51 | 52 | c.logger.Info("ACL added", 53 | zap.Any("resource", resource), 54 | zap.Any("acl", acl)) 55 | return nil 56 | } 57 | 58 | // ListAcls lists ACLs based on the provided filter 59 | func (c *Client) ListAcls(filter sarama.AclFilter) ([]sarama.ResourceAcls, error) { 60 | admin, err := c.newClusterAdmin() 61 | if err != nil { 62 | return nil, fmt.Errorf("failed to create cluster admin: %w", err) 63 | } 64 | defer admin.Close() 65 | 66 | acls, err := admin.ListAcls(filter) 67 | if err != nil { 68 | return nil, fmt.Errorf("failed to list ACLs: %w", err) 69 | } 70 | 71 | for _, acl := range acls { 72 | c.logger.Info("ACL found", 73 | zap.Any("resource", acl.Resource), 74 | zap.Any("acls", acl.Acls)) 75 | } 76 | 77 | return acls, nil 78 | } 79 | 80 | // DeleteACL deletes ACLs based on the provided filter 81 | func (c *Client) DeleteACL(filter sarama.AclFilter) ([]sarama.MatchingAcl, error) { 82 | admin, err := c.newClusterAdmin() 83 | if err != nil { 84 | return nil, fmt.Errorf("failed to create cluster admin: %w", err) 85 | } 86 | defer admin.Close() 87 | 88 | matchingACLs, err := admin.DeleteACL(filter, false) 89 | if err != nil { 90 | return nil, fmt.Errorf("failed to delete ACL: %w", err) 91 | } 92 | 93 | for _, matchingACL := range matchingACLs { 94 | c.logger.Info("Deleted ACL", 95 | zap.Any("matchingACL", matchingACL)) 96 | } 97 | 98 | return matchingACLs, nil 99 | } 100 | -------------------------------------------------------------------------------- /pkg/pipeline/peer/kafka/config.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "fmt" 7 | "os" 8 | 9 | "github.com/IBM/sarama" 10 | ) 11 | 12 | // Config represents Kafka-specific configuration 13 | type Config struct { 14 | Brokers []string `json:"brokers"` 15 | TopicPrefix string `json:"topicPrefix"` 16 | Version string `json:"version,omitempty"` 17 | SASL *SASL `json:"sasl,omitempty"` 18 | Partitions int32 `json:"partitions,omitempty"` 19 | Replicas int16 `json:"replicas,omitempty"` 20 | RetentionMS int64 `json:"retentionMs,omitempty"` 21 | TLS TLS 22 | } 23 | 24 | // SASL represents SASL authentication configuration 25 | type SASL struct { 26 | Username string 27 | Password string 28 | Algorithm string 29 | Enable bool 30 | } 31 | 32 | // TLS represents TLS configuration 33 | type TLS struct { 34 | CertFile string 35 | KeyFile string 36 | CAFile string 37 | Enable bool 38 | SkipVerify bool 39 | } 40 | 41 | // NewConfig creates a new Kafka configuration with default values 42 | // func NewConfig() *Config { 43 | // return &Config{ 44 | // Version: sarama.DefaultVersion.String(), 45 | // SASL: SASLConfig{ 46 | // Algorithm: "sha512", 47 | // }, 48 | // } 49 | // } 50 | 51 | // ToSaramaConfig converts the Config to a sarama.Config 52 | func (c *Config) ToSaramaConfig() (*sarama.Config, error) { 53 | conf := sarama.NewConfig() 54 | 55 | // Set Kafka version 56 | version, err := sarama.ParseKafkaVersion(c.Version) 57 | if err != nil { 58 | return nil, fmt.Errorf("error parsing Kafka version: %w", err) 59 | } 60 | conf.Version = version 61 | 62 | // Configure SASL 63 | if c.SASL.Enable { 64 | conf.Net.SASL.Enable = true 65 | conf.Net.SASL.User = c.SASL.Username 66 | conf.Net.SASL.Password = c.SASL.Password 67 | conf.Net.SASL.Handshake = true 68 | 69 | switch c.SASL.Algorithm { 70 | case "sha512": 71 | conf.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: SHA512} } 72 | conf.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA512 73 | case "sha256": 74 | conf.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: SHA256} } 75 | conf.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA256 76 | default: 77 | return nil, fmt.Errorf("invalid SASL algorithm: %s", c.SASL.Algorithm) 78 | } 79 | } 80 | 81 | // Configure TLS 82 | if c.TLS.Enable { 83 | conf.Net.TLS.Enable = true 84 | conf.Net.TLS.Config = createTLSConfiguration(c.TLS) 85 | } 86 | 87 | // Set other default configurations 88 | conf.Producer.Retry.Max = 1 89 | conf.Producer.RequiredAcks = sarama.WaitForAll 90 | conf.Producer.Return.Successes = true 91 | conf.ClientID = "sasl_scram_client" 92 | conf.Metadata.Full = true 93 | 94 | return conf, nil 95 | } 96 | 97 | func createTLSConfiguration(tlsCfg TLS) *tls.Config { 98 | t := &tls.Config{ 99 | InsecureSkipVerify: tlsCfg.SkipVerify, 100 | } 101 | 102 | if tlsCfg.CertFile != "" && tlsCfg.KeyFile != "" && tlsCfg.CAFile != "" { 103 | cert, err := tls.LoadX509KeyPair(tlsCfg.CertFile, tlsCfg.KeyFile) 104 | if err != nil { 105 | return nil 106 | } 107 | 108 | caCert, err := os.ReadFile(tlsCfg.CAFile) 109 | if err != nil { 110 | return nil 111 | } 112 | 113 | caCertPool := x509.NewCertPool() 114 | caCertPool.AppendCertsFromPEM(caCert) 115 | 116 | t.Certificates = []tls.Certificate{cert} 117 | t.RootCAs = caCertPool 118 | } 119 | 120 | return t 121 | } 122 | 123 | // GetBrokers returns the list of Kafka brokers 124 | func (c *Config) GetBrokers() []string { 125 | return c.Brokers 126 | } 127 | -------------------------------------------------------------------------------- /pkg/pipeline/peer/kafka/doc.go: -------------------------------------------------------------------------------- 1 | // Package kafka provides a real-time Kafka-based interface for PostgreSQL, 2 | // similar to how PostgREST exposes PostgreSQL over HTTP. 3 | // 4 | // Kafka topic naming conventions: 5 | // - Case-sensitive, no spaces 6 | // - Valid chars: alphanumeric, `.`, `-`, `_` 7 | // - Recommended max length: 249 bytes (to avoid potential issues) 8 | // - Forward slash (`/`) can be used for logical separation but requires proper escaping 9 | // 10 | // pgo uses `[prefix].[schema_name].[table_name].[operation]` topic pattern to interact with PostgreSQL 11 | // 12 | // Operations: 13 | // - create (or c): Insert operations 14 | // - update (or u): Update operations 15 | // - delete (or d): Delete operations 16 | // - read (or r): Query operations 17 | // - truncate (or t): Truncate operations 18 | // 19 | // Examples: 20 | // - public.users.c → Create user 21 | // - inventory.products.u → Update product 22 | // - accounting.invoices.r → Read invoices 23 | // 24 | // Payload: JSON 25 | // 26 | // Message Format: 27 | // - Key: Unique identifier (e.g., primary key) 28 | // - Value: JSON payload 29 | // - Headers: Metadata including timestamp, operation type, etc. 30 | // 31 | // Partitioning Strategy: 32 | // - Default: Hash partitioning based on primary key 33 | // - Custom: Can be configured based on specific fields 34 | // 35 | // Query Parameters: 36 | // [schema_name].[table_name].read.[field].[value] 37 | // Example: public.users.r.id.123 → read by id 123 38 | // 39 | // Consumer Groups: 40 | // - Use meaningful names: `[app_name].[purpose]` 41 | // - Example: myapp.user_updates 42 | // 43 | // Configuration: 44 | // - Replication Factor: Minimum 2 recommended for production 45 | // - Number of Partitions: Based on throughput requirements 46 | // - Retention: Configurable per topic 47 | // 48 | // Use 'prefix.pg' topic for system operations 49 | // 50 | // Note: Ensure proper ACLs are configured for topic access 51 | package kafka 52 | -------------------------------------------------------------------------------- /pkg/pipeline/peer/kafka/kafka.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/signal" 7 | 8 | "github.com/IBM/sarama" 9 | "go.uber.org/zap" 10 | ) 11 | 12 | // CreateProducer creates a new SyncProducer 13 | func (c *Client) CreateProducer() (sarama.SyncProducer, error) { 14 | conf, err := c.config.ToSaramaConfig() 15 | if err != nil { 16 | return nil, fmt.Errorf("failed to create sarama config: %w", err) 17 | } 18 | 19 | producer, err := sarama.NewSyncProducer(c.config.GetBrokers(), conf) 20 | if err != nil { 21 | return nil, fmt.Errorf("failed to create sync producer: %w", err) 22 | } 23 | 24 | return producer, nil 25 | } 26 | 27 | // CreateConsumer creates a new Consumer 28 | func (c *Client) CreateConsumer() (sarama.Consumer, error) { 29 | conf, err := c.config.ToSaramaConfig() 30 | if err != nil { 31 | return nil, fmt.Errorf("failed to create sarama config: %w", err) 32 | } 33 | 34 | consumer, err := sarama.NewConsumer(c.config.GetBrokers(), conf) 35 | if err != nil { 36 | return nil, fmt.Errorf("failed to create consumer: %w", err) 37 | } 38 | 39 | return consumer, nil 40 | } 41 | 42 | // ConsumeMessages consumes messages from a topic 43 | func (c *Client) ConsumeMessages(consumer sarama.Consumer, topic string, logMsg bool) { 44 | partitionConsumer, err := consumer.ConsumePartition(topic, 0, sarama.OffsetOldest) 45 | if err != nil { 46 | c.logger.Fatal("Failed to start consumer", zap.Error(err)) 47 | } 48 | defer func() { 49 | if err := partitionConsumer.Close(); err != nil { 50 | c.logger.Fatal("Failed to close partition consumer", zap.Error(err)) 51 | } 52 | }() 53 | 54 | signals := make(chan os.Signal, 1) 55 | signal.Notify(signals, os.Interrupt) 56 | 57 | consumed := 0 58 | ConsumerLoop: 59 | for { 60 | select { 61 | case msg := <-partitionConsumer.Messages(): 62 | c.logger.Info("Consumed message", 63 | zap.Int64("offset", msg.Offset), 64 | zap.String("topic", msg.Topic), 65 | zap.Int32("partition", msg.Partition)) 66 | if logMsg { 67 | c.logger.Info("Message content", 68 | zap.ByteString("key", msg.Key), 69 | zap.ByteString("value", msg.Value)) 70 | } 71 | consumed++ 72 | case <-signals: 73 | break ConsumerLoop 74 | } 75 | } 76 | 77 | c.logger.Info("Consumption finished", zap.Int("consumed", consumed)) 78 | } 79 | 80 | // ProduceMessage produces a message to a topic 81 | func (c *Client) ProduceMessage(producer sarama.SyncProducer, topic string, message []byte) error { 82 | msg := &sarama.ProducerMessage{ 83 | Topic: topic, 84 | Value: sarama.ByteEncoder(message), 85 | } 86 | partition, offset, err := producer.SendMessage(msg) 87 | if err != nil { 88 | return fmt.Errorf("failed to send message: %w", err) 89 | } 90 | c.logger.Info("Message produced", 91 | zap.String("topic", topic), 92 | zap.Int32("partition", partition), 93 | zap.Int64("offset", offset)) 94 | return nil 95 | } 96 | 97 | // ListTopics lists all topics 98 | func (c *Client) ListTopics() (map[string]sarama.TopicDetail, error) { 99 | admin, err := c.newClusterAdmin() 100 | if err != nil { 101 | return nil, err 102 | } 103 | defer admin.Close() 104 | 105 | topics, err := admin.ListTopics() 106 | if err != nil { 107 | return nil, fmt.Errorf("failed to list topics: %w", err) 108 | } 109 | 110 | return topics, nil 111 | } 112 | 113 | // CreateTopic creates a new topic 114 | func (c *Client) CreateTopic(topicName string, detail *sarama.TopicDetail) error { 115 | admin, err := c.newClusterAdmin() 116 | if err != nil { 117 | return err 118 | } 119 | defer admin.Close() 120 | 121 | err = admin.CreateTopic(topicName, detail, false) 122 | if err != nil { 123 | return fmt.Errorf("failed to create topic: %w", err) 124 | } 125 | 126 | c.logger.Info("Topic created", zap.String("topic", topicName)) 127 | return nil 128 | } 129 | -------------------------------------------------------------------------------- /pkg/pipeline/peer/kafka/scram.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "crypto/sha256" 5 | "crypto/sha512" 6 | 7 | "github.com/xdg-go/scram" 8 | ) 9 | 10 | var ( 11 | SHA256 scram.HashGeneratorFcn = sha256.New 12 | SHA512 scram.HashGeneratorFcn = sha512.New 13 | ) 14 | 15 | type XDGSCRAMClient struct { 16 | *scram.Client 17 | *scram.ClientConversation 18 | scram.HashGeneratorFcn 19 | } 20 | 21 | func (x *XDGSCRAMClient) Begin(userName, password, authzID string) (err error) { 22 | x.Client, err = x.HashGeneratorFcn.NewClient(userName, password, authzID) 23 | if err != nil { 24 | return err 25 | } 26 | x.ClientConversation = x.Client.NewConversation() 27 | return nil 28 | } 29 | 30 | func (x *XDGSCRAMClient) Step(challenge string) (response string, err error) { 31 | response, err = x.ClientConversation.Step(challenge) 32 | return 33 | } 34 | 35 | func (x *XDGSCRAMClient) Done() bool { 36 | return x.ClientConversation.Done() 37 | } 38 | -------------------------------------------------------------------------------- /pkg/pipeline/peer/mqtt/doc.go: -------------------------------------------------------------------------------- 1 | // Package mqtt provides a real-time MQTT-based interface for PostgreSQL, 2 | // similar to how PostgREST exposes PostgreSQL over HTTP. 3 | package mqtt 4 | 5 | // topic: /prefix/OPTIONAL_SCHEMA/TABLE/OPERATION/COL1/VAL1/COL2/VAL2/... 6 | // payload: JSON (object / array) 7 | // 8 | // OPERATION 9 | // c=create, u=update, d=delete, r=read 10 | // 11 | // Example: 12 | // mosquitto_pub -t /example/prefix/public/devices/c -m '{"name":"kitchen-light"}' 13 | // 14 | // mosquitto_sub -t '/example/prefix/public/devices/#' // 15 | // 16 | // mosquitto_pub -t /example/prefix/iot/sensors/r/name/kitchen-light 17 | // mosquitto_pub -t /example/prefix/iot/sensors/read/name/kitchen-light 18 | // mosquitto_pub -t /example/prefix/iot/sensors/u/id/100 -m '{"name":"kitchen-light", "status": 0}' 19 | // mosquitto_pub -t /example/prefix/iot/sensors/d/id/100 20 | 21 | // In all cases, a response is published, unless disabled, to the /response/original/topic 22 | // mosquitto_sub -t /response/example/prefix/iot/sensors/d/id/100 23 | // 24 | // With a trailing /batch in topic, it's possible to supply an array of json objects for supported operations 25 | // mosquitto_pub -t /example/prefix/devices/c/batch -m '[{"name":"device1"}, {"name":"device2"}]' 26 | -------------------------------------------------------------------------------- /pkg/pipeline/peer/mqtt/options.go: -------------------------------------------------------------------------------- 1 | package mqtt 2 | 3 | import ( 4 | "crypto/tls" 5 | "net" 6 | "net/http" 7 | "net/url" 8 | "time" 9 | 10 | mqtt "github.com/eclipse/paho.mqtt.golang" 11 | ) 12 | 13 | type ClientOptions struct { 14 | Store mqtt.Store 15 | OnConnectAttempt mqtt.ConnectionAttemptHandler 16 | CustomOpenConnectionFn mqtt.OpenConnectionFunc 17 | DefaultPublishHandler mqtt.MessageHandler 18 | CredentialsProvider mqtt.CredentialsProvider 19 | HTTPHeaders http.Header 20 | Dialer *net.Dialer 21 | WebsocketOptions *mqtt.WebsocketOptions 22 | OnConnect mqtt.OnConnectHandler 23 | OnConnectionLost mqtt.ConnectionLostHandler 24 | OnReconnecting mqtt.ReconnectHandler 25 | TLSConfig *tls.Config 26 | ClientID string `json:"clientID"` 27 | WillTopic string 28 | Username string `json:"username"` 29 | Password string `json:"password"` 30 | Servers []*url.URL `json:"servers"` 31 | WillPayload []byte 32 | WriteTimeout time.Duration 33 | ConnectRetryInterval time.Duration 34 | MaxResumePubInFlight int 35 | MessageChannelDepth uint 36 | MaxReconnectInterval time.Duration 37 | ConnectTimeout time.Duration 38 | PingTimeout time.Duration 39 | KeepAlive int64 40 | ProtocolVersion uint 41 | WillRetained bool 42 | AutoReconnect bool 43 | ResumeSubs bool 44 | WillQos byte 45 | WillEnabled bool 46 | ConnectRetry bool 47 | Order bool 48 | CleanSession bool 49 | AutoAckDisabled bool 50 | } 51 | -------------------------------------------------------------------------------- /pkg/pipeline/peer/nats/doc.go: -------------------------------------------------------------------------------- 1 | // Package nats provides a real-time NATS-based interface for PostgreSQL, 2 | // similar to how PostgREST exposes PostgreSQL over HTTP. 3 | // 4 | // NATS subject (aka topic) patterns: 5 | // - Case-sensitive, dot-separated, no spaces 6 | // - Valid chars: alphanumeric, `-` or `_` 7 | // - Max length: 255 bytes 8 | // 9 | // pgo uses `any.nested.prefix.schema_name.table_name.operation` topic to interact with PostgreSQL 10 | // 11 | // Operations: 12 | // - create (or c): Insert operations 13 | // - update (or u): Update operations 14 | // - delete (or d): Delete operations 15 | // - read (or r): Query operations 16 | // - truncate (or t): Truncate operations 17 | // 18 | // Examples: 19 | // - public.users.c → Create user 20 | // - inventory.products.u → Update product 21 | // - accounting.invoices.r → Read invoices 22 | // 23 | // Payload: JSON 24 | // 25 | // Query Parameters: 26 | // 27 | // schema_name.table_name.read.field.value 28 | // Example: public.users.r.id.123 → read by id 123 29 | // 30 | // Use '$PG.' prefix for system operations 31 | package nats 32 | -------------------------------------------------------------------------------- /pkg/pipeline/peer/plugin_example/peer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | 7 | "github.com/edgeflare/pgo/pkg/pipeline" 8 | "github.com/edgeflare/pgo/pkg/pipeline/cdc" 9 | ) 10 | 11 | type PeerExample struct{} 12 | 13 | func (p *PeerExample) Pub(event cdc.Event, args ...any) error { 14 | log.Println("example connector plugin publish", event) 15 | return nil 16 | } 17 | 18 | func (p *PeerExample) Connect(config json.RawMessage, args ...any) error { 19 | log.Println("example connector plugin init", config) 20 | return nil 21 | } 22 | 23 | func (p *PeerExample) Sub(args ...any) (<-chan cdc.Event, error) { 24 | // for pub-only peers (sinks), or implement for sub/pubsub peers 25 | return nil, pipeline.ErrConnectorTypeMismatch 26 | } 27 | 28 | func (p *PeerExample) Type() pipeline.ConnectorType { 29 | return pipeline.ConnectorTypePub 30 | } 31 | 32 | func (p *PeerExample) Disconnect() error { 33 | return nil 34 | } 35 | 36 | var Connector pipeline.Connector = &PeerExample{} 37 | -------------------------------------------------------------------------------- /pkg/pipeline/pipeline.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/edgeflare/pgo/pkg/pipeline/cdc" 9 | "github.com/edgeflare/pgo/pkg/pipeline/transform" 10 | ) 11 | 12 | // Sink is a pipeline output with its transformations. 13 | type Sink struct { 14 | // Name must match one of configured peers 15 | Name string `mapstructure:"name"` 16 | // Sink-specific transformations are applied after source transformations, pipeline transformations and before sending to speceific sink 17 | Transformations []transform.Transformation `mapstructure:"transformations"` 18 | } 19 | 20 | // Pipeline configures a complete data processing pipeline. 21 | type Pipeline struct { 22 | Name string `mapstructure:"name"` 23 | Sources []Source `mapstructure:"sources"` 24 | // Pipeline transformations are applied after source transformations and before sink transformations. 25 | // These are applied to all CDC events flowing through a pipeline from its all sources to all sinks 26 | Transformations []transform.Transformation `mapstructure:"transformations"` 27 | Sinks []Sink `mapstructure:"sinks"` 28 | } 29 | 30 | type Config struct { 31 | Peers []Peer `mapstructure:"peers"` 32 | Pipelines []Pipeline `mapstructure:"pipelines"` 33 | } 34 | 35 | func (c *Config) GetPeer(peerName string) *Peer { 36 | for _, peer := range c.Peers { 37 | if peer.Name == peerName { 38 | return &peer 39 | } 40 | } 41 | return nil 42 | } 43 | 44 | func (c *Config) GetPipeline(pipelineName string) *Pipeline { 45 | for _, pipeline := range c.Pipelines { 46 | if pipeline.Name == pipelineName { 47 | return &pipeline 48 | } 49 | } 50 | return nil 51 | } 52 | 53 | func SetupSinks( 54 | ctx context.Context, 55 | m *Manager, 56 | wg *sync.WaitGroup, 57 | pl Pipeline, 58 | sinkChannels map[string]chan cdc.Event, 59 | ) error { 60 | for _, sink := range pl.Sinks { 61 | sinkPeer, err := m.GetPeer(sink.Name) 62 | if err != nil { 63 | return fmt.Errorf("sink peer %s not found: %w", sink.Name, err) 64 | } 65 | 66 | ch := sinkChannels[sink.Name] 67 | wg.Add(1) 68 | go processSinkEvents(ctx, wg, pl, sink, sinkPeer, ch) 69 | } 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /pkg/pipeline/process.go: -------------------------------------------------------------------------------- 1 | package pipeline 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "sync" 8 | 9 | "github.com/edgeflare/pgo/pkg/metrics" 10 | "github.com/edgeflare/pgo/pkg/pipeline/cdc" 11 | "github.com/edgeflare/pgo/pkg/pipeline/transform" 12 | "github.com/prometheus/client_golang/prometheus" 13 | ) 14 | 15 | func distributeToSinks( 16 | pl Pipeline, 17 | source Source, 18 | event cdc.Event, 19 | sinkChannels map[string]chan cdc.Event, 20 | ) { 21 | for _, sink := range pl.Sinks { 22 | if ch, ok := sinkChannels[sink.Name]; ok { 23 | select { 24 | case ch <- event: 25 | metrics.ProcessedEvents.WithLabelValues( 26 | pl.Name, 27 | source.Name, 28 | sink.Name, 29 | ).Inc() 30 | default: 31 | log.Printf("Warning: Sink channel %s is full", sink.Name) 32 | } 33 | } 34 | } 35 | } 36 | 37 | func applyTransformations(event *cdc.Event, transformations []transform.Transformation) (*cdc.Event, error) { 38 | if len(transformations) == 0 { 39 | return event, nil 40 | } 41 | 42 | if event == nil { 43 | return nil, fmt.Errorf("cannot transform nil event") 44 | } 45 | 46 | manager := transform.NewManager() 47 | manager.RegisterBuiltins() 48 | 49 | chainTransformations, err := manager.Chain(transformations) 50 | if err != nil { 51 | return nil, fmt.Errorf("error creating transformation pipeline: %w", err) 52 | } 53 | 54 | result, err := chainTransformations(event) 55 | if result == nil && err == nil { 56 | // Transform indicated event should be filtered out 57 | return nil, nil 58 | } 59 | return result, err 60 | } 61 | 62 | // processSinkEvents handles events from multiple sources 63 | func processSinkEvents( 64 | ctx context.Context, 65 | wg *sync.WaitGroup, 66 | pl Pipeline, 67 | sink Sink, 68 | peer *Peer, 69 | ch <-chan cdc.Event, 70 | ) { 71 | defer wg.Done() 72 | 73 | for { 74 | select { 75 | case event, ok := <-ch: 76 | if !ok { 77 | return 78 | } 79 | 80 | // Apply sink-specific transformations 81 | transformedEvent, err := applyTransformations(&event, sink.Transformations) 82 | if err != nil { 83 | metrics.TransformationErrors.WithLabelValues( 84 | "sink", 85 | pl.Name, 86 | "multiple", 87 | sink.Name, 88 | ).Inc() 89 | log.Printf("Sink transformation error: %v", err) 90 | continue 91 | } 92 | if transformedEvent == nil { 93 | continue 94 | } 95 | 96 | // Publish the transformed event 97 | if err := peer.Connector().Pub(*transformedEvent); err != nil { 98 | metrics.PublishErrors.WithLabelValues(sink.Name).Inc() 99 | log.Printf("Publish error to %s: %v", peer.Name, err) 100 | } 101 | 102 | case <-ctx.Done(): 103 | return 104 | } 105 | } 106 | } 107 | 108 | // processEvent handles the processing of a single event 109 | func ProcessEvent( 110 | pl Pipeline, 111 | source Source, 112 | event cdc.Event, 113 | sinkChannels map[string]chan cdc.Event, 114 | ) { 115 | timer := prometheus.NewTimer(metrics.EventProcessingDuration.WithLabelValues( 116 | pl.Name, 117 | source.Name, 118 | "", 119 | )) 120 | defer timer.ObserveDuration() 121 | 122 | // Process source transformations 123 | transformedEvent := applyEventTransformations(event, source, pl, sinkChannels) 124 | if transformedEvent == nil { 125 | return 126 | } 127 | 128 | // Distribute to sinks 129 | distributeToSinks(pl, source, *transformedEvent, sinkChannels) 130 | } 131 | 132 | // applyEventTransformations applies all transformations to an event 133 | func applyEventTransformations( 134 | event cdc.Event, 135 | source Source, 136 | pl Pipeline, 137 | sinkChannels map[string]chan cdc.Event, 138 | ) *cdc.Event { 139 | // Source transformations 140 | transformed, err := applyTransformations(&event, source.Transformations) 141 | if err != nil { 142 | metrics.TransformationErrors.WithLabelValues( 143 | "source", 144 | pl.Name, 145 | source.Name, 146 | "", 147 | ).Inc() 148 | log.Printf("Source transformation error: %v", err) 149 | return nil 150 | } 151 | if transformed == nil { 152 | return nil 153 | } 154 | 155 | // Pipeline transformations 156 | transformed, err = applyTransformations(transformed, pl.Transformations) 157 | if err != nil { 158 | metrics.TransformationErrors.WithLabelValues( 159 | "pipeline", 160 | pl.Name, 161 | source.Name, 162 | "", 163 | ).Inc() 164 | log.Printf("Pipeline transformation error: %v", err) 165 | return nil 166 | } 167 | 168 | return transformed 169 | } 170 | -------------------------------------------------------------------------------- /pkg/pipeline/transform/doc.go: -------------------------------------------------------------------------------- 1 | // Package transform provides utilities for applying transformations to change data capture (CDC) events in pipelines. 2 | // It's inspired by Debezium's [Single Message Transformations (SMTs)](https://docs.confluent.io/platform/current/connect/transforms/overview.html) usage. 3 | package transform 4 | -------------------------------------------------------------------------------- /pkg/pipeline/transform/extract.go: -------------------------------------------------------------------------------- 1 | package transform 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/edgeflare/pgo/pkg/pipeline/cdc" 7 | ) 8 | 9 | // ExtractConfig holds the configuration for the extract transformation 10 | type ExtractConfig struct { 11 | Fields []string `json:"fields"` 12 | } 13 | 14 | // Validate validates the ExtractConfig 15 | func (c *ExtractConfig) Validate() error { 16 | if len(c.Fields) == 0 { 17 | return fmt.Errorf("at least one field is required") 18 | } 19 | return nil 20 | } 21 | 22 | // Type returns the type of the transformation 23 | func (c *ExtractConfig) Type() string { 24 | return "extract" 25 | } 26 | 27 | // Extract creates a Func that extracts specified fields from the CDC event 28 | func Extract(config *ExtractConfig) Func { 29 | return func(cdc *cdc.Event) (*cdc.Event, error) { 30 | if err := config.Validate(); err != nil { 31 | return cdc, fmt.Errorf("invalid extract configuration: %w", err) 32 | } 33 | current := cdc 34 | fields := config.Fields 35 | newBefore := make(map[string]interface{}) 36 | newAfter := make(map[string]interface{}) 37 | 38 | if before, ok := current.Payload.Before.(map[string]interface{}); ok { 39 | for _, field := range fields { 40 | if value, exists := before[field]; exists { 41 | newBefore[field] = value 42 | } 43 | } 44 | } 45 | if after, ok := current.Payload.After.(map[string]interface{}); ok { 46 | for _, field := range fields { 47 | if value, exists := after[field]; exists { 48 | newAfter[field] = value 49 | } 50 | } 51 | } 52 | 53 | current.Payload.Before = newBefore 54 | current.Payload.After = newAfter 55 | 56 | return current, nil 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /pkg/pipeline/transform/extract_test.go: -------------------------------------------------------------------------------- 1 | package transform 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/edgeflare/pgo/internal/testutil" 7 | "github.com/edgeflare/pgo/pkg/pipeline/cdc" 8 | ) 9 | 10 | func TestExtract(t *testing.T) { 11 | var cdc cdc.Event 12 | _, err := testutil.LoadJSON("cdc.json", &cdc) 13 | if err != nil { 14 | t.Fatalf("Failed to load JSON into struct: %v", err) 15 | } 16 | // t.Logf("Loaded CDC: %+v", cdc) 17 | 18 | testCases := []struct { 19 | want map[string]any 20 | name string 21 | fields []string 22 | }{ 23 | { 24 | name: "Extract multi fields of different types", 25 | fields: []string{"email", "id"}, 26 | want: map[string]any{ 27 | "email": "annek@noanswer.org", 28 | "id": float64(1), // JSON numbers are decoded as float64 29 | }, 30 | }, 31 | { 32 | name: "Extract only one field", 33 | fields: []string{"email"}, 34 | want: map[string]any{ 35 | "email": "annek@noanswer.org", 36 | }, 37 | }, 38 | } 39 | 40 | registry := NewRegistry() 41 | registry.Register("extract", func(config Config) Func { 42 | return Extract(config.(*ExtractConfig)) 43 | }) 44 | 45 | for _, tc := range testCases { 46 | t.Run(tc.name, func(t *testing.T) { 47 | extractConfig := &ExtractConfig{Fields: tc.fields} 48 | transform, err := registry.Get("extract") 49 | if err != nil { 50 | t.Fatalf("Failed to get transform function: %v", err) 51 | } 52 | 53 | transformedCDC, err := transform(extractConfig)(&cdc) 54 | if err != nil { 55 | t.Fatalf("Failed to apply transform: %v", err) 56 | } 57 | 58 | after, ok := transformedCDC.Payload.After.(map[string]interface{}) 59 | if !ok { 60 | t.Fatalf("After payload is not in expected format") 61 | } 62 | 63 | // Verify all expected fields are present with correct values 64 | for field, expectedValue := range tc.want { 65 | value, exists := after[field] 66 | if !exists { 67 | t.Errorf("Field '%s' not found in the transformed CDC", field) 68 | continue 69 | } 70 | if value != expectedValue { 71 | t.Errorf("For field '%s': expected %v, got %v", field, expectedValue, value) 72 | } 73 | } 74 | 75 | // Verify only requested fields are present 76 | for field := range after { 77 | if _, expected := tc.want[field]; !expected { 78 | t.Errorf("Unexpected field '%s' found in the transformed CDC", field) 79 | } 80 | } 81 | }) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /pkg/pipeline/transform/replace.go: -------------------------------------------------------------------------------- 1 | package transform 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | 7 | "github.com/edgeflare/pgo/pkg/pipeline/cdc" 8 | ) 9 | 10 | // ReplaceConfig holds the configuration for the replace transformation 11 | type ReplaceConfig struct { 12 | // Schema replacements 13 | Schemas map[string]string `json:"schemas,omitempty"` 14 | 15 | // Table replacements 16 | Tables map[string]string `json:"tables,omitempty"` 17 | 18 | // Column/field replacements 19 | Columns map[string]string `json:"columns,omitempty"` 20 | 21 | // Regex replacements 22 | Regex []RegexReplacement `json:"regex,omitempty"` 23 | } 24 | 25 | // RegexReplacement defines a regex-based replacement rule 26 | type RegexReplacement struct { 27 | Type string `json:"type"` // "schema", "table", or "column" 28 | Pattern string `json:"pattern"` // Regex pattern to match 29 | Replace string `json:"replace"` // Replacement string (can use regex groups) 30 | } 31 | 32 | // Validate validates the ReplaceConfig 33 | func (c *ReplaceConfig) Validate() error { 34 | // Ensure at least one replacement type is configured 35 | if len(c.Schemas) == 0 && 36 | len(c.Tables) == 0 && 37 | len(c.Columns) == 0 && 38 | len(c.Regex) == 0 { 39 | return fmt.Errorf("at least one replacement configuration is required") 40 | } 41 | 42 | // Validate regex patterns 43 | for _, regex := range c.Regex { 44 | if !isValidReplacementType(regex.Type) { 45 | return fmt.Errorf("invalid replacement type: %s", regex.Type) 46 | } 47 | if _, err := regexp.Compile(regex.Pattern); err != nil { 48 | return fmt.Errorf("invalid regex pattern %s: %w", regex.Pattern, err) 49 | } 50 | } 51 | 52 | return nil 53 | } 54 | 55 | func isValidReplacementType(t string) bool { 56 | return t == "schema" || t == "table" || t == "column" 57 | } 58 | 59 | // Type returns the type of the transformation 60 | func (c *ReplaceConfig) Type() string { 61 | return "replace" 62 | } 63 | 64 | // Replace creates a Func that performs the configured replacements 65 | func Replace(config *ReplaceConfig) Func { 66 | return func(cdc *cdc.Event) (*cdc.Event, error) { 67 | if err := config.Validate(); err != nil { 68 | return cdc, fmt.Errorf("invalid replace configuration: %w", err) 69 | } 70 | 71 | // Create a copy of the CDC event 72 | current := *cdc 73 | 74 | // Apply schema replacements 75 | if newSchema, exists := config.Schemas[current.Payload.Source.Schema]; exists { 76 | current.Payload.Source.Schema = newSchema 77 | } 78 | 79 | // Apply table replacements 80 | if newTable, exists := config.Tables[current.Payload.Source.Table]; exists { 81 | current.Payload.Source.Table = newTable 82 | } 83 | 84 | // Apply regex replacements 85 | for _, regex := range config.Regex { 86 | re := regexp.MustCompile(regex.Pattern) 87 | switch regex.Type { 88 | case "schema": 89 | current.Payload.Source.Schema = re.ReplaceAllString(current.Payload.Source.Schema, regex.Replace) 90 | case "table": 91 | current.Payload.Source.Table = re.ReplaceAllString(current.Payload.Source.Table, regex.Replace) 92 | } 93 | } 94 | 95 | // Apply column replacements to both Before and After payloads 96 | if len(config.Columns) > 0 { 97 | current.Payload.Before = replaceMapKeys(current.Payload.Before, config.Columns) 98 | current.Payload.After = replaceMapKeys(current.Payload.After, config.Columns) 99 | 100 | // Update schema fields if present 101 | for i, field := range current.Schema.Fields { 102 | if newName, exists := config.Columns[field.Field]; exists { 103 | current.Schema.Fields[i].Field = newName 104 | if field.Name != "" { 105 | current.Schema.Fields[i].Name = newName 106 | } 107 | } 108 | } 109 | } 110 | 111 | return ¤t, nil 112 | } 113 | } 114 | 115 | // replaceMapKeys creates a new map with replaced keys according to the replacements map 116 | func replaceMapKeys(data interface{}, replacements map[string]string) interface{} { 117 | if data == nil { 118 | return nil 119 | } 120 | 121 | if m, ok := data.(map[string]interface{}); ok { 122 | newMap := make(map[string]interface{}) 123 | for k, v := range m { 124 | newKey := k 125 | if replacement, exists := replacements[k]; exists { 126 | newKey = replacement 127 | } 128 | newMap[newKey] = v 129 | } 130 | return newMap 131 | } 132 | 133 | return data 134 | } 135 | -------------------------------------------------------------------------------- /pkg/rest/apache_age_test.go: -------------------------------------------------------------------------------- 1 | package rest_test 2 | 3 | import ( 4 | "cmp" 5 | "context" 6 | "fmt" 7 | "log" 8 | "net/http" 9 | "net/http/httptest" 10 | "os" 11 | "strings" 12 | 13 | "github.com/edgeflare/pgo/pkg/rest" 14 | "github.com/jackc/pgx/v5/pgxpool" 15 | ) 16 | 17 | /* 18 | // Example requests: 19 | 20 | alias curlcmd='curl -X POST http://localhost:8080/graph -H "Content-Type: application/json"' 21 | 22 | curlcmd -d '{"cypher": "CREATE (n:Person {name: \"Joe\", age: 65})"}' 23 | 24 | curlcmd -d '{"cypher": "CREATE (n:Person {name: \"Jack\", age: 55})"}' 25 | 26 | curlcmd -d '{"cypher": "CREATE (n:Person {name: \"Jane\", age: 45})"}' 27 | 28 | curlcmd -d '{"cypher": "MATCH (a:Person {name: \"Joe\"}), (b:Person {name: \"Jack\"}) CREATE (a)-[r:KNOWS {since: 2020}]->(b)"}' 29 | 30 | curlcmd -d '{"cypher": "MATCH (v) RETURN v", "columnCount": 1}' 31 | 32 | curlcmd -d '{"cypher": "MATCH p=()-[r]-() RETURN p, r", "columnCount": 2}' 33 | */ 34 | 35 | func ExampleAGEHandler() { 36 | // Initialize a connection pool 37 | ctx := context.Background() 38 | graphName := "test_cypher" 39 | 40 | pool, err := pgxpool.New(ctx, cmp.Or(os.Getenv("DATABASE_URL"), "postgres://postgres:secret@localhost:5432/testdb")) 41 | if err != nil { 42 | log.Fatalf("failed to create connection pool: %v", err) 43 | } 44 | defer pool.Close() 45 | 46 | // Create an AGE handler with a default graph 47 | handler, err := rest.NewAGEHandler(ctx, pool, graphName) 48 | if err != nil { 49 | log.Fatalf("failed to create AGE handler: %v", err) 50 | } 51 | defer handler.Close() 52 | 53 | // Setup a test HTTP server 54 | server := httptest.NewServer(handler) 55 | defer server.Close() 56 | 57 | // // or mount on specific /path 58 | // mux := http.NewServeMux() 59 | // mux.Handle("POST /graph", handler) 60 | // if err = http.ListenAndServe(":8080", mux); err != nil { 61 | // log.Fatalf("Server error: %v", err) 62 | // } 63 | 64 | // Example 1: Create a vertex 65 | createReq := `{ 66 | "cypher": "CREATE (n:Person {name: 'Alice', age: 30}) RETURN n", 67 | "columnCount": 1 68 | }` 69 | 70 | resp, err := http.Post(server.URL, "application/json", strings.NewReader(createReq)) 71 | if err != nil { 72 | log.Fatalf("failed to send request: %v", err) 73 | } 74 | defer resp.Body.Close() 75 | 76 | fmt.Printf("Create vertex status: %d\n", resp.StatusCode) 77 | 78 | // Example 2: Query vertices 79 | queryReq := `{ 80 | "cypher": "MATCH (n:Person) RETURN n", 81 | "columnCount": 1 82 | }` 83 | 84 | resp, err = http.Post(server.URL, "application/json", strings.NewReader(queryReq)) 85 | if err != nil { 86 | log.Fatalf("failed to send request: %v", err) 87 | } 88 | defer resp.Body.Close() 89 | 90 | fmt.Printf("Query vertices status: %d\n", resp.StatusCode) 91 | 92 | // Output: 93 | // Create vertex status: 200 94 | // Query vertices status: 200 95 | } 96 | -------------------------------------------------------------------------------- /pkg/rest/doc.go: -------------------------------------------------------------------------------- 1 | // Package rest provides a PostgreSQL REST API server similar to PostgREST. 2 | // 3 | // The server automatically exposes database tables and views as REST endpoints. 4 | // Each endpoint supports standard HTTP methods: GET, POST, PATCH, DELETE. 5 | // 6 | // Query parameters control filtering, pagination, and ordering: 7 | // 8 | // Parameter | Description 9 | // ------------------|------------------------------------------------ 10 | // ?select=col1,col2 | Select specific columns 11 | // ?order=col.desc | Order results (supports nullsfirst/nullslast) 12 | // ?limit=100 | Limit number of results (default: 100) 13 | // ?offset=0 | Pagination offset (default: 0) 14 | // ?col=eq.val | Filter by column equality 15 | // ?col=gt.val | Filter with greater than comparison 16 | // ?col=lt.val | Filter with less than comparison 17 | // ?col=gte.val | Filter with greater than or equal comparison 18 | // ?col=lte.val | Filter with less than or equal comparison 19 | // ?col=like.val | Filter with pattern matching 20 | // ?col=in.(a,b,c) | Filter with value lists 21 | // ?col=is.null | Filter for null values 22 | // ?or=(a.eq.x,b.lt.y) | Combine filters with logical operators 23 | // 24 | // API is compatible with PostgREST. For more details, see: 25 | // https://docs.postgrest.org/en/stable/references/api/tables_views.html 26 | // 27 | // Example usage: 28 | // 29 | // server, err := rest.NewServer("postgres://user:pass@localhost/db", "") 30 | // if err != nil { 31 | // log.Fatal(err) 32 | // } 33 | // defer server.Shutdown() 34 | // log.Fatal(server.Start(":8080")) 35 | package rest 36 | -------------------------------------------------------------------------------- /pkg/util/cert.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "crypto/ecdsa" 5 | "crypto/elliptic" 6 | "crypto/rand" 7 | "crypto/tls" 8 | "crypto/x509" 9 | "crypto/x509/pkix" 10 | "encoding/pem" 11 | "fmt" 12 | "math/big" 13 | "os" 14 | "path/filepath" 15 | "time" 16 | ) 17 | 18 | // LoadOrGenerateCert generates a self-signed certificate and private key if they do not exist at the specified paths. 19 | // If the files already exist, they are loaded and returned. 20 | func LoadOrGenerateCert(certPath, keyPath string) (tls.Certificate, error) { 21 | priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) 22 | if err != nil { 23 | return tls.Certificate{}, fmt.Errorf("failed to generate private key: %v", err) 24 | } 25 | 26 | template := x509.Certificate{ 27 | SerialNumber: big.NewInt(1), 28 | Subject: pkix.Name{ 29 | Organization: []string{"Self-Signed"}, 30 | }, 31 | NotBefore: time.Now(), 32 | NotAfter: time.Now().Add(365 * 24 * time.Hour), 33 | 34 | KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, 35 | ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, 36 | BasicConstraintsValid: true, 37 | } 38 | 39 | derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) 40 | if err != nil { 41 | return tls.Certificate{}, fmt.Errorf("failed to create certificate: %v", err) 42 | } 43 | 44 | cert := tls.Certificate{ 45 | Certificate: [][]byte{derBytes}, 46 | PrivateKey: priv, 47 | } 48 | 49 | // Ensure the directory exists 50 | err = os.MkdirAll(filepath.Dir(certPath), os.ModePerm) 51 | if err != nil { 52 | return tls.Certificate{}, fmt.Errorf("failed to create tls directory: %v", err) 53 | } 54 | 55 | // Write the cert and key to files 56 | err = writeCert(certPath, derBytes) 57 | if err != nil { 58 | return tls.Certificate{}, err 59 | } 60 | 61 | err = writeKey(keyPath, priv) 62 | if err != nil { 63 | return tls.Certificate{}, err 64 | } 65 | 66 | return cert, nil 67 | } 68 | 69 | // LoadCertFromFiles loads a TLS certificate from the specified paths. 70 | func loadCertFromFiles(certPath, keyPath string) (tls.Certificate, error) { 71 | cert, err := tls.LoadX509KeyPair(certPath, keyPath) 72 | if err != nil { 73 | return tls.Certificate{}, fmt.Errorf("failed to load TLS certificate: %v", err) 74 | } 75 | return cert, nil 76 | } 77 | 78 | func writeCert(certPath string, derBytes []byte) error { 79 | certOut, err := os.Create(certPath) 80 | if err != nil { 81 | return fmt.Errorf("failed to create cert file: %v", err) 82 | } 83 | defer certOut.Close() 84 | 85 | err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) 86 | if err != nil { 87 | return fmt.Errorf("failed to write certificate to file: %v", err) 88 | } 89 | return nil 90 | } 91 | 92 | func writeKey(keyPath string, priv *ecdsa.PrivateKey) error { 93 | keyOut, err := os.Create(keyPath) 94 | if err != nil { 95 | return fmt.Errorf("failed to create key file: %v", err) 96 | } 97 | defer keyOut.Close() 98 | 99 | privBytes, err := x509.MarshalECPrivateKey(priv) 100 | if err != nil { 101 | return fmt.Errorf("failed to marshal private key: %v", err) 102 | } 103 | 104 | err = pem.Encode(keyOut, &pem.Block{Type: "EC PRIVATE KEY", Bytes: privBytes}) 105 | if err != nil { 106 | return fmt.Errorf("failed to write private key to file: %v", err) 107 | } 108 | return nil 109 | } 110 | -------------------------------------------------------------------------------- /pkg/util/cert_test.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | ) 8 | 9 | // TestGenerateSelfSignedCert tests the LoadOrGenerateCert function for correctness. 10 | func TestGenerateSelfSignedCert(t *testing.T) { 11 | certPath := "./test_tls/tls.crt" 12 | keyPath := "./test_tls/tls.key" 13 | 14 | // Clean up the test files after the test 15 | defer func() { 16 | os.Remove(certPath) 17 | os.Remove(keyPath) 18 | os.Remove(filepath.Dir(certPath)) 19 | }() 20 | 21 | cert, err := LoadOrGenerateCert(certPath, keyPath) 22 | if err != nil { 23 | t.Fatalf("LoadOrGenerateCert() failed: %v", err) 24 | } 25 | 26 | // Check if the certificate and key files were created 27 | if _, err := os.Stat(certPath); os.IsNotExist(err) { 28 | t.Errorf("Expected certificate file %s does not exist", certPath) 29 | } 30 | 31 | if _, err := os.Stat(keyPath); os.IsNotExist(err) { 32 | t.Errorf("Expected key file %s does not exist", keyPath) 33 | } 34 | 35 | // Validate the certificate structure 36 | if len(cert.Certificate) == 0 { 37 | t.Errorf("Generated certificate has no data") 38 | } 39 | 40 | // Validate the private key structure 41 | if cert.PrivateKey == nil { 42 | t.Errorf("Generated certificate has no private key") 43 | } 44 | } 45 | 46 | // TestLoadCertFromFiles tests the LoadCertFromFiles function for correctness. 47 | func TestLoadCertFromFiles(t *testing.T) { 48 | certPath := "./test_tls/tls.crt" 49 | keyPath := "./test_tls/tls.key" 50 | 51 | // Generate a self-signed certificate to be loaded later 52 | _, err := LoadOrGenerateCert(certPath, keyPath) 53 | if err != nil { 54 | t.Fatalf("LoadOrGenerateCert() failed: %v", err) 55 | } 56 | 57 | // Clean up the test files after the test 58 | defer func() { 59 | os.Remove(certPath) 60 | os.Remove(keyPath) 61 | os.Remove(filepath.Dir(certPath)) 62 | }() 63 | 64 | cert, err := loadCertFromFiles(certPath, keyPath) 65 | if err != nil { 66 | t.Fatalf("LoadCertFromFiles() failed: %v", err) 67 | } 68 | 69 | // Validate the loaded certificate 70 | if len(cert.Certificate) == 0 { 71 | t.Errorf("Loaded certificate has no data") 72 | } 73 | 74 | // Validate the private key structure 75 | if cert.PrivateKey == nil { 76 | t.Errorf("Loaded certificate has no private key") 77 | } 78 | } 79 | 80 | // TestGenerateSelfSignedCert_DirectoryCreation tests the directory creation for the certificate and key files. 81 | func TestGenerateSelfSignedCert_DirectoryCreation(t *testing.T) { 82 | certPath := "./test_tls_subdir/tls.crt" 83 | keyPath := "./test_tls_subdir/tls.key" 84 | 85 | // Clean up the test files after the test 86 | defer func() { 87 | os.Remove(certPath) 88 | os.Remove(keyPath) 89 | os.Remove(filepath.Dir(certPath)) 90 | }() 91 | 92 | cert, err := LoadOrGenerateCert(certPath, keyPath) 93 | if err != nil { 94 | t.Fatalf("LoadOrGenerateCert() failed: %v", err) 95 | } 96 | 97 | // Check if the directory was created 98 | if _, err := os.Stat(filepath.Dir(certPath)); os.IsNotExist(err) { 99 | t.Errorf("Expected directory %s does not exist", filepath.Dir(certPath)) 100 | } 101 | 102 | // Check if the certificate and key files were created 103 | if _, err := os.Stat(certPath); os.IsNotExist(err) { 104 | t.Errorf("Expected certificate file %s does not exist", certPath) 105 | } 106 | 107 | if _, err := os.Stat(keyPath); os.IsNotExist(err) { 108 | t.Errorf("Expected key file %s does not exist", keyPath) 109 | } 110 | 111 | // Validate the certificate structure 112 | if len(cert.Certificate) == 0 { 113 | t.Errorf("Generated certificate has no data") 114 | } 115 | 116 | // Validate the private key structure 117 | if cert.PrivateKey == nil { 118 | t.Errorf("Generated certificate has no private key") 119 | } 120 | } 121 | 122 | // TestLoadCertFromFiles_InvalidPath tests loading certificates from invalid file paths. 123 | func TestLoadCertFromFiles_InvalidPath(t *testing.T) { 124 | invalidCertPath := "./test_tls/invalid.crt" 125 | invalidKeyPath := "./test_tls/invalid.key" 126 | 127 | _, err := loadCertFromFiles(invalidCertPath, invalidKeyPath) 128 | if err == nil { 129 | t.Error("Expected error loading certificate from invalid paths, got nil") 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /pkg/util/env.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import "os" 4 | 5 | // GetEnvOrDefault returns the environment variable value if set, otherwise the default value 6 | func GetEnvOrDefault(env, def string) string { 7 | if val := os.Getenv(env); val != "" { 8 | return val 9 | } 10 | return def 11 | } 12 | -------------------------------------------------------------------------------- /pkg/util/jq.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | ) 9 | 10 | var ( 11 | errInvalidInput = errors.New("invalid input or empty path") 12 | errNoWildcard = errors.New("no matching elements found for wildcard path") 13 | ) 14 | 15 | // Jq extracts values from a JSON-like map using a dotted path notation like the jq cli 16 | func Jq(input map[string]any, path string) (any, error) { 17 | if input == nil || path == "" { 18 | return nil, errInvalidInput 19 | } 20 | 21 | // Avoid allocation if no leading dot 22 | if path[0] == '.' { 23 | path = path[1:] 24 | } 25 | 26 | // Preallocate keys slice with estimated capacity 27 | keys := make([]string, 0, 5) // Most paths are < 5 segments 28 | start := 0 29 | for i := 0; i < len(path); i++ { 30 | if path[i] == '.' { 31 | if i > start { 32 | keys = append(keys, path[start:i]) 33 | } 34 | start = i + 1 35 | } 36 | } 37 | if start < len(path) { 38 | keys = append(keys, path[start:]) 39 | } 40 | 41 | var current any = input 42 | for i, key := range keys { 43 | isLastKey := i == len(keys)-1 44 | 45 | currentMap, ok := current.(map[string]any) 46 | if !ok { 47 | return nil, fmt.Errorf("expected map at path segment: %s", key) 48 | } 49 | 50 | // Fast path for non-array keys 51 | if !strings.ContainsRune(key, '[') { 52 | value, exists := currentMap[key] 53 | if !exists { 54 | return nil, fmt.Errorf("key not found: %s", key) 55 | } 56 | current = value 57 | continue 58 | } 59 | 60 | // Handle array notation 61 | arrayKey, indexStr, err := splitKeyAndIndex(key) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | array, ok := currentMap[arrayKey].([]any) 67 | if !ok { 68 | return nil, fmt.Errorf("expected array at key: %s", arrayKey) 69 | } 70 | 71 | // Handle wildcards 72 | if indexStr == "*" || indexStr == "" { 73 | if isLastKey { 74 | return array, nil 75 | } 76 | return handleWildcard(array, keys[i+1:]) 77 | } 78 | 79 | // Parse index 80 | index, err := strconv.Atoi(indexStr) 81 | if err != nil || index < 0 || index >= len(array) { 82 | return nil, fmt.Errorf("invalid index %s at key: %s", indexStr, arrayKey) 83 | } 84 | current = array[index] 85 | } 86 | 87 | return current, nil 88 | } 89 | 90 | // splitKeyAndIndex separates a key and its array index with minimal allocations 91 | func splitKeyAndIndex(key string) (string, string, error) { 92 | start := strings.IndexByte(key, '[') 93 | end := strings.IndexByte(key, ']') 94 | if start == -1 || end == -1 || end < start { 95 | return "", "", fmt.Errorf("malformed array syntax in key: %s", key) 96 | } 97 | return key[:start], key[start+1 : end], nil 98 | } 99 | 100 | // handleWildcard processes wildcard notation with pre-allocated results slice 101 | func handleWildcard(array []any, remainingKeys []string) (any, error) { 102 | remainingPath := strings.Join(remainingKeys, ".") 103 | results := make([]any, 0, len(array)) // Preallocate with capacity 104 | 105 | for _, item := range array { 106 | if itemMap, ok := item.(map[string]any); ok { 107 | value, err := Jq(itemMap, remainingPath) 108 | if err == nil { 109 | switch v := value.(type) { 110 | case []any: 111 | results = append(results, v...) 112 | default: 113 | results = append(results, v) 114 | } 115 | } 116 | } 117 | } 118 | 119 | if len(results) == 0 { 120 | return nil, errNoWildcard 121 | } 122 | return results, nil 123 | } 124 | -------------------------------------------------------------------------------- /pkg/util/rand/name.go: -------------------------------------------------------------------------------- 1 | package rand 2 | 3 | import ( 4 | mrand "math/rand" 5 | ) 6 | 7 | var adjectives = []string{ 8 | "agile", "brave", "calm", "daring", "eager", 9 | "fancy", "gentle", "happy", "intelligent", "jolly", 10 | "kind", "lively", "mighty", "noble", "optimistic", 11 | "playful", "quick", "radiant", "spirited", "trusty", 12 | "upbeat", "vibrant", "wise", "youthful", "zealous", 13 | "ambitious", "bright", "cheerful", "dynamic", "elegant", 14 | "fearless", "graceful", "hopeful", "inspired", "jovial", 15 | "keen", "loyal", "motivated", "nimble", "passionate", 16 | "resourceful", "sturdy", "tenacious", "uplifted", "vigorous", 17 | "warm", "xenial", "zesty", 18 | } 19 | 20 | var birds = []string{ 21 | "albatross", "bluebird", "canary", "dove", "eagle", 22 | "falcon", "goldfinch", "hawk", "ibis", "jay", 23 | "kingfisher", "lark", "magpie", "nightingale", "oriole", 24 | "parrot", "quail", "robin", "sparrow", "toucan", 25 | "umbrella bird", "vulture", "woodpecker", "yellowhammer", 26 | "zebra finch", "avocet", "bunting", "crane", "duck", 27 | "egret", "flamingo", "goose", "heron", "indigo bunting", 28 | "junco", "kestrel", "loon", "mockingbird", "nuthatch", 29 | "owl", "pelican", "raven", "starling", 30 | "tern", "vireo", "wren", "xantus's hummingbird", "yellowthroat", 31 | } 32 | 33 | func NewName() string { 34 | adj := adjectives[mrand.Intn(len(adjectives))] 35 | bird := birds[mrand.Intn(len(birds))] 36 | return adj + "-" + bird 37 | } 38 | -------------------------------------------------------------------------------- /pkg/util/rand/password.go: -------------------------------------------------------------------------------- 1 | package rand 2 | 3 | import ( 4 | "crypto/rand" 5 | "math/big" 6 | ) 7 | 8 | // NewPassword generates a cryptographically secure random password. 9 | // 10 | // The password consists of a mix of lowercase letters, uppercase letters, 11 | // numbers, and symbols. The default length is 16 characters. You can optionally 12 | // provide a `length` argument to specify a different length. 13 | // 14 | // If the provided length is less than or equal to 0, the default length of 16 15 | // is used. 16 | // 17 | // This function uses the `crypto/rand` package to ensure a high level of 18 | // randomness and security, suitable for generating passwords 19 | func NewPassword(length ...int) string { 20 | const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*()-_=+[]{}|;:,.<>?" 21 | passwordLength := 16 // Default length 22 | 23 | if len(length) > 0 { 24 | passwordLength = length[0] 25 | if passwordLength <= 0 { 26 | passwordLength = 16 // Reset to default if invalid length is provided 27 | } 28 | } 29 | 30 | // Use crypto/rand for a cryptographically secure random number generator 31 | random := rand.Reader 32 | 33 | b := make([]byte, passwordLength) 34 | for i := range b { 35 | num, err := rand.Int(random, big.NewInt(int64(len(charset)))) 36 | if err != nil { 37 | // Handle the error appropriately (e.g., panic, log, retry) 38 | panic(err) 39 | } 40 | b[i] = charset[num.Int64()] 41 | } 42 | return string(b) 43 | } 44 | -------------------------------------------------------------------------------- /pkg/x/experiments.go: -------------------------------------------------------------------------------- 1 | package x 2 | 3 | // Experimental stuff 4 | -------------------------------------------------------------------------------- /pkg/x/pgcache/pgcache.go: -------------------------------------------------------------------------------- 1 | package pgcache 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "net" 8 | 9 | pg_query "github.com/pganalyze/pg_query_go/v5" 10 | ) 11 | 12 | func main() { 13 | listener, err := net.Listen("tcp", "localhost:5430") 14 | if err != nil { 15 | log.Fatalf("Error starting TCP server: %v", err) 16 | } 17 | defer listener.Close() 18 | log.Println("Server is listening on localhost:5430") 19 | 20 | for { 21 | conn, err := listener.Accept() 22 | if err != nil { 23 | log.Printf("Error accepting connection: %v", err) 24 | continue 25 | } 26 | go handleClient(conn) 27 | } 28 | } 29 | 30 | func handleClient(clientConn net.Conn) { 31 | defer clientConn.Close() 32 | 33 | serverConn, err := net.Dial("tcp", "localhost:5443") 34 | if err != nil { 35 | log.Printf("Error connecting to PostgreSQL server: %v", err) 36 | return 37 | } 38 | defer serverConn.Close() 39 | 40 | go func() { 41 | // Forward responses from PostgreSQL server to the client 42 | if _, err := io.Copy(clientConn, serverConn); err != nil { 43 | log.Printf("Error forwarding response to client: %v", err) 44 | } 45 | }() 46 | 47 | // Read and parse client queries 48 | buf := make([]byte, 4096) 49 | for { 50 | n, err := clientConn.Read(buf) 51 | if err != nil { 52 | if err != io.EOF { 53 | log.Printf("Error reading from client: %v", err) 54 | } 55 | break 56 | } 57 | 58 | query := string(buf[:n]) 59 | log.Printf("Received data: %s", query) 60 | 61 | // Skip non-SQL messages (like startup messages) 62 | // if !isSQLQuery(query) { 63 | // log.Printf("Skipping non-SQL message: %s", query) 64 | // // Forward the message to the PostgreSQL server 65 | // if _, err := serverConn.Write(buf[:n]); err != nil { 66 | // log.Printf("Error forwarding message to PostgreSQL server: %v", err) 67 | // break 68 | // } 69 | // continue 70 | // } 71 | 72 | // Parse and log metrics 73 | parseQuery(query) 74 | 75 | // Forward the query to the PostgreSQL server 76 | if _, err := serverConn.Write(buf[:n]); err != nil { 77 | log.Printf("Error forwarding query to PostgreSQL server: %v", err) 78 | break 79 | } 80 | } 81 | } 82 | 83 | // func isSQLQuery(query string) bool { 84 | // // Simple heuristic to check if the message is a SQL query 85 | // query = strings.TrimSpace(query) 86 | // if len(query) == 0 { 87 | // return false 88 | // } 89 | 90 | // firstChar := query[0] 91 | // // Check if the first character is likely the start of a SQL command 92 | // return firstChar == 'S' || firstChar == 'I' || firstChar == 'U' || firstChar == 'D' || firstChar == 's' || firstChar == 'i' || firstChar == 'u' || firstChar == 'd' 93 | // } 94 | 95 | func parseQuery(query string) { 96 | tree, err := pg_query.ParseToJSON(query) 97 | if err != nil { 98 | fmt.Println(err) 99 | return 100 | } 101 | fmt.Printf("\n%+v\n", tree) 102 | } 103 | -------------------------------------------------------------------------------- /pkg/x/pgproxy/main.go: -------------------------------------------------------------------------------- 1 | package pgproxy 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "net" 8 | "os" 9 | "os/exec" 10 | ) 11 | 12 | var options struct { 13 | listenAddress string 14 | responseCommand string 15 | } 16 | 17 | func main() { 18 | flag.Usage = func() { 19 | fmt.Fprintf(os.Stderr, "usage: %s [options]\n", os.Args[0]) 20 | flag.PrintDefaults() 21 | } 22 | 23 | flag.StringVar(&options.listenAddress, "listen", "127.0.0.1:15432", "Listen address") 24 | flag.StringVar(&options.responseCommand, "response-command", "echo 'fortune | cowsay -f elephant'", "Command to execute to generate query response") 25 | flag.Parse() 26 | 27 | ln, err := net.Listen("tcp", options.listenAddress) 28 | if err != nil { 29 | log.Fatal(err) 30 | } 31 | log.Println("Listening on", ln.Addr()) 32 | 33 | for { 34 | conn, err := ln.Accept() 35 | if err != nil { 36 | log.Fatal(err) 37 | } 38 | log.Println("Accepted connection from", conn.RemoteAddr()) 39 | 40 | b := NewPgFortuneBackend(conn, func() ([]byte, error) { 41 | return exec.Command("sh", "-c", options.responseCommand).CombinedOutput() 42 | }) 43 | go func() { 44 | err := b.Run() 45 | if err != nil { 46 | log.Println(err) 47 | } 48 | log.Println("Closed connection from", conn.RemoteAddr()) 49 | }() 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /pkg/x/pgproxy/server.go: -------------------------------------------------------------------------------- 1 | package pgproxy 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | 7 | "github.com/jackc/pgx/v5/pgproto3" 8 | ) 9 | 10 | type PgFortuneBackend struct { 11 | backend *pgproto3.Backend 12 | conn net.Conn 13 | responder func() ([]byte, error) 14 | } 15 | 16 | func NewPgFortuneBackend(conn net.Conn, responder func() ([]byte, error)) *PgFortuneBackend { 17 | backend := pgproto3.NewBackend(conn, conn) 18 | 19 | connHandler := &PgFortuneBackend{ 20 | backend: backend, 21 | conn: conn, 22 | responder: responder, 23 | } 24 | 25 | return connHandler 26 | } 27 | 28 | func (p *PgFortuneBackend) Run() error { 29 | defer p.Close() 30 | 31 | err := p.handleStartup() 32 | if err != nil { 33 | return err 34 | } 35 | 36 | for { 37 | msg, err := p.backend.Receive() 38 | if err != nil { 39 | return fmt.Errorf("error receiving message: %w", err) 40 | } 41 | 42 | switch msg.(type) { 43 | case *pgproto3.Query: 44 | response, err := p.responder() 45 | if err != nil { 46 | return fmt.Errorf("error generating query response: %w", err) 47 | } 48 | 49 | buf := mustEncode((&pgproto3.RowDescription{Fields: []pgproto3.FieldDescription{ 50 | { 51 | Name: []byte("fortune"), 52 | TableOID: 0, 53 | TableAttributeNumber: 0, 54 | DataTypeOID: 25, 55 | DataTypeSize: -1, 56 | TypeModifier: -1, 57 | Format: 0, 58 | }, 59 | }}).Encode(nil)) 60 | buf = mustEncode((&pgproto3.DataRow{Values: [][]byte{response}}).Encode(buf)) 61 | buf = mustEncode((&pgproto3.CommandComplete{CommandTag: []byte("SELECT 1")}).Encode(buf)) 62 | buf = mustEncode((&pgproto3.ReadyForQuery{TxStatus: 'I'}).Encode(buf)) 63 | _, err = p.conn.Write(buf) 64 | if err != nil { 65 | return fmt.Errorf("error writing query response: %w", err) 66 | } 67 | case *pgproto3.Terminate: 68 | return nil 69 | default: 70 | return fmt.Errorf("received message other than Query from client: %#v", msg) 71 | } 72 | } 73 | } 74 | 75 | func (p *PgFortuneBackend) handleStartup() error { 76 | startupMessage, err := p.backend.ReceiveStartupMessage() 77 | if err != nil { 78 | return fmt.Errorf("error receiving startup message: %w", err) 79 | } 80 | 81 | fmt.Printf("Received startup message: %#v\n", startupMessage) 82 | 83 | switch startupMessage.(type) { 84 | case *pgproto3.StartupMessage: 85 | buf := mustEncode((&pgproto3.AuthenticationOk{}).Encode(nil)) 86 | buf = mustEncode((&pgproto3.ReadyForQuery{TxStatus: 'I'}).Encode(buf)) 87 | _, err = p.conn.Write(buf) 88 | if err != nil { 89 | return fmt.Errorf("error sending ready for query: %w", err) 90 | } 91 | case *pgproto3.SSLRequest: 92 | _, err = p.conn.Write([]byte("N")) 93 | if err != nil { 94 | return fmt.Errorf("error sending deny SSL request: %w", err) 95 | } 96 | return p.handleStartup() 97 | default: 98 | return fmt.Errorf("unknown startup message: %#v", startupMessage) 99 | } 100 | 101 | return nil 102 | } 103 | 104 | func (p *PgFortuneBackend) Close() error { 105 | return p.conn.Close() 106 | } 107 | 108 | func mustEncode(buf []byte, err error) []byte { 109 | if err != nil { 110 | panic(err) 111 | } 112 | return buf 113 | } 114 | -------------------------------------------------------------------------------- /pkg/x/rag/client_test.go: -------------------------------------------------------------------------------- 1 | package rag 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | 8 | "github.com/jackc/pgx/v5" 9 | ) 10 | 11 | func TestNewClient(t *testing.T) { 12 | ctx := context.Background() 13 | conn, err := pgx.Connect(ctx, os.Getenv("TEST_DATABASE_URL")) 14 | if err != nil { 15 | t.Fatalf("failed to connect to database: %v", err) 16 | } 17 | 18 | client, err := NewClient(conn, Config{ 19 | TableName: "test_table", 20 | Dimensions: 3072, 21 | }) 22 | if err != nil { 23 | t.Fatalf("failed to create client: %v", err) 24 | } 25 | 26 | defer client.conn.Close(ctx) 27 | } 28 | -------------------------------------------------------------------------------- /pkg/x/rag/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package rag provides functions to integrate Retrieval Augmented Generation (RAG) 3 | capabilities in PostgreSQL tables using the pgvector extension. It facilitates 4 | operations for adding embeddings on tables from popular language model APIs such 5 | as OpenAI, Ollama, and LMStudio. 6 | */ 7 | package rag 8 | -------------------------------------------------------------------------------- /pkg/x/rag/embedding.go: -------------------------------------------------------------------------------- 1 | package rag 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | 9 | "github.com/edgeflare/pgo/pkg/httputil" 10 | ) 11 | 12 | // EmbeddingRequest is the request body for the FetchEmbedding function 13 | type EmbeddingRequest struct { 14 | Model string `json:"model"` 15 | Input []string `json:"input"` 16 | } 17 | 18 | // EmbeddingResponse is the response body for the FetchEmbedding function 19 | // https://platform.openai.com/docs/api-reference/embeddings/create 20 | // https://github.com/ollama/ollama/blob/main/docs/api.md#embeddings 21 | type EmbeddingResponse struct { 22 | Data []struct { 23 | Embedding []float32 `json:"embedding"` 24 | } `json:"data"` 25 | } 26 | 27 | // FetchEmbedding fetches embeddings from the LLM API 28 | func (c *Client) FetchEmbedding(ctx context.Context, input []string) ([][]float32, error) { 29 | // check if input is empty 30 | if len(input) == 0 { 31 | return [][]float32{}, fmt.Errorf("input is empty") 32 | } 33 | 34 | data := &EmbeddingRequest{ 35 | Input: input, 36 | Model: c.Config.ModelID, 37 | } 38 | 39 | dataBytes, err := json.Marshal(data) 40 | if err != nil { 41 | return [][]float32{}, fmt.Errorf("failed to marshal request data: %w", err) 42 | } 43 | 44 | config := httputil.DefaultRequestConfig( 45 | http.MethodPost, 46 | fmt.Sprintf("%s%s", c.Config.APIURL, c.Config.EmbeddingsPath), 47 | ) 48 | config.Headers = map[string][]string{ 49 | "Authorization": {fmt.Sprintf("Bearer %s", c.Config.APIKey)}, 50 | } 51 | 52 | response, err := httputil.Request(ctx, config, dataBytes) 53 | if err != nil { 54 | return [][]float32{}, fmt.Errorf("failed to fetch embeddings: %w", err) 55 | } 56 | 57 | var embeddingResponse EmbeddingResponse 58 | if err := json.Unmarshal(response.Body, &embeddingResponse); err != nil { 59 | return [][]float32{}, fmt.Errorf("failed to unmarshal response: %w", err) 60 | } 61 | 62 | embeddings := make([][]float32, len(embeddingResponse.Data)) 63 | for i, d := range embeddingResponse.Data { 64 | embeddings[i] = d.Embedding 65 | } 66 | 67 | return embeddings, nil 68 | } 69 | -------------------------------------------------------------------------------- /pkg/x/rag/embedding_test.go: -------------------------------------------------------------------------------- 1 | package rag 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | ) 7 | 8 | func TestFetchEmbedding(t *testing.T) { 9 | // Create a test client with default config 10 | config := DefaultConfig() 11 | 12 | ctx := context.Background() 13 | conn, err := setupTestDatabase(t) 14 | if err != nil { 15 | t.Fatalf("Failed to set up test database: %v", err) 16 | } 17 | defer conn.Close(ctx) 18 | 19 | c, err := NewClient(conn, config) 20 | if err != nil { 21 | t.Fatalf("Failed to create client: %v", err) 22 | } 23 | 24 | // Test input 25 | input := []string{"Hello", "World"} 26 | 27 | // Call FetchEmbedding 28 | embeddings, err := c.FetchEmbedding(context.Background(), input) 29 | 30 | // Check for errors 31 | if err != nil { 32 | t.Fatalf("FetchEmbedding returned an error: %v", err) 33 | } 34 | 35 | // Check the number of embeddings 36 | if len(embeddings) != len(input) { 37 | t.Errorf("Expected %d embeddings, got %d", len(input), len(embeddings)) 38 | } 39 | 40 | // Check that embeddings are not empty 41 | for i, embedding := range embeddings { 42 | if len(embedding) == 0 { 43 | t.Errorf("Embedding %d is empty", i) 44 | } 45 | } 46 | 47 | // Print the first few values of each embedding for manual inspection 48 | t.Logf("Embeddings (first 5 values):") 49 | for i, embedding := range embeddings { 50 | t.Logf(" Input '%s': %v", input[i], embedding[:5]) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /pkg/x/rag/pgvector_test.go: -------------------------------------------------------------------------------- 1 | package rag 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "testing" 8 | 9 | "github.com/jackc/pgx/v5" 10 | ) 11 | 12 | func TestEnsureTableConfig(t *testing.T) { 13 | ctx := context.Background() 14 | conn, err := setupTestDatabase(t) 15 | if err != nil { 16 | t.Fatalf("Failed to set up test database: %v", err) 17 | } 18 | defer conn.Close(ctx) 19 | 20 | c, err := NewClient(conn, Config{ 21 | TableName: "test_table", 22 | Dimensions: 3072, 23 | TablePrimaryKeyCol: "id", 24 | }) 25 | if err != nil { 26 | t.Fatalf("Failed to create client: %v", err) 27 | } 28 | 29 | // Ensure the test table is dropped before and after the test 30 | dropTable := func() { 31 | _, err := conn.Exec(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s", c.Config.TableName)) 32 | if err != nil { 33 | t.Fatalf("Failed to drop test table: %v", err) 34 | } 35 | } 36 | dropTable() 37 | defer dropTable() 38 | 39 | // Test case 1: Ensure column is added when the table doesn't exist 40 | t.Run("AddColumnWhenTableNotExists", func(t *testing.T) { 41 | err = c.ensureTableConfig(ctx) 42 | if err != nil { 43 | t.Fatalf("Failed to ensure embedding column: %v", err) 44 | } 45 | 46 | // Verify that the table was created with the correct columns 47 | var tableExists bool 48 | err = conn.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name=$1)", c.Config.TableName).Scan(&tableExists) 49 | if err != nil { 50 | t.Fatalf("Failed to check if table exists: %v", err) 51 | } 52 | if !tableExists { 53 | t.Errorf("Table was not created") 54 | } 55 | 56 | // Verify that the table has the correct primary key column 57 | var primaryKeyCol string 58 | err = conn.QueryRow(ctx, "SELECT column_name FROM information_schema.columns WHERE table_name=$1 AND column_name='id'", c.Config.TableName).Scan(&primaryKeyCol) 59 | if err != nil { 60 | t.Fatalf("Failed to check if primary key column exists: %v", err) 61 | } 62 | if primaryKeyCol != c.Config.TablePrimaryKeyCol { 63 | t.Errorf("Primary key column was not added correctly") 64 | } 65 | 66 | // Verify that the embedding column exists 67 | var columnExists bool 68 | err = conn.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name=$1 AND column_name='embedding')", c.Config.TableName).Scan(&columnExists) 69 | if err != nil { 70 | t.Fatalf("Failed to check if column exists: %v", err) 71 | } 72 | if !columnExists { 73 | t.Errorf("Embedding column was not added") 74 | } 75 | 76 | // Verify that the table has the correct content column 77 | var contentCol string 78 | err = conn.QueryRow(ctx, "SELECT column_name FROM information_schema.columns WHERE table_name=$1 AND column_name='content'", c.Config.TableName).Scan(&contentCol) 79 | if err != nil { 80 | t.Fatalf("Failed to check if content column exists: %v", err) 81 | } 82 | if contentCol != "content" { 83 | t.Errorf("Content column was not added correctly") 84 | } 85 | }) 86 | 87 | // Test case 2: Ensure no error when column already exists 88 | t.Run("NoErrorWhenColumnExists", func(t *testing.T) { 89 | err = c.ensureTableConfig(ctx) 90 | if err != nil { 91 | t.Fatalf("Unexpected error when ensuring existing column: %v", err) 92 | } 93 | }) 94 | } 95 | 96 | func setupTestDatabase(t *testing.T) (*pgx.Conn, error) { 97 | ctx := context.Background() 98 | conn, err := pgx.Connect(ctx, os.Getenv("TEST_DATABASE_URL")) 99 | if err != nil { 100 | t.Fatalf("failed to connect to database: %v", err) 101 | } 102 | 103 | return conn, nil 104 | } 105 | -------------------------------------------------------------------------------- /proto/cdc.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package grpc; 4 | 5 | option go_package = "proto/generated"; 6 | 7 | service CDCStream { 8 | rpc Stream(StreamRequest) returns (stream CDCEvent) {} 9 | } 10 | 11 | message StreamRequest {} 12 | 13 | message CDCEvent { 14 | string table = 1; // schema_name.table_name 15 | bytes data = 2; // JSON encoded event.Payload.After data 16 | } --------------------------------------------------------------------------------