├── .github ├── FUNDING.yml ├── dependabot.yml └── workflows │ └── deploy_docs.yml ├── docs ├── .gitignore ├── public │ ├── favicon.ico │ ├── flash-og.png │ ├── favicon-16x16.png │ ├── favicon-32x32.png │ ├── apple-touch-icon.png │ ├── android-chrome-192x192.png │ ├── android-chrome-512x512.png │ ├── site.webmanifest │ └── logo.svg ├── guide │ ├── index.md │ ├── planned-features.md │ ├── start-listening.md │ ├── advanced-features.md │ ├── what-is-flash.md │ ├── drivers │ │ ├── trigger │ │ │ ├── index.md │ │ │ └── WORKFLOW.md │ │ ├── wal_logical │ │ │ ├── index.md │ │ │ └── WORKFLOW.md │ │ └── index.md │ ├── upgrade.md │ ├── installation.md │ └── contributing.md ├── package.json ├── .vitepress │ ├── theme │ │ ├── index.ts │ │ ├── Layout.vue │ │ └── style.css │ └── config.mts ├── team.data.ts ├── team.md └── index.md ├── drivers ├── trigger │ ├── driver_test.go │ ├── go.mod │ ├── driver.go │ └── queries.go └── wal_logical │ ├── driver_test.go │ ├── go.mod │ ├── queries.go │ ├── driver.go │ ├── querying.go │ ├── replicator.go │ └── process.go ├── Makefile ├── .gitignore ├── client_test.go ├── driver.go ├── events.go ├── docker-compose.yaml ├── _examples ├── trigger_insert │ └── main.go ├── parallel_callback │ └── main.go ├── specific_fields │ └── main.go ├── trigger_all │ └── main.go ├── debug_trace │ └── main.go └── development │ └── main.go ├── LICENSE.md ├── README.md ├── operations.go ├── go.mod ├── client.go ├── listener.go ├── operations_test.go └── driver_testcase.go /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: alancolant 2 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | .vitepress/cache 2 | .vitepress/dist 3 | node_modules -------------------------------------------------------------------------------- /docs/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quix-labs/flash/HEAD/docs/public/favicon.ico -------------------------------------------------------------------------------- /docs/public/flash-og.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quix-labs/flash/HEAD/docs/public/flash-og.png -------------------------------------------------------------------------------- /docs/public/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quix-labs/flash/HEAD/docs/public/favicon-16x16.png -------------------------------------------------------------------------------- /docs/public/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quix-labs/flash/HEAD/docs/public/favicon-32x32.png -------------------------------------------------------------------------------- /docs/public/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quix-labs/flash/HEAD/docs/public/apple-touch-icon.png -------------------------------------------------------------------------------- /docs/public/android-chrome-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quix-labs/flash/HEAD/docs/public/android-chrome-192x192.png -------------------------------------------------------------------------------- /docs/public/android-chrome-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/quix-labs/flash/HEAD/docs/public/android-chrome-512x512.png -------------------------------------------------------------------------------- /docs/guide/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | head: 3 | - - meta 4 | - http-equiv: refresh 5 | content: "0;URL=/guide/what-is-flash" 6 | - - link 7 | - rel: canonical 8 | href: /guide/what-is-flash 9 | --- -------------------------------------------------------------------------------- /docs/public/site.webmanifest: -------------------------------------------------------------------------------- 1 | {"name":"Flash (Quix Labs)","short_name":"Flash","icons":[{"src":"/android-chrome-192x192.png","sizes":"192x192","type":"image/png"},{"src":"/android-chrome-512x512.png","sizes":"512x512","type":"image/png"}],"theme_color":"#ea7816","background_color":"#1b1b1f","display":"standalone"} -------------------------------------------------------------------------------- /drivers/trigger/driver_test.go: -------------------------------------------------------------------------------- 1 | package trigger 2 | 3 | import ( 4 | "github.com/quix-labs/flash" 5 | "testing" 6 | ) 7 | 8 | func TestDriver(t *testing.T) { 9 | flash.RunFlashDriverTestCase(t, flash.DefaultDriverTestConfig, func() *Driver { 10 | return NewDriver(&DriverConfig{}) 11 | }) 12 | } 13 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test test-coverage 2 | 3 | test: 4 | go test -v ./... ./drivers/trigger/ ./drivers/wal_logical/ 5 | 6 | test-coverage: 7 | go test -coverprofile=coverage.out ./... ./drivers/trigger/ ./drivers/wal_logical/ 8 | go tool cover -html=coverage.out -o coverage.html 9 | xdg-open coverage.html 10 | -------------------------------------------------------------------------------- /docs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "scripts": { 3 | "docs:dev": "vitepress dev", 4 | "docs:build": "vitepress build", 5 | "docs:preview": "vitepress preview" 6 | }, 7 | "devDependencies": { 8 | "mermaid": "^11.4.0", 9 | "vitepress": "^1.5.0", 10 | "vitepress-plugin-mermaid": "^2.0.17" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Please see the documentation for all configuration options: 2 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 3 | 4 | version: 2 5 | updates: 6 | 7 | - package-ecosystem: "github-actions" 8 | directory: "/" 9 | schedule: 10 | interval: "weekly" 11 | labels: 12 | - "dependencies" -------------------------------------------------------------------------------- /docs/.vitepress/theme/index.ts: -------------------------------------------------------------------------------- 1 | // https://vitepress.dev/guide/custom-theme 2 | import type {Theme} from 'vitepress' 3 | import DefaultTheme from 'vitepress/theme' 4 | import './style.css' 5 | import Layout from "./Layout.vue"; 6 | 7 | export default { 8 | extends: DefaultTheme, 9 | Layout: Layout, 10 | enhanceApp({app, router, siteData}) { 11 | // ... 12 | } 13 | } satisfies Theme 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IDE and editor specific files 2 | .idea/ 3 | .vscode/ 4 | *.swp 5 | *.swo 6 | *.swn 7 | *.bak 8 | 9 | # OS generated files 10 | .DS_Store 11 | Thumbs.db 12 | 13 | # Binaries for programs and plugins 14 | *.exe 15 | *.exe~ 16 | *.dll 17 | *.so 18 | *.dylib 19 | *.test 20 | 21 | # Go specific 22 | /go/bin/ 23 | /go/pkg/ 24 | /go/src/ 25 | go.sum 26 | *.prof 27 | /development 28 | go.work 29 | go.work.sum 30 | coverage.* -------------------------------------------------------------------------------- /client_test.go: -------------------------------------------------------------------------------- 1 | package flash 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestNewClient(t *testing.T) { 8 | t.Skip("TODO") 9 | } 10 | 11 | func TestClientInit(t *testing.T) { 12 | t.Skip("TODO") 13 | } 14 | 15 | func TestClientStart(t *testing.T) { 16 | t.Skip("TODO") 17 | } 18 | 19 | func TestClientClose(t *testing.T) { 20 | t.Skip("TODO") 21 | } 22 | 23 | func TestClientAttach(t *testing.T) { 24 | t.Skip("TODO") 25 | } 26 | -------------------------------------------------------------------------------- /drivers/trigger/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/quix-labs/flash/drivers/trigger 2 | 3 | go 1.21.6 4 | 5 | replace github.com/quix-labs/flash => ../../ 6 | 7 | require ( 8 | github.com/lib/pq v1.10.9 9 | github.com/quix-labs/flash v0.0.0-00010101000000-000000000000 10 | ) 11 | 12 | require ( 13 | github.com/mattn/go-colorable v0.1.13 // indirect 14 | github.com/mattn/go-isatty v0.0.20 // indirect 15 | github.com/rs/zerolog v1.33.0 // indirect 16 | golang.org/x/sys v0.22.0 // indirect 17 | ) 18 | -------------------------------------------------------------------------------- /driver.go: -------------------------------------------------------------------------------- 1 | package flash 2 | 3 | type DatabaseEvent struct { 4 | ListenerUid string 5 | Event Event 6 | } 7 | type DatabaseEventsChan chan *DatabaseEvent 8 | type Driver interface { 9 | Init(clientConfig *ClientConfig) error 10 | Close() error 11 | 12 | HandleOperationListenStart(listenerUid string, listenerConfig *ListenerConfig, operation Operation) error 13 | HandleOperationListenStop(listenerUid string, listenerConfig *ListenerConfig, operation Operation) error 14 | Listen(eventsChan *DatabaseEventsChan) error 15 | } 16 | -------------------------------------------------------------------------------- /docs/team.data.ts: -------------------------------------------------------------------------------- 1 | export default { 2 | load() { 3 | return [{ 4 | avatar: 'https://www.github.com/alancolant.png', 5 | name: 'COLANT Alan', 6 | org: 'Quix Labs', 7 | orgLink: 'https://github.com/quix-labs', 8 | sponsor: 'https://github.com/sponsors/alancolant', 9 | title: 'Maintainer', 10 | links: [ 11 | {icon: 'github', link: 'https://github.com/alancolant'}, 12 | {icon: 'linkedin', link: 'https://www.linkedin.com/in/alancolant/'}, 13 | ] 14 | }]; 15 | } 16 | } -------------------------------------------------------------------------------- /events.go: -------------------------------------------------------------------------------- 1 | package flash 2 | 3 | type EventData map[string]any 4 | type Event interface { 5 | GetOperation() Operation 6 | } 7 | 8 | type InsertEvent struct { 9 | New *EventData 10 | } 11 | type UpdateEvent struct { 12 | Old *EventData 13 | New *EventData 14 | } 15 | type DeleteEvent struct { 16 | Old *EventData 17 | } 18 | type TruncateEvent struct{} 19 | 20 | func (e *InsertEvent) GetOperation() Operation { 21 | return OperationInsert 22 | } 23 | func (e *UpdateEvent) GetOperation() Operation { 24 | return OperationUpdate 25 | } 26 | func (e *DeleteEvent) GetOperation() Operation { 27 | return OperationDelete 28 | } 29 | func (e *TruncateEvent) GetOperation() Operation { 30 | return OperationTruncate 31 | } 32 | -------------------------------------------------------------------------------- /drivers/wal_logical/driver_test.go: -------------------------------------------------------------------------------- 1 | package wal_logical 2 | 3 | import ( 4 | "github.com/quix-labs/flash" 5 | "github.com/testcontainers/testcontainers-go" 6 | "testing" 7 | ) 8 | 9 | func WithWalLogical() testcontainers.CustomizeRequestOption { 10 | return func(req *testcontainers.GenericContainerRequest) error { 11 | req.Cmd = append(req.Cmd, "-c", "wal_level=logical") 12 | return nil 13 | } 14 | } 15 | func TestDriver(t *testing.T) { 16 | driverConfig := flash.DefaultDriverTestConfig 17 | driverConfig.ContainerCustomizers = append(driverConfig.ContainerCustomizers, WithWalLogical()) 18 | flash.RunFlashDriverTestCase(t, driverConfig, func() *Driver { 19 | return NewDriver(&DriverConfig{}) 20 | }) 21 | } 22 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | postgres: 5 | image: postgres:latest 6 | container_name: postgres 7 | environment: 8 | POSTGRES_USER: devuser 9 | POSTGRES_PASSWORD: devpass 10 | POSTGRES_DB: devdb 11 | ports: 12 | - "5432:5432" 13 | volumes: 14 | - postgres-data:/var/lib/postgresql/data 15 | command: 16 | - "postgres" 17 | - "-c" 18 | - "wal_level=logical" 19 | 20 | pgadmin: 21 | image: dpage/pgadmin4:latest 22 | container_name: pgadmin 23 | environment: 24 | PGADMIN_DEFAULT_EMAIL: admin@alancolant.com 25 | PGADMIN_DEFAULT_PASSWORD: admin 26 | ports: 27 | - "8080:80" 28 | depends_on: 29 | - postgres 30 | 31 | volumes: 32 | postgres-data: -------------------------------------------------------------------------------- /docs/team.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | --- 4 | 5 | 13 | 14 | 15 | 16 | 19 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /drivers/wal_logical/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/quix-labs/flash/drivers/wal_logical 2 | 3 | go 1.21.6 4 | 5 | replace github.com/quix-labs/flash => ../../ 6 | 7 | require ( 8 | github.com/jackc/pglogrepl v0.0.0-20240307033717-828fbfe908e9 9 | github.com/jackc/pgx/v5 v5.6.0 10 | github.com/quix-labs/flash v0.0.0-00010101000000-000000000000 11 | ) 12 | 13 | require ( 14 | github.com/jackc/pgio v1.0.0 // indirect 15 | github.com/jackc/pgpassfile v1.0.0 // indirect 16 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect 17 | github.com/mattn/go-colorable v0.1.13 // indirect 18 | github.com/mattn/go-isatty v0.0.20 // indirect 19 | github.com/rs/zerolog v1.33.0 // indirect 20 | golang.org/x/crypto v0.17.0 // indirect 21 | golang.org/x/sys v0.22.0 // indirect 22 | golang.org/x/text v0.14.0 // indirect 23 | ) 24 | -------------------------------------------------------------------------------- /docs/guide/planned-features.md: -------------------------------------------------------------------------------- 1 | # Planned Features 2 | 3 | The following features are planned for future implementation: 4 | 5 | - ⏳ Support for conditional listens. 6 | 7 | | Operator | trigger | wal_logical | 8 | |:--------:|:-----------------:|:-----------------:| 9 | | equals | ✅ | ✅ | 10 | | neq | ❌ | ❌ | 11 | | lt | ❌ | ❌ | 12 | | lte | ❌ | ❌ | 13 | | gte | ❌ | ❌ | 14 | | not null | ❌ | ❌ | 15 | | is null | ⚠️ using eq + nil | ⚠️ using eq + nil | 16 | 17 | - ⏳ Handling custom primary for fake insert/delete when change appears 18 | - ⏳ Tests implementation 19 | - ⬜ Remove client in favor of direct listener start 20 | - ⬜ Support attaching/detaching new listener during runtime. 21 | - ... any suggestions is welcome. 22 | -------------------------------------------------------------------------------- /_examples/trigger_insert/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/quix-labs/flash" 6 | "github.com/quix-labs/flash/drivers/trigger" 7 | ) 8 | 9 | func main() { 10 | postsListener, _ := flash.NewListener(&flash.ListenerConfig{Table: "public.posts"}) 11 | 12 | // Registering your callbacks 13 | stop, err := postsListener.On(flash.OperationInsert, func(event flash.Event) { 14 | typedEvent := event.(*flash.InsertEvent) 15 | fmt.Printf("insert - new: %+v\n", typedEvent.New) 16 | }) 17 | if err != nil { 18 | fmt.Println(err) 19 | } 20 | defer stop() 21 | 22 | flashClient, _ := flash.NewClient(&flash.ClientConfig{ 23 | DatabaseCnx: "postgresql://devuser:devpass@localhost:5432/devdb?sslmode=disable", 24 | Driver: trigger.NewDriver(&trigger.DriverConfig{}), 25 | }) 26 | flashClient.Attach(postsListener) 27 | go flashClient.Start() // Error Handling 28 | defer flashClient.Close() 29 | 30 | select {} // Keep process running 31 | } 32 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) Quix-Labs 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /_examples/parallel_callback/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/quix-labs/flash" 6 | "github.com/quix-labs/flash/drivers/trigger" 7 | ) 8 | 9 | func main() { 10 | postsListener, _ := flash.NewListener(&flash.ListenerConfig{ 11 | Table: "public.posts", 12 | MaxParallelProcess: 50, // Default to 1, you can use -1 for infinite goroutine 13 | }) 14 | 15 | stop, err := postsListener.On(flash.OperationInsert|flash.OperationDelete, func(event flash.Event) { 16 | switch typedEvent := event.(type) { 17 | case *flash.InsertEvent: 18 | fmt.Printf("insert - new: %+v\n", typedEvent.New) 19 | case *flash.DeleteEvent: 20 | fmt.Printf("delete - old: %+v \n", typedEvent.Old) 21 | } 22 | }) 23 | if err != nil { 24 | fmt.Println(err) 25 | } 26 | defer stop() 27 | 28 | flashClient, _ := flash.NewClient(&flash.ClientConfig{ 29 | DatabaseCnx: "postgresql://devuser:devpass@localhost:5432/devdb?sslmode=disable", 30 | Driver: trigger.NewDriver(&trigger.DriverConfig{}), 31 | }) 32 | flashClient.Attach(postsListener) 33 | go flashClient.Start() // Error Handling 34 | defer flashClient.Close() 35 | 36 | select {} // Keep process running 37 | } 38 | -------------------------------------------------------------------------------- /_examples/specific_fields/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/quix-labs/flash" 6 | "github.com/quix-labs/flash/drivers/trigger" 7 | ) 8 | 9 | func main() { 10 | postsListener, _ := flash.NewListener(&flash.ListenerConfig{ 11 | Table: "public.posts", 12 | Fields: []string{"id", "slug"}, 13 | }) 14 | postsListener.On(flash.OperationAll, func(event flash.Event) { 15 | switch typedEvent := event.(type) { 16 | case *flash.InsertEvent: 17 | fmt.Printf("insert - new: %+v\n", typedEvent.New) 18 | case *flash.UpdateEvent: 19 | fmt.Printf("update - old: %+v - new: %+v\n", typedEvent.Old, typedEvent.New) 20 | case *flash.DeleteEvent: 21 | fmt.Printf("delete - old: %+v \n", typedEvent.Old) 22 | case *flash.TruncateEvent: 23 | fmt.Printf("truncate \n") 24 | } 25 | }) 26 | 27 | // Create client 28 | flashClient, _ := flash.NewClient(&flash.ClientConfig{ 29 | DatabaseCnx: "postgresql://devuser:devpass@localhost:5432/devdb?sslmode=disable", 30 | Driver: trigger.NewDriver(&trigger.DriverConfig{}), 31 | }) 32 | flashClient.Attach(postsListener) 33 | 34 | go func() { 35 | err := flashClient.Start() 36 | if err != nil { 37 | panic(err) 38 | } 39 | }() 40 | defer flashClient.Close() 41 | 42 | // Keep process running 43 | select {} 44 | } 45 | -------------------------------------------------------------------------------- /_examples/trigger_all/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/quix-labs/flash" 6 | "github.com/quix-labs/flash/drivers/trigger" 7 | ) 8 | 9 | func main() { 10 | postsListener, _ := flash.NewListener(&flash.ListenerConfig{Table: "public.posts"}) 11 | 12 | // Registering your callbacks -> Can be simplified with types.EventAll 13 | stop, err := postsListener.On(flash.OperationTruncate|flash.OperationInsert|flash.OperationUpdate|flash.OperationDelete, func(event flash.Event) { 14 | switch typedEvent := event.(type) { 15 | case *flash.InsertEvent: 16 | fmt.Printf("insert - new: %+v\n", typedEvent.New) 17 | case *flash.UpdateEvent: 18 | fmt.Printf("update - old: %+v - new: %+v\n", typedEvent.Old, typedEvent.New) 19 | case *flash.DeleteEvent: 20 | fmt.Printf("delete - old: %+v \n", typedEvent.Old) 21 | case *flash.TruncateEvent: 22 | fmt.Printf("truncate \n") 23 | } 24 | }) 25 | if err != nil { 26 | fmt.Println(err) 27 | } 28 | defer stop() 29 | 30 | flashClient, _ := flash.NewClient(&flash.ClientConfig{ 31 | DatabaseCnx: "postgresql://devuser:devpass@localhost:5432/devdb?sslmode=disable", 32 | Driver: trigger.NewDriver(&trigger.DriverConfig{}), 33 | }) 34 | flashClient.Attach(postsListener) 35 | go flashClient.Start() // Error Handling 36 | defer flashClient.Close() 37 | 38 | select {} // Keep process running 39 | } 40 | -------------------------------------------------------------------------------- /docs/public/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 9 | 10 | 12 | 15 | 18 | 19 | -------------------------------------------------------------------------------- /.github/workflows/deploy_docs.yml: -------------------------------------------------------------------------------- 1 | name: Deploy Documentation site to Github Pages 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | paths: 7 | - 'docs/**' 8 | - '.github/workflows/deploy_docs.yml' 9 | workflow_dispatch: 10 | 11 | permissions: 12 | contents: read 13 | pages: write 14 | id-token: write 15 | 16 | concurrency: 17 | group: pages 18 | cancel-in-progress: false 19 | 20 | jobs: 21 | build: 22 | runs-on: ubuntu-latest 23 | steps: 24 | - name: Checkout 25 | uses: actions/checkout@v4 26 | 27 | - name: Setup Pages 28 | uses: actions/configure-pages@v4 29 | 30 | - name: Setup Node 31 | uses: actions/setup-node@v4 32 | with: 33 | node-version: 22 34 | cache: yarn # or pnpm / yarn 35 | cache-dependency-path: docs/yarn.lock 36 | 37 | - name: Install dependencies 38 | working-directory: docs 39 | run: yarn install 40 | 41 | - name: Build with VitePress 42 | working-directory: docs 43 | run: yarn docs:build 44 | 45 | - name: Upload artifact 46 | uses: actions/upload-pages-artifact@v3 47 | with: 48 | path: docs/.vitepress/dist 49 | 50 | # Deployment job 51 | deploy: 52 | environment: 53 | name: github-pages 54 | url: ${{ steps.deployment.outputs.page_url }} 55 | needs: build 56 | runs-on: ubuntu-latest 57 | name: Deploy 58 | steps: 59 | - name: Deploy to GitHub Pages 60 | id: deployment 61 | uses: actions/deploy-pages@v4 -------------------------------------------------------------------------------- /docs/guide/start-listening.md: -------------------------------------------------------------------------------- 1 | # Start listening 2 | 3 | Here's a basic example of how to use Flash: 4 | 5 | ```go 6 | package main 7 | 8 | import ( 9 | "fmt" 10 | "github.com/quix-labs/flash" 11 | "github.com/quix-labs/flash/drivers/trigger" 12 | "os" 13 | "os/signal" 14 | ) 15 | 16 | func main() { 17 | // Example with listener and client setup 18 | postsListener, _ := flash.NewListener(&flash.ListenerConfig{Table: "public.posts"}) 19 | 20 | postsListener.On(flash.OperationAll, func(event flash.Event) { 21 | switch typedEvent := event.(type) { 22 | case *flash.InsertEvent: 23 | fmt.Printf("insert - new: %+v\n", typedEvent.New) 24 | case *flash.UpdateEvent: 25 | fmt.Printf("update - old: %+v - new: %+v\n", typedEvent.Old, typedEvent.New) 26 | case *flash.DeleteEvent: 27 | fmt.Printf("delete - old: %+v \n", typedEvent.Old) 28 | case *flash.TruncateEvent: 29 | fmt.Printf("truncate \n") 30 | } 31 | }) 32 | 33 | // Create client 34 | flashClient, _ := flash.NewClient(&flash.ClientConfig{ 35 | DatabaseCnx: "postgresql://devuser:devpass@localhost:5432/devdb", 36 | Driver: trigger.NewDriver(&trigger.DriverConfig{}), 37 | }) 38 | flashClient.Attach(postsListener) 39 | 40 | // Start listening 41 | go flashClient.Start() 42 | defer flashClient.Close() 43 | 44 | // Wait for interrupt signal (Ctrl+C) 45 | interrupt := make(chan os.Signal, 1) 46 | signal.Notify(interrupt, os.Interrupt) 47 | <-interrupt 48 | 49 | fmt.Println("Program terminated.") 50 | } 51 | 52 | ``` 53 | 54 | ## TODO: How events working ? (Listen for all, for truncate + delete, ...) -------------------------------------------------------------------------------- /_examples/debug_trace/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/quix-labs/flash" 6 | "github.com/quix-labs/flash/drivers/trigger" 7 | "github.com/rs/zerolog" 8 | "os" 9 | ) 10 | 11 | func main() { 12 | 13 | postsListenerConfig := &flash.ListenerConfig{Table: "public.posts"} 14 | postsListener, _ := flash.NewListener(postsListenerConfig) 15 | 16 | // Registering your callbacks 17 | stop, err := postsListener.On(flash.OperationInsert, func(event flash.Event) { 18 | typedEvent := event.(*flash.InsertEvent) 19 | fmt.Printf("Insert received - new: %+v\n", typedEvent.New) 20 | }) 21 | if err != nil { 22 | fmt.Println(err) 23 | } 24 | defer stop() 25 | 26 | // Create custom logger with Level Trace <-> Default is Debug 27 | logger := zerolog.New(os.Stdout).Level(zerolog.TraceLevel).With().Stack().Timestamp().Logger() 28 | driver := trigger.NewDriver(&trigger.DriverConfig{}) 29 | // Create client 30 | clientConfig := &flash.ClientConfig{ 31 | DatabaseCnx: "postgresql://devuser:devpass@localhost:5432/devdb?sslmode=disable", 32 | Logger: &logger, // Define your custom zerolog.Logger here 33 | Driver: driver, 34 | } 35 | 36 | flashClient, err := flash.NewClient(clientConfig) 37 | if err != nil { 38 | fmt.Println(err) 39 | } 40 | flashClient.Attach(postsListener) 41 | 42 | // Start listening 43 | go func() { 44 | err := flashClient.Start() 45 | if err != nil { 46 | panic(err) 47 | } 48 | }() // Error Handling 49 | defer func(flashClient *flash.Client) { 50 | err := flashClient.Close() 51 | if err != nil { 52 | panic(err) 53 | } 54 | }(flashClient) 55 | 56 | // Keep process running 57 | select {} 58 | } 59 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: home 3 | 4 | 5 | hero: 6 | name: "Flash" 7 | text: "Monitor your database events" 8 | tagline: "Without compromising performance" 9 | image: 10 | src: /logo.svg 11 | alt: "Flash Logo" 12 | actions: 13 | - theme: brand 14 | text: "Get Started" 15 | link: /guide/what-is-flash 16 | - theme: alt 17 | text: "View on GitHub" 18 | link: https://github.com/quix-labs/flash 19 | 20 | features: 21 | - title: "Event Filtering" 22 | icon: 🎛️ 23 | details: "Apply custom conditions to receive only the events that matter, reducing noise and improving efficiency." 24 | 25 | - title: "Comprehensive PostgreSQL Support" 26 | icon: 🗄️ 27 | details: "Track essential database events like Insert, Update, Delete, and Truncate with precision and reliability." 28 | 29 | - title: "Parallel Processing" 30 | icon: 🚀 31 | details: "Leverage parallel callback execution using goroutines for maximum performance and efficiency." 32 | 33 | - title: "Lightweight Design" 34 | icon: 🪶 35 | details: "Built with performance in mind, Flash operates with minimal overhead, keeping your systems fast and responsive." 36 | 37 | - title: "WAL Replication Support" 38 | icon: 📡 39 | details: "Utilize Write-Ahead Logging (WAL) replication to ensure accurate and efficient change tracking." 40 | 41 | - title: Open Source 42 | icon: 💻 43 | details: Flash is open-source and welcomes contributions from the community. 44 | 45 | --- 46 | 47 | :::tip Notes 48 | 49 | **This library is currently under active development.** 50 | 51 | Features and APIs may change. 52 | 53 | Contributions and feedback are welcome! 54 | ::: -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Flash 2 | 3 | [![Documentation](https://img.shields.io/github/actions/workflow/status/quix-labs/flash/deploy_docs.yml?label=Documentation)](https://flash.quix-labs.com/guide) 4 | [![License](https://img.shields.io/github/license/quix-labs/flash?color=blue)](https://github.com/quix-labs/flash/blob/main/LICENSE.md) 5 | 6 | **Flash** is a lightweight Go library for managing real-time PostgreSQL changes using event management. 7 | 8 | ## Notes 9 | 10 | **This library is currently under active development.** 11 | 12 | Features and APIs may change. 13 | 14 | Contributions and feedback are welcome! 15 | 16 | ## Features 17 | 18 | - ✅ Start/Stop listening during runtime. 19 | - ✅ Supports common PostgreSQL events: Insert, Update, Delete, Truncate. 20 | - ✅ Driver interfaces for creating new drivers. 21 | - ✅ Parallel Callback execution using goroutine 22 | - ✅ Listen for changes in specific columns, not the entire row. 23 | - ✅ Listen changes using WAL replication 24 | 25 | ## 🌐 Visit Our Website 26 | 27 | For more information, updates, and resources, check out the official website: 28 | 29 | - [Flash Official Website](https://flash.quix-labs.com) 30 | 31 | ## 📚 Documentation 32 | 33 | Our detailed documentation is available to help you get started, learn how to configure and use Flash, and explore 34 | advanced features: 35 | 36 | - [Full Documentation](https://flash.quix-labs.com/guide) 37 | 38 | ## Contributing 39 | 40 | 1. Fork the repository. 41 | 2. Create a new branch for your feature or bugfix. 42 | 3. Commit your changes. 43 | 4. Push your branch. 44 | 5. Create a pull request. 45 | 46 | ## Credits 47 | 48 | - [COLANT Alan](https://github.com/alancolant) 49 | - [All Contributors](../../contributors) 50 | 51 | ## License 52 | 53 | MIT. See the [License File](LICENSE.md) for more information. 54 | -------------------------------------------------------------------------------- /docs/.vitepress/theme/Layout.vue: -------------------------------------------------------------------------------- 1 | 41 | 42 | 45 | 46 | -------------------------------------------------------------------------------- /docs/guide/advanced-features.md: -------------------------------------------------------------------------------- 1 | # Advanced Features 2 | 3 | :::warning ⚠️ Important Notes ⚠️ 4 | 5 | Some of these features may be incompatible with your driver. 6 | 7 | Check [Drivers Overview](./drivers/) to see if the driver you have chosen supports these features. 8 | 9 | ::: 10 | 11 | [//]: # (TODO Better doc) 12 | 13 | For more detailed examples, check out the following files: 14 | 15 | - [Debug queries](https://github.com/quix-labs/flash/tree/main/_examples/debug_trace/main.go) 16 | - [Trigger insert events on table](https://github.com/quix-labs/flash/tree/main/_examples/trigger_insert/main.go) 17 | - [Trigger all events on table](https://github.com/quix-labs/flash/tree/main/_examples/trigger_all/main.go) 18 | - [Listen for specific fields](https://github.com/quix-labs/flash/tree/main/_examples/specific_fields/main.go) 19 | - [Parallel Callback](https://github.com/quix-labs/flash/tree/main/_examples/parallel_callback/main.go) 20 | 21 | 22 | 23 | ## 1. Configurable Primary Key ⏳ 24 | 25 | When you define a primary key, instead of receiving an update event when the column changes, you will receive two 26 | events: 27 | 28 | - A delete event with the old value of this column (and other fields). 29 | - An insert event with the new value of this column (and other fields). 30 | 31 | ## 2. Custom Conditions ⏳ 32 | 33 | You can configure conditions, and if a database row does not match the criteria, you will not receive any event. 34 | 35 | In the case of an update: 36 | 37 | - If the row previously matched the criteria but the new row does not, you will receive a delete event. 38 | - If the row previously did not match the criteria but the new row does, you will receive an insert event. 39 | 40 | ## 3. Partial Fields ✅ 41 | 42 | Ability to listen only to certain columns in your table. If no changes occur in one of these columns, you will not 43 | receive any event. 44 | 45 | -------------------------------------------------------------------------------- /docs/guide/what-is-flash.md: -------------------------------------------------------------------------------- 1 | # What is Flash? 2 | 3 | Flash is a lightweight Go library that monitors and processes real-time changes in PostgreSQL databases. Designed for event-driven architectures, Flash makes it easy to track database events like inserts, updates, deletes, and truncations while minimizing performance overhead. 4 | 5 | Built for developers who need precision and reliability, Flash ensures your database changes are handled efficiently and seamlessly integrated into your application workflows. 6 | 7 |
8 | 9 | Want to try it out? Jump straight to the [Quickstart](./installation). 10 | 11 |
12 | 13 | ## Use Cases 14 | 15 | Flash is perfect for scenarios where real-time database monitoring is essential: 16 | 17 | - **Live Data Dashboards**: Update UI components dynamically as data changes in your database. 18 | - **Event-Driven Architectures**: Trigger workflows or notifications in response to specific database events. 19 | - **Data Syncing**: Sync changes to downstream systems like caches, search engines, or analytics platforms. 20 | - **Audit Logging**: Track and log database modifications for compliance and traceability. 21 | 22 | ## Features 23 | 24 | - ✅ Start/Stop listening during runtime. 25 | - ✅ Supports common PostgreSQL events: Insert, Update, Delete, Truncate. 26 | - ✅ Driver interfaces for creating new drivers. 27 | - ✅ Parallel Callback execution using goroutine 28 | - ✅ Listen for changes in specific columns, not the entire row. (see [Advanced Features](./advanced-features.md)) 29 | - ✅ Listen changes using WAL replication (see [Drivers](./drivers/)) 30 | 31 | 32 | ## Supported Platforms 33 | 34 | **Database**: PostgreSQL 35 | **Drivers**: 36 | - Trigger-based 37 | - WAL-based 38 | 39 | 40 | --- 41 | 42 | Check out the [Quickstart](./installation) and see how easy it is to integrate real-time database monitoring into your Go applications. 43 | -------------------------------------------------------------------------------- /docs/guide/drivers/trigger/index.md: -------------------------------------------------------------------------------- 1 | # Trigger Driver (trigger) 2 | 3 | ## Description 4 | 5 | For each event that is listened to, this driver dynamically creates a trigger that uses `pg_notify` to notify the application. 6 | 7 | This approach can introduce latencies in the database due to the overhead of creating and managing triggers on-the-fly. 8 | 9 | ## Prerequisites 10 | 11 | ### Database Setup 12 | 13 | - No configuration needed, triggers are natively supported in PostgreSQL. 14 | 15 | ## How to Use 16 | 17 | Initialize this driver and pass it to the `clientConfig` Driver parameter. 18 | 19 | ```go 20 | package main 21 | import ( 22 | "github.com/quix-labs/flash/drivers/trigger" 23 | "github.com/quix-labs/flash" 24 | ) 25 | 26 | func main() { 27 | // ... BOOTSTRAPPING 28 | driver := trigger.NewDriver(&trigger.DriverConfig{}) 29 | clientConfig := &flash.ClientConfig{ 30 | DatabaseCnx: "postgresql://devuser:devpass@localhost:5432/devdb", 31 | Driver: driver, 32 | } 33 | // ...START 34 | } 35 | ``` 36 | ## Configuration 37 | 38 | ### Schema 39 | 40 | - **Type**: `string` 41 | - **Default**: `flash` 42 | - **Description**: Must be unique across all your instances. This schema is used to sandbox all created resources. 43 | 44 | ## Notes 45 | 46 | This driver creates a schema. If you have multiple instances without distinct `Schema` values, you may create conflicts between your applications. 47 | 48 | When running multiple clients in parallel, ensure each has unique values for these configurations to avoid conflicts. 49 | 50 | 51 | ## Manually deletion 52 | 53 | If you encounter any artifacts, you can simply drop the PostgreSQL schema with your custom-defined schema or the default `flash`. Use `CASCADE` to ensure triggers are deleted. 54 | 55 | 56 | ## Detailed Information 57 | 58 | ### Advanced Features support 59 | 60 | See [Drivers Overview](../) for compatibility table 61 | 62 | ### Internal workflow 63 | 64 | You can find a workflow graph [here](./WORKFLOW). -------------------------------------------------------------------------------- /docs/guide/upgrade.md: -------------------------------------------------------------------------------- 1 | 2 | # Upgrade from Old Structure 3 | 4 | In the previous structure, our project was divided into three distinct sub-modules, making it cumbersome to manage and 5 | integrate changes. 6 | 7 | We have now merged these sub-modules into a single, unified 8 | module: [github.com/quix-labs/flash](https://github.com/quix-labs/flash). 9 | 10 | This consolidation simplifies the codebase and streamlines development. 11 | 12 | ### Key Changes: 13 | 14 | * **Unified Repository**: 15 | 16 | The previously separate sub-modules are now combined into one repository. 17 | This allows for easier dependency management and a more cohesive development process. 18 | 19 | 20 | * **Separate Driver Installation**: 21 | 22 | While the core functionality is now in one place, the drivers need to be installed separately. 23 | This modular approach ensures that you only include what you need, keeping your projects lightweight. 24 | 25 | * **No default driver**: 26 | 27 | By default, we are previously using trigger driver, to keep user informed, the user require now to instanciate the 28 | driver and pass it in ClientConfig 29 | 30 | ### Upgrade Guide 31 | 32 | * Replace all your `client.NewClient(&type.ClientConfig{})` by `flash.NewClient(&flash.ClientConfig{})` 33 | * Replace all your `listeners.NewListener(types.ListenerConfig{})` by `flash.NewListener(&flash.ListenerConfig{})` 34 | * Instantiate the `Driver` in your codebase and pass it to `flash.ClientConfig{}` 35 | 36 | ```go 37 | package main 38 | 39 | import ( 40 | "github.com/quix-labs/flash" 41 | "github.com/quix-labs/flash/drivers/trigger" 42 | ) 43 | 44 | func main() { 45 | // Instantiation of driver is now required 46 | driver := trigger.NewDriver(&trigger.DriverConfig{}) 47 | client := flash.NewClient(&flash.ClientConfig{ 48 | Driver: driver, 49 | }) 50 | 51 | // Instead of listeners.NewListener, use flash.NewListener 52 | listener := flash.NewListener(&flash.ListenerConfig{}) 53 | 54 | // Your additional code here 55 | } 56 | ``` 57 | 58 | ## Next steps 59 | 60 | Checkout the [Start Listening Guide](./start-listening) to begin. -------------------------------------------------------------------------------- /docs/guide/installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | To get started with Flash, follow the steps below to install the library and set it up in your Go project. 4 | 5 | ## Install in your own project 6 | 7 | ### Step 1: Install Flash 8 | 9 | You can install Flash directly from GitHub using the following `go get` command. The `main` branch is currently used for 10 | development, but the library is stable enough for most use cases: 11 | 12 | ```bash 13 | go get -u github.com/quix-labs/flash@main 14 | ``` 15 | 16 | ### Step 2: Install Additional Drivers (Optional) 17 | 18 | Flash supports a variety of drivers for different configurations. For example, you can install 19 | the [Trigger](./drivers/trigger/) driver with this command: 20 | 21 | ```bash 22 | go get -u github.com/quix-labs/flash/drivers/trigger@main 23 | ``` 24 | 25 | You can explore other drivers in [Drivers Overview](./drivers/) page. 26 | 27 | If you need a specific driver, just install it the same way. 28 | 29 | ### Step 3: Set up Your Main Package 30 | 31 | After installing Flash and the necessary drivers, you can now start using it in your Go project. 32 | 33 | Begin by creating your `main.go` file and importing the Flash library. 34 | 35 | ### Step 4: Run `go mod tidy` (Optional) 36 | 37 | In case there are nested dependencies or missing packages, run the following command to tidy up your Go modules: 38 | 39 | ```bash 40 | go mod tidy 41 | ``` 42 | 43 | This step ensures that all required dependencies are downloaded, and it also removes any unused dependencies. 44 | 45 | ## Troubleshooting 46 | 47 | If you encounter any issues during installation, here are a few things to check: 48 | 49 | - Make sure Go is installed correctly on your system. You can verify this by running: 50 | ```bash 51 | go version 52 | ``` 53 | - Ensure that your `$GOPATH` and `$GOROOT` are set correctly, especially if you're using a custom Go workspace. 54 | - If the installation fails due to permissions, try running the `go get` command with elevated permissions (e.g., `sudo` on Linux or macOS): 55 | ```bash 56 | sudo go get -u github.com/quix-labs/flash@main 57 | ``` 58 | 59 | If you're still having trouble, feel free to open an issue on our [GitHub repository](https://github.com/quix-labs/flash/issues). 60 | 61 | 62 | ## Next Steps 63 | Once you've successfully installed Flash, you're ready to start listening to PostgreSQL database events. 64 | 65 | Check out the [usage guide](./start-listening) to dive deeper into setting up your listeners and configuring your events. -------------------------------------------------------------------------------- /operations.go: -------------------------------------------------------------------------------- 1 | package flash 2 | 3 | import ( 4 | "errors" 5 | "strings" 6 | ) 7 | 8 | type Operation uint8 9 | 10 | const ( 11 | OperationInsert Operation = 1 << iota 12 | OperationUpdate 13 | OperationDelete 14 | OperationTruncate 15 | ) 16 | const ( 17 | OperationAll = OperationInsert | OperationUpdate | OperationDelete | OperationTruncate 18 | ) 19 | 20 | func (o Operation) IsAtomic() bool { 21 | return o == OperationInsert || 22 | o == OperationUpdate || 23 | o == OperationDelete || 24 | o == OperationTruncate 25 | } 26 | 27 | func (o Operation) GetAtomics() []Operation { 28 | var operations []Operation 29 | for mask := OperationInsert; mask != 0 && mask <= OperationTruncate; mask <<= 1 { 30 | if o&mask != 0 { 31 | operations = append(operations, mask) 32 | } 33 | } 34 | return operations 35 | } 36 | 37 | // IncludeAll checks if the current operation includes all specified atomic operations. 38 | func (o Operation) IncludeAll(targetOperation Operation) bool { 39 | return o&targetOperation == targetOperation 40 | } 41 | 42 | // IncludeOne checks if the current operation includes at least one of the specified atomic operations. 43 | func (o Operation) IncludeOne(targetOperation Operation) bool { 44 | return o&targetOperation > 0 45 | } 46 | 47 | // StrictName returns the name of the operation, or throws an error if it doesn't exist 48 | func (o Operation) StrictName() (string, error) { 49 | switch o { 50 | case OperationInsert: 51 | return "INSERT", nil 52 | case OperationUpdate: 53 | return "UPDATE", nil 54 | case OperationDelete: 55 | return "DELETE", nil 56 | case OperationTruncate: 57 | return "TRUNCATE", nil 58 | default: 59 | return "UNKNOWN", errors.New("unknown operation") 60 | } 61 | } 62 | 63 | // Use with caution, because no errors are returned when invalid 64 | func (o Operation) String() string { 65 | if o.IsAtomic() { 66 | name, _ := o.StrictName() 67 | return name 68 | } else { 69 | atomicString := []string{} 70 | for _, atomicOperation := range o.GetAtomics() { 71 | atomicString = append(atomicString, atomicOperation.String()) 72 | } 73 | if len(atomicString) > 1 { 74 | return strings.Join(atomicString, " | ") 75 | } else { 76 | return "UNKNOWN" 77 | } 78 | } 79 | } 80 | 81 | func OperationFromName(name string) (Operation, error) { 82 | switch strings.ToUpper(name) { 83 | case "INSERT": 84 | return OperationInsert, nil 85 | case "UPDATE": 86 | return OperationUpdate, nil 87 | case "DELETE": 88 | return OperationDelete, nil 89 | case "TRUNCATE": 90 | return OperationTruncate, nil 91 | default: 92 | return 0, errors.New("unknown operation name") 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /docs/guide/drivers/wal_logical/index.md: -------------------------------------------------------------------------------- 1 | # WAL Logical Driver (wal_logical) 2 | 3 | ## Description 4 | 5 | This driver operates as a replica slave to intercept the replication logs, capturing changes from the primary database. 6 | 7 | This approach has a minimal impact on the database performance as it leverages PostgreSQL's built-in replication mechanisms. 8 | 9 | ## Prerequisites 10 | 11 | ### Database setup 12 | - Set `replication_level=logical`. 13 | - Set `max_replication_slots` with value of 1 or more. 14 | - Set up your `DatabaseCnx` using a user with replication privileges. 15 | 16 | ## How to Use 17 | 18 | Initialize this driver and pass it to the `clientConfig` Driver parameter. 19 | 20 | ```go 21 | package main 22 | 23 | import ( 24 | "github.com/quix-labs/flash" 25 | "github.com/quix-labs/flash/drivers/wal_logical" 26 | ) 27 | 28 | func main() { 29 | // ... BOOTSTRAPPING 30 | driver := wal_logical.NewDriver(&wal_logical.DriverConfig{}) 31 | clientConfig := &flash.ClientConfig{ 32 | DatabaseCnx: "postgresql://devuser:devpass@localhost:5432/devdb", 33 | Driver: driver, 34 | } 35 | // ...START 36 | } 37 | 38 | ``` 39 | ## Configuration 40 | 41 | 42 | ### PublicationSlotPrefix 43 | 44 | - **Type**: `string` 45 | - **Default**: `flash_publication` 46 | - **Description**: Must be unique across all your instances. This prefix is used to create publication slots in the PostgreSQL database. 47 | 48 | ### ReplicationSlot 49 | - **Type**: `string` 50 | - **Default**: `flash_replication` 51 | - **Description**: Must be unique across all your instances. This slot is used to manage replication data. 52 | 53 | ### UseStreaming 54 | - **Type**: `bool` 55 | - **Default**: false 56 | - **Description**: Allows the usage of streaming for large transactions. Enabling this can have a significant memory impact. 57 | 58 | ## Notes 59 | 60 | This driver creates a replication slot. If you have multiple instances without distinct `PublicationSlotPrefix` and `ReplicationSlot` values, you may create conflicts between your applications. 61 | 62 | When running multiple clients in parallel, ensure each has unique values for these configurations to avoid conflicts. 63 | 64 | ## Known Issues 65 | 66 | * Currently, this driver can crash on restart if it was not properly closed by calling `client.Close()` during shutdown. 67 | 68 | If you encounter this issue, you can manually delete all publication slots from your PostgreSQL instance that start with your defined `PublicationSlotPrefix` or the default fallback `flash_publication`. 69 | 70 | 71 | ## Detailed Information 72 | 73 | ### Advanced Features support 74 | 75 | See [Drivers Overview](../) for compatibility table 76 | 77 | ### Internal workflow 78 | 79 | You can find a workflow graph [here](./WORKFLOW). -------------------------------------------------------------------------------- /docs/guide/drivers/trigger/WORKFLOW.md: -------------------------------------------------------------------------------- 1 | ```mermaid 2 | --- 3 | title: Interaction workflow for trigger driver 4 | legend: TEST 5 | --- 6 | sequenceDiagram 7 | participant Your App 8 | participant Listener 9 | participant Client 10 | participant Driver 11 | participant Database 12 | participant External 13 | rect rgba(34,211,238,0.5) 14 | note over Your App, External: Bootstraping 15 | Your App ->> Listener: on(eventUpdate^EventInsert) 16 | Your App ->> Client: AddListener(listener) 17 | end 18 | rect rgba(250,204,21,0.5) 19 | note over Your App, External: Starting 20 | Your App ->> Client: start() 21 | Client ->> Driver: driver.Init() 22 | Client ->> Driver: driver.Start() 23 | Driver ->> Database: CREATE SCHEMA ... 24 | loop For each actives listeners 25 | Client ->> Listener: Listener.Init() 26 | loop For each listened operations 27 | Listener ->> Client: start listening for operation 28 | Client ->> Driver: send start listening signal for operation 29 | Driver ->> Database: CREATE TRIGGER ... 30 | end 31 | end 32 | end 33 | rect rgba(45,212,191,0.5) 34 | par 35 | note over Your App, External: Change listeners during runtime 36 | loop 37 | Your App ->> Listener: on(eventDelete) 38 | Listener ->> Client: start listening for delete 39 | Client ->> Driver: send listen for delete signal 40 | Driver ->> Database: create trigger on delete 41 | end 42 | note over Your App, External: Listened external operation 43 | loop 44 | External -->> Database: DELETE FROM ... 45 | Database -->> Driver: Send pg_notify events 46 | Driver -->> Client: Dispatch received event 47 | Client -->> Listener: Notify listener 48 | Listener -->> Your App: Event processed 49 | end 50 | note over Your App, External: Un-listened external operation 51 | loop 52 | External -->> Database: UPDATE FROM ... 53 | end 54 | end 55 | end 56 | rect rgba(248,113,113,0.5) 57 | Note over Your App, External: Application Shutdown 58 | Your App ->> Client: stop() 59 | loop For each actives listeners 60 | Client ->> Listener: Listener.Close() 61 | loop For each listened Operation 62 | Listener ->> Client: stop listening for operation 63 | Client ->> Driver: send stop listening signal for operation 64 | Driver ->> Database: DROP TRIGGER ... 65 | end 66 | end 67 | Client ->> Driver: Driver.Close() 68 | Driver ->> Database: DROP SCHEMA ... 69 | end 70 | ``` -------------------------------------------------------------------------------- /docs/.vitepress/theme/style.css: -------------------------------------------------------------------------------- 1 | :root { 2 | --vp-c-default-1: var(--vp-c-gray-1); 3 | --vp-c-default-2: var(--vp-c-gray-2); 4 | --vp-c-default-3: var(--vp-c-gray-3); 5 | --vp-c-default-soft: var(--vp-c-gray-soft); 6 | 7 | --vp-c-brand-1: #ea7816; 8 | --vp-c-brand-2: #f19d33; 9 | --vp-c-brand-3: #ea7816; 10 | --vp-c-brand-soft: #f7d29060; 11 | 12 | --vp-c-tip-1: var(--vp-c-brand-1); 13 | --vp-c-tip-2: var(--vp-c-brand-2); 14 | --vp-c-tip-3: var(--vp-c-brand-3); 15 | --vp-c-tip-soft: var(--vp-c-brand-soft); 16 | 17 | --vp-c-warning-1: var(--vp-c-yellow-1); 18 | --vp-c-warning-2: var(--vp-c-yellow-2); 19 | --vp-c-warning-3: var(--vp-c-yellow-3); 20 | --vp-c-warning-soft: var(--vp-c-yellow-soft); 21 | 22 | --vp-c-danger-1: var(--vp-c-red-1); 23 | --vp-c-danger-2: var(--vp-c-red-2); 24 | --vp-c-danger-3: var(--vp-c-red-3); 25 | --vp-c-danger-soft: var(--vp-c-red-soft); 26 | } 27 | 28 | .dark { 29 | --vp-c-brand-1: #ea7816; 30 | --vp-c-brand-2: #f19d33; 31 | --vp-c-brand-3: #cf5710; 32 | --vp-c-brand-soft: #f19d3320; 33 | } 34 | 35 | /** 36 | * Component: Button 37 | * -------------------------------------------------------------------------- */ 38 | 39 | :root { 40 | --vp-button-brand-border: transparent; 41 | --vp-button-brand-text: var(--vp-c-white); 42 | --vp-button-brand-bg: var(--vp-c-brand-3); 43 | --vp-button-brand-hover-border: transparent; 44 | --vp-button-brand-hover-text: var(--vp-c-white); 45 | --vp-button-brand-hover-bg: var(--vp-c-brand-2); 46 | --vp-button-brand-active-border: transparent; 47 | --vp-button-brand-active-text: var(--vp-c-white); 48 | --vp-button-brand-active-bg: var(--vp-c-brand-1); 49 | } 50 | 51 | /** 52 | * Component: Home 53 | * -------------------------------------------------------------------------- */ 54 | 55 | :root { 56 | --vp-home-hero-name-color: transparent; 57 | --vp-home-hero-name-background: -webkit-linear-gradient( 58 | 120deg, 59 | var(--vp-c-brand-1) 30%, 60 | var(--vp-c-brand-2) 61 | ); 62 | 63 | --vp-home-hero-image-background-image: linear-gradient( 64 | 45deg, 65 | #efc433 50%, 66 | #f19d33 50% 67 | ); 68 | --vp-home-hero-image-filter: blur(44px) opacity(0.3); 69 | } 70 | 71 | @media (min-width: 640px) { 72 | :root { 73 | --vp-home-hero-image-filter: blur(56px) opacity(0.3); 74 | } 75 | } 76 | 77 | @media (min-width: 960px) { 78 | :root { 79 | --vp-home-hero-image-filter: blur(68px) opacity(0.3); 80 | } 81 | } 82 | 83 | /** 84 | * Component: Custom Block 85 | * -------------------------------------------------------------------------- */ 86 | 87 | :root { 88 | --vp-custom-block-tip-border: transparent; 89 | --vp-custom-block-tip-text: var(--vp-c-text-1); 90 | --vp-custom-block-tip-bg: var(--vp-c-brand-soft); 91 | --vp-custom-block-tip-code-bg: var(--vp-c-brand-soft); 92 | } 93 | 94 | -------------------------------------------------------------------------------- /drivers/wal_logical/queries.go: -------------------------------------------------------------------------------- 1 | package wal_logical 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "github.com/jackc/pgx/v5/pgconn" 8 | "github.com/quix-labs/flash" 9 | "strings" 10 | ) 11 | 12 | func (d *Driver) getFullSlotName(slotName string) string { 13 | return d.Config.PublicationSlotPrefix + "-" + slotName 14 | } 15 | 16 | func (d *Driver) getCreatePublicationSlotSql(fullSlotName string, config *flash.ListenerConfig, operation *flash.Operation) (string, error) { 17 | if config == nil { 18 | return fmt.Sprintf(`CREATE PUBLICATION "%s";`, fullSlotName), nil 19 | } 20 | 21 | rawSql := d.getDropPublicationSlotSql(fullSlotName) 22 | // SET REPLICA IDENTITY TO FULL ON CREATION 23 | quotedTableName := d.sanitizeTableName(config.Table, true) 24 | rawSql += fmt.Sprintf(`ALTER TABLE %s REPLICA IDENTITY FULL;CREATE PUBLICATION "%s" FOR TABLE %s`, quotedTableName, fullSlotName, quotedTableName) 25 | 26 | if operation != nil { 27 | //TODO THROW ERROR IF NOT ATOMIC OR JOIN EACH ATOMIC (see .getAlterPublicationEventsSql() ) 28 | operationName, err := operation.StrictName() 29 | if err != nil { 30 | return "", err 31 | } 32 | rawSql += fmt.Sprintf(` WITH (publish = '%s')`, strings.ToLower(operationName)) 33 | } 34 | return rawSql + ";", nil 35 | } 36 | 37 | func (d *Driver) getAlterPublicationEventsSql(publication *activePublication) (string, error) { 38 | if publication == nil { 39 | return "", errors.New("publication is nil") 40 | } 41 | 42 | var rawOperations []string 43 | for _, targetOperation := range publication.operations.GetAtomics() { 44 | operation, err := targetOperation.StrictName() 45 | if err != nil { 46 | return "", err 47 | } 48 | rawOperations = append(rawOperations, strings.ToLower(operation)) 49 | } 50 | 51 | return fmt.Sprintf(`ALTER PUBLICATION "%s" SET (publish = '%s');`, publication.slotName, strings.Join(rawOperations, ", ")), nil 52 | } 53 | 54 | func (d *Driver) getDropPublicationSlotSql(fullSlotName string) string { 55 | return fmt.Sprintf(`DROP PUBLICATION IF EXISTS "%s";`, fullSlotName) 56 | } 57 | 58 | // Returns tablename as format public.posts. 59 | // posts -> public.posts 60 | // "stats"."name" -> stats.name 61 | // public."posts" -> public.posts 62 | func (d *Driver) sanitizeTableName(tableName string, quote bool) string { 63 | splits := strings.Split(tableName, ".") 64 | if len(splits) == 1 { 65 | splits = []string{"public", strings.ReplaceAll(splits[0], `"`, "")} 66 | } else { 67 | splits = []string{strings.ReplaceAll(splits[0], `"`, ""), strings.ReplaceAll(splits[1], `"`, "")} 68 | } 69 | 70 | if quote { 71 | splits[0] = `"` + splits[0] + `"` 72 | splits[1] = `"` + splits[1] + `"` 73 | } 74 | return strings.Join(splits, ".") 75 | } 76 | 77 | func (d *Driver) sqlExec(conn *pgconn.PgConn, query string) ([]*pgconn.Result, error) { 78 | d._clientConfig.Logger.Trace().Str("query", query).Msg("sending sql request") 79 | result := conn.Exec(context.Background(), query) 80 | return result.ReadAll() 81 | } 82 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/quix-labs/flash 2 | 3 | go 1.21.6 4 | 5 | require ( 6 | github.com/rs/zerolog v1.33.0 7 | github.com/testcontainers/testcontainers-go v0.32.0 8 | github.com/testcontainers/testcontainers-go/modules/postgres v0.32.0 9 | ) 10 | 11 | require ( 12 | dario.cat/mergo v1.0.0 // indirect 13 | github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect 14 | github.com/Microsoft/go-winio v0.6.2 // indirect 15 | github.com/Microsoft/hcsshim v0.11.5 // indirect 16 | github.com/cenkalti/backoff/v4 v4.2.1 // indirect 17 | github.com/containerd/containerd v1.7.18 // indirect 18 | github.com/containerd/errdefs v0.1.0 // indirect 19 | github.com/containerd/log v0.1.0 // indirect 20 | github.com/cpuguy83/dockercfg v0.3.1 // indirect 21 | github.com/distribution/reference v0.6.0 // indirect 22 | github.com/docker/docker v27.0.3+incompatible // indirect 23 | github.com/docker/go-connections v0.5.0 // indirect 24 | github.com/docker/go-units v0.5.0 // indirect 25 | github.com/felixge/httpsnoop v1.0.4 // indirect 26 | github.com/go-logr/logr v1.4.1 // indirect 27 | github.com/go-logr/stdr v1.2.2 // indirect 28 | github.com/go-ole/go-ole v1.2.6 // indirect 29 | github.com/gogo/protobuf v1.3.2 // indirect 30 | github.com/golang/protobuf v1.5.4 // indirect 31 | github.com/google/uuid v1.6.0 // indirect 32 | github.com/klauspost/compress v1.17.4 // indirect 33 | github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect 34 | github.com/magiconair/properties v1.8.7 // indirect 35 | github.com/mattn/go-colorable v0.1.13 // indirect 36 | github.com/mattn/go-isatty v0.0.20 // indirect 37 | github.com/moby/docker-image-spec v1.3.1 // indirect 38 | github.com/moby/patternmatcher v0.6.0 // indirect 39 | github.com/moby/sys/sequential v0.5.0 // indirect 40 | github.com/moby/sys/user v0.1.0 // indirect 41 | github.com/moby/term v0.5.0 // indirect 42 | github.com/morikuni/aec v1.0.0 // indirect 43 | github.com/opencontainers/go-digest v1.0.0 // indirect 44 | github.com/opencontainers/image-spec v1.1.0 // indirect 45 | github.com/pkg/errors v0.9.1 // indirect 46 | github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect 47 | github.com/shirou/gopsutil/v3 v3.23.12 // indirect 48 | github.com/shoenig/go-m1cpu v0.1.6 // indirect 49 | github.com/sirupsen/logrus v1.9.3 // indirect 50 | github.com/tklauser/go-sysconf v0.3.12 // indirect 51 | github.com/tklauser/numcpus v0.6.1 // indirect 52 | github.com/yusufpapurcu/wmi v1.2.3 // indirect 53 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect 54 | go.opentelemetry.io/otel v1.24.0 // indirect 55 | go.opentelemetry.io/otel/metric v1.24.0 // indirect 56 | go.opentelemetry.io/otel/trace v1.24.0 // indirect 57 | golang.org/x/crypto v0.22.0 // indirect 58 | golang.org/x/sys v0.22.0 // indirect 59 | google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect 60 | google.golang.org/grpc v1.59.0 // indirect 61 | google.golang.org/protobuf v1.33.0 // indirect 62 | ) 63 | -------------------------------------------------------------------------------- /docs/guide/drivers/index.md: -------------------------------------------------------------------------------- 1 | # Drivers Overview 2 | 3 | ## Implemented 4 | 5 | | Name | DB impact | Operations | Configurable primary key | Custom Conditions | Partial Fields | Graceful Shutdown/Restart | 6 | |-------------------------------|:------------:|:----------:|:------------------------:|:-----------------:|:--------------:|:--------------------------------------------------------------:| 7 | | [trigger](./trigger/) | high ⚠️ | All | not implemented | ✅ | ✅ | ✅ | 8 | | [wal_logical](./wal_logical/) | low ⚡ | All | not implemented | ✅ | ✅ | partial ⚠️
cannot restart if crash without client.Close() | 9 | 10 | ## NOT IMPLEMENTED 11 | 12 | ### GLOBAL UPDATE/DELETE/INSERT TRIGGER + TRUNCATE TRIGGER FOR EACH ROW *(Seems legit)* 13 | 14 | #### Bootstrapping 15 | 16 | - Generation of a unique name: 17 | - If TRUNCATE: Unique reference + truncate -> e.g., flash_posts_truncate 18 | - Otherwise, Unique reference + other -> e.g., flash_posts_other 19 | 20 | - Creation: 21 | - If TRUNCATE -> CREATE TRIGGER ON ... BEFORE TRUNCATE FOR EACH STATEMENT ... 22 | - Otherwise: 23 | - If a global trigger already exists -> ignore 24 | - If no global trigger is registered, create it -> CREATE TRIGGER ON ... BEFORE UPDATE, DELETE, INSERT FOR EACH 25 | STATEMENT ... 26 | - Iterate over old_table and new_table -> for each entry call pg_notify passing TG_OP 27 | 28 | #### Event Reception 29 | 30 | In this case, we will receive unlistened events. 31 | 32 | We need to check if the received event is in the list of listened events. 33 | 34 | - If yes, send it to the callback 35 | - If not, ignore it 36 | 37 | ___ 38 | 39 | ### GLOBAL UPDATE/DELETE/INSERT TRIGGER + TRUNCATE TRIGGER [FOR EACH STATEMENT] *(Seems Legit)* 40 | 41 | #### Bootstrapping 42 | 43 | - Like Approach 2 but instead of calling pg_notify for each row, generate a JSON array and send the complete payload 44 | only once 45 | 46 | #### Event Reception 47 | 48 | - Like Approach 2 but if we receive the payload, decode it and iterate over each entry to send an event for each entry 49 | 50 | ___ 51 | 52 | ### PG EXTENSION - *(FURTHER THOUGHT REQUIRED)* 53 | 54 | #### Bootstrapping 55 | 56 | - CREATION: 57 | - Call custom function to listen 58 | - DELETION: 59 | - Call custom function to stop listening 60 | 61 | #### Event Reception 62 | 63 | - Retrieve the emitted event 64 | - Forward it to the callback 65 | 66 | ___ 67 | 68 | ### ~~GATEWAY (Rejected)~~ 69 | 70 | #### Bootstrapping 71 | 72 | - Open a TCP port 73 | - Intercept SQL queries 74 | 75 | #### Event Reception 76 | 77 | - Parse the SQL query 78 | - Detect the altered rows 79 | - If listening: forward it to the callback 80 | - Otherwise: ignore it 81 | 82 | #### Rejection Reason 83 | 84 | For UPDATE FROM (SELECT id from posts) queries, it is impossible to track the rows without making database queries. 85 | -------------------------------------------------------------------------------- /docs/guide/contributing.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thank you for your interest in contributing to Flash! We welcome contributions from the community. To ensure a smooth process for everyone, please follow these guidelines when contributing. 4 | 5 | ## How to Contribute 6 | 7 | ### 1. Fork the Repository 8 | 9 | Start by forking the repository to your own GitHub account. This allows you to make changes without affecting the main project. 10 | 11 | - Navigate to the [Flash repository](https://github.com/quix-labs/flash). 12 | - Click on the **Fork** button at the top right of the page to create a copy in your own GitHub account. 13 | 14 | ### 2. Create a New Branch 15 | 16 | Once you've forked the repository, create a new branch for the changes you want to make. It's important to keep your changes isolated in a separate branch. 17 | 18 | You can create a branch from the command line like this: 19 | 20 | `git checkout -b my-feature-branch` 21 | 22 | Make sure to give your branch a descriptive name related to the feature or bugfix you're working on. 23 | 24 | ### 3. Make Your Changes 25 | 26 | Edit, refactor, or improve the code as needed. Be sure to follow the project's coding style and best practices. 27 | 28 | - If you're adding a new feature, consider writing tests to cover your changes. 29 | - If you're fixing a bug, ensure that your fix solves the issue without introducing new problems. 30 | 31 | ### 4. Commit Your Changes 32 | 33 | Once you've made your changes, commit them with a clear, concise commit message that describes what you've done. Use conventional commit messages to keep the history clean and understandable. 34 | 35 | For example: 36 | 37 | `git commit -m "Add feature to listen for specific columns"` 38 | 39 | ### 5. Push Your Branch 40 | 41 | After committing your changes, push your branch to your forked repository on GitHub: 42 | 43 | `git push origin my-feature-branch` 44 | 45 | ### 6. Open a Pull Request 46 | 47 | Once your changes are pushed to your forked repository, open a pull request (PR) to merge your changes into the main repository. 48 | 49 | - Go to the original repository (not your fork) on GitHub. 50 | - Click on the **New Pull Request** button. 51 | - Select your feature branch and the `main` branch of the repository as the base. 52 | - Provide a detailed description of what your PR does and any context or explanations for your changes. 53 | 54 | ### 7. Review and Feedback 55 | 56 | Once your PR is submitted, the maintainers will review it. They may request changes or provide feedback. Be open to feedback and make the necessary adjustments. 57 | 58 | ### 8. Merging 59 | 60 | After your PR has been reviewed and approved, a maintainer will merge it into the main repository. 61 | 62 | 63 | ## Code of Conduct 64 | 65 | Please be respectful and kind to others when contributing. We want to maintain a positive and welcoming environment for all contributors. 66 | 67 | - **Be respectful**: Treat everyone with respect and consideration. 68 | - **Be collaborative**: Work with others to create the best solutions. 69 | - **Be inclusive**: Embrace diversity and welcome contributions from everyone. 70 | 71 | 72 | Thank you for contributing to Flash! Your help makes the project better for everyone. 73 | -------------------------------------------------------------------------------- /_examples/development/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/quix-labs/flash" 6 | "github.com/quix-labs/flash/drivers/wal_logical" 7 | "github.com/rs/zerolog" 8 | "os" 9 | "runtime/pprof" 10 | "sync" 11 | "time" 12 | ) 13 | 14 | func main() { 15 | f, err := os.Create("myprogram.prof") 16 | if err != nil { 17 | panic(err) 18 | } 19 | pprof.StartCPUProfile(f) 20 | defer pprof.StopCPUProfile() 21 | 22 | postsListenerConfig := &flash.ListenerConfig{ 23 | Table: "public.posts", 24 | MaxParallelProcess: 1, // In most case 1 is ideal because sync between goroutine introduce some delay 25 | Fields: []string{"id", "slug"}, 26 | Conditions: []*flash.ListenerCondition{{Column: "active", Value: true}}, 27 | } 28 | postsListener, _ := flash.NewListener(postsListenerConfig) 29 | 30 | postsListener2Config := &flash.ListenerConfig{ 31 | Table: "public.posts", 32 | MaxParallelProcess: 1, // In most case 1 is ideal because sync between goroutine introduce some delay 33 | Fields: []string{"active"}, 34 | Conditions: []*flash.ListenerCondition{{Column: "slug", Value: nil}}, 35 | } 36 | postsListener2, _ := flash.NewListener(postsListener2Config) 37 | 38 | // Registering your callbacks 39 | var i = 0 40 | var mutex sync.Mutex 41 | 42 | stopAll, err := postsListener.On(flash.OperationAll, func(event flash.Event) { 43 | mutex.Lock() 44 | i++ 45 | mutex.Unlock() 46 | 47 | //switch typedEvent := event.(type) { 48 | //case *flash.InsertEvent: 49 | // fmt.Printf("insert - new: %+v\n", typedEvent.New) 50 | //case *flash.UpdateEvent: 51 | // fmt.Printf("update - old: %+v - new: %+v\n", typedEvent.Old, typedEvent.New) 52 | //case *flash.DeleteEvent: 53 | // fmt.Printf("delete - old: %+v \n", typedEvent.Old) 54 | //case *flash.TruncateEvent: 55 | // fmt.Printf("truncate \n") 56 | //} 57 | }) 58 | if err != nil { 59 | panic(err) 60 | } 61 | 62 | defer func() { 63 | err := stopAll() 64 | if err != nil { 65 | panic(err) 66 | } 67 | }() 68 | 69 | stopAll2, err := postsListener2.On(flash.OperationAll, func(event flash.Event) { 70 | mutex.Lock() 71 | i++ 72 | mutex.Unlock() 73 | 74 | //switch typedEvent := event.(type) { 75 | //case *flash.InsertEvent: 76 | // fmt.Printf("2-insert - new: %+v\n", typedEvent.New) 77 | //case *flash.UpdateEvent: 78 | // fmt.Printf("2-update - old: %+v - new: %+v\n", typedEvent.Old, typedEvent.New) 79 | //case *flash.DeleteEvent: 80 | // fmt.Printf("2-delete - old: %+v \n", typedEvent.Old) 81 | //case *flash.TruncateEvent: 82 | // fmt.Printf("2-truncate \n") 83 | //} 84 | }) 85 | if err != nil { 86 | panic(err) 87 | } 88 | 89 | defer func() { 90 | err := stopAll2() 91 | if err != nil { 92 | panic(err) 93 | } 94 | }() 95 | 96 | go func() { 97 | for { 98 | time.Sleep(time.Second * 1) 99 | mutex.Lock() 100 | fmt.Println(i) 101 | i = 0 102 | mutex.Unlock() 103 | } 104 | }() 105 | 106 | // Create custom logger 107 | logger := zerolog.New(os.Stdout).Level(zerolog.TraceLevel).With().Caller().Stack().Timestamp().Logger() 108 | 109 | driver := wal_logical.NewDriver(&wal_logical.DriverConfig{ 110 | //UseStreaming: true, 111 | }) 112 | 113 | // Create client 114 | clientConfig := &flash.ClientConfig{ 115 | DatabaseCnx: "postgresql://devuser:devpass@localhost:5432/devdb?sslmode=disable", 116 | Logger: &logger, // Define your custom zerolog.Logger here 117 | ShutdownTimeout: time.Second * 2, 118 | Driver: driver, 119 | } 120 | flashClient, _ := flash.NewClient(clientConfig) 121 | flashClient.Attach(postsListener, postsListener2) 122 | 123 | // Start listening 124 | go func() { 125 | err := flashClient.Start() 126 | if err != nil { 127 | panic(err) 128 | } 129 | }() // Error Handling 130 | 131 | defer func() { 132 | err := flashClient.Close() 133 | if err != nil { 134 | panic(err) 135 | } 136 | }() 137 | 138 | select {} 139 | // 140 | //// Wait for interrupt signal (Ctrl+C) 141 | //interrupt := make(chan os.Signal, 1) 142 | //signal.Notify(interrupt, os.Interrupt) 143 | //<-interrupt 144 | // 145 | //fmt.Println("Program terminated.") 146 | } 147 | -------------------------------------------------------------------------------- /drivers/wal_logical/driver.go: -------------------------------------------------------------------------------- 1 | package wal_logical 2 | 3 | import ( 4 | "github.com/jackc/pgx/v5/pgconn" 5 | "github.com/quix-labs/flash" 6 | ) 7 | 8 | type DriverConfig struct { 9 | PublicationSlotPrefix string // Default to flash_publication -> Must be unique across all your instances 10 | ReplicationSlot string // Default to flash_replication -> Must be unique across all your instances 11 | UseStreaming bool // Default to false -> allow usage of stream for big transaction, can have big memory impact 12 | } 13 | 14 | var ( 15 | _ flash.Driver = (*Driver)(nil) // Interface implementation 16 | ) 17 | 18 | func NewDriver(config *DriverConfig) *Driver { 19 | if config == nil { 20 | config = &DriverConfig{} 21 | } 22 | if config.PublicationSlotPrefix == "" { 23 | config.PublicationSlotPrefix = "flash_publication" 24 | } 25 | if config.ReplicationSlot == "" { 26 | config.ReplicationSlot = "flash_replication" 27 | } 28 | return &Driver{ 29 | Config: config, 30 | activeListeners: make(map[string]map[string]*flash.ListenerConfig), 31 | } 32 | } 33 | 34 | // TODO 35 | type PublicationState map[string]*struct { 36 | listenedEvents []flash.Operation 37 | listenerMapping map[flash.Operation]struct { 38 | _listenerUid *string 39 | _config *flash.ListenerConfig 40 | } 41 | } 42 | 43 | type Driver struct { 44 | Config *DriverConfig 45 | 46 | queryConn *pgconn.PgConn 47 | 48 | // Replication handling 49 | replicationConn *pgconn.PgConn 50 | 51 | replicationState *replicationState 52 | activePublications map[string]bool 53 | activeListeners map[string]map[string]*flash.ListenerConfig // key 1: tableName -> key 2: listenerUid 54 | 55 | eventsChan *flash.DatabaseEventsChan 56 | 57 | subscriptionState *subscriptionState 58 | _clientConfig *flash.ClientConfig 59 | } 60 | 61 | func (d *Driver) Init(clientConfig *flash.ClientConfig) error { 62 | d._clientConfig = clientConfig 63 | 64 | if err := d.initQuerying(); err != nil { 65 | return err 66 | } 67 | 68 | if err := d.initReplicator(); err != nil { 69 | return err 70 | } 71 | 72 | return nil 73 | } 74 | 75 | func (d *Driver) HandleOperationListenStart(listenerUid string, listenerConfig *flash.ListenerConfig, event flash.Operation) error { 76 | tableName := d.sanitizeTableName(listenerConfig.Table, false) 77 | 78 | //TODO ALTER PUBLICATION noinsert SET (publish = 'update, delete'); 79 | if _, exists := d.activeListeners[tableName]; !exists { 80 | d.activeListeners[tableName] = make(map[string]*flash.ListenerConfig) 81 | } 82 | 83 | // Keep in goroutine because channel is listened on start 84 | go func() { 85 | d.subscriptionState.subChan <- &subscriptionClaim{ 86 | listenerUid: listenerUid, 87 | listenerConfig: listenerConfig, 88 | operation: &event, 89 | } 90 | }() 91 | 92 | d.activeListeners[tableName][listenerUid] = listenerConfig //TODO MORE PERFORMANT STRUCTURE 93 | return nil 94 | } 95 | 96 | func (d *Driver) HandleOperationListenStop(listenerUid string, listenerConfig *flash.ListenerConfig, event flash.Operation) error { 97 | tableName := d.sanitizeTableName(listenerConfig.Table, false) 98 | 99 | // Keep in goroutine because channel is listened on start 100 | go func() { 101 | d.subscriptionState.unsubChan <- &subscriptionClaim{ 102 | listenerUid: listenerUid, 103 | listenerConfig: listenerConfig, 104 | operation: &event, 105 | } 106 | }() 107 | 108 | delete(d.activeListeners[tableName], listenerUid) //TODO MORE PERFORMANT STRUCTURE 109 | return nil 110 | } 111 | 112 | func (d *Driver) Listen(eventsChan *flash.DatabaseEventsChan) error { 113 | d.eventsChan = eventsChan 114 | 115 | var errChan = make(chan error, 1) 116 | var readyChan = make(chan struct{}, 1) 117 | 118 | go func() { 119 | if err := d.startQuerying(&readyChan); err != nil { 120 | errChan <- err 121 | } 122 | }() 123 | 124 | select { 125 | case err := <-errChan: 126 | return err 127 | case <-readyChan: 128 | break 129 | } 130 | 131 | go func() { 132 | if err := d.startReplicator(); err != nil { 133 | errChan <- err 134 | } 135 | }() 136 | 137 | for { 138 | select { 139 | case err := <-errChan: 140 | return err 141 | } 142 | } 143 | } 144 | 145 | func (d *Driver) Close() error { 146 | err := d.closeQuerying() 147 | if err != nil { 148 | return err 149 | } 150 | return d.closeReplicator() 151 | } 152 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package flash 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "github.com/rs/zerolog" 8 | "os" 9 | "strings" 10 | "sync" 11 | "time" 12 | ) 13 | 14 | type ClientConfig struct { 15 | DatabaseCnx string 16 | Driver Driver 17 | Logger *zerolog.Logger 18 | 19 | ShutdownTimeout time.Duration 20 | } 21 | 22 | type Client struct { 23 | Config *ClientConfig 24 | listeners map[string]*Listener 25 | } 26 | 27 | func NewClient(config *ClientConfig) (*Client, error) { 28 | if config == nil { 29 | return nil, errors.New("config required") 30 | } 31 | if config.DatabaseCnx == "" { 32 | return nil, errors.New("database connection required") 33 | } 34 | if config.Driver == nil { 35 | return nil, errors.New("driver required") 36 | } 37 | if config.Logger == nil { 38 | logger := zerolog.New(os.Stdout).Level(zerolog.DebugLevel).With().Stack().Timestamp().Logger() 39 | config.Logger = &logger 40 | } 41 | if config.ShutdownTimeout == time.Duration(0) { 42 | config.ShutdownTimeout = 10 * time.Second 43 | } 44 | return &Client{ 45 | Config: config, 46 | listeners: make(map[string]*Listener), 47 | }, nil 48 | } 49 | 50 | func (c *Client) Attach(listeners ...*Listener) { 51 | for _, l := range listeners { 52 | listenerUid := c.getUniqueNameForListener(l) 53 | c.listeners[listenerUid] = l 54 | } 55 | } 56 | 57 | func (c *Client) Init() error { 58 | c.Config.Logger.Debug().Msg("Init driver") 59 | if err := c.Config.Driver.Init(c.Config); err != nil { 60 | return err 61 | } 62 | c.Config.Logger.Debug().Msg("Init listeners") 63 | 64 | // Init listeners (parallel) 65 | var wg sync.WaitGroup 66 | for lUid, l := range c.listeners { 67 | wg.Add(1) 68 | 69 | listenerUid := lUid // Keep intermediate value to avoid conflict between loop iterations 70 | listener := l // Keep intermediate value to avoid conflict between loop iterations 71 | 72 | errChan := make(chan error) 73 | go func() { 74 | defer wg.Done() 75 | err := listener.Init(func(event Operation) error { 76 | return c.Config.Driver.HandleOperationListenStart(listenerUid, listener.Config, event) 77 | }, func(event Operation) error { 78 | return c.Config.Driver.HandleOperationListenStop(listenerUid, listener.Config, event) 79 | }) 80 | errChan <- err 81 | }() 82 | err := <-errChan 83 | if err != nil { 84 | return err 85 | } 86 | } 87 | wg.Wait() 88 | 89 | c.Config.Logger.Debug().Msg("Listener initialized") 90 | return nil 91 | } 92 | 93 | func (c *Client) Start() error { 94 | err := c.Init() 95 | if err != nil { 96 | return err 97 | } 98 | 99 | eventChan := make(DatabaseEventsChan) 100 | errChan := make(chan error) 101 | go func() { 102 | if err := c.Config.Driver.Listen(&eventChan); err != nil { 103 | errChan <- err 104 | } 105 | }() 106 | 107 | for { 108 | select { 109 | case receivedEvent := <-eventChan: 110 | listener, exists := c.listeners[receivedEvent.ListenerUid] 111 | if !exists { 112 | return fmt.Errorf("listener %s not found", receivedEvent.ListenerUid) // I think simply can be ignored 113 | } 114 | listener.Dispatch(&receivedEvent.Event) 115 | case err := <-errChan: 116 | return err 117 | } 118 | } 119 | } 120 | 121 | func (c *Client) Close() error { 122 | errChan := make(chan error, 1) 123 | go func() { 124 | //TODO PARALLEL 125 | c.Config.Logger.Debug().Msg("Closing listeners") 126 | for _, l := range c.listeners { 127 | if err := l.Close(); err != nil { 128 | c.Config.Logger.Error().Err(err).Msg("Error closing listener") 129 | errChan <- err 130 | return 131 | } 132 | } 133 | c.Config.Logger.Debug().Msg("Listeners closed") 134 | 135 | c.Config.Logger.Debug().Msg("Closing driver") 136 | errChan <- c.Config.Driver.Close() 137 | }() 138 | 139 | // Create timeout context for graceful shutdown 140 | ctx, cancel := context.WithTimeout(context.Background(), c.Config.ShutdownTimeout) 141 | defer cancel() 142 | 143 | select { 144 | case err := <-errChan: 145 | if err != nil { 146 | c.Config.Logger.Error().Err(err).Msg("Failed to close driver") 147 | return err 148 | } 149 | c.Config.Logger.Debug().Msg("Driver closed") 150 | 151 | case <-ctx.Done(): 152 | c.Config.Logger.Error().Msg("timeout reached while closing, some events can be loss") 153 | } 154 | 155 | return nil 156 | } 157 | 158 | func (c *Client) getUniqueNameForListener(lc *Listener) string { 159 | return strings.ReplaceAll(fmt.Sprintf("%p", lc), "0x", "") 160 | } 161 | -------------------------------------------------------------------------------- /docs/.vitepress/config.mts: -------------------------------------------------------------------------------- 1 | import {type DefaultTheme, defineConfig} from 'vitepress' 2 | import { withMermaid } from "vitepress-plugin-mermaid"; 3 | 4 | export default withMermaid(defineConfig({ 5 | title: "Flash (Quix Labs)", 6 | lang: 'en-US', 7 | description: "A lightweight Go library for tracking and managing real-time PostgreSQL changes seamlessly and efficiently.", 8 | 9 | lastUpdated: false, 10 | cleanUrls: true, 11 | 12 | srcExclude: [ 13 | 'README.md' 14 | ], 15 | 16 | head: [ 17 | ['link', {rel: 'icon', type: 'image/svg+xml', href: '/logo.svg'}], 18 | ['link', {rel: 'icon', type: 'image/png', sizes: '32x32', href: '/favicon-32x32.png'}], 19 | ['link', {rel: 'icon', type: 'image/png', sizes: '16x16', href: '/favicon-16x16.png'}], 20 | ['link', {rel: 'apple-touch-icon', sizes: '180x180', href: '/apple-touch-icon.png'}], 21 | ['meta', {name: 'theme-color', content: '#5f67ee'}], 22 | ['meta', {property: 'og:type', content: 'website'}], 23 | ['meta', {property: 'og:locale', content: 'en'}], 24 | ['meta', {property: 'og:title', content: 'Flash | Keep track of your database changes'}], 25 | ['meta', {property: 'twitter:title', content: 'Flash | Keep track of your database changes'}], 26 | ['meta', {property: 'og:site_name', content: 'Flash'}], 27 | ['meta', {property: 'twitter:card', content: 'summary_large_image'}], 28 | ['meta', {property: 'twitter:image:src', content: 'https://flash.quix-labs.com/flash-og.png'}], 29 | ['meta', {property: 'og:image', content: 'https://flash.quix-labs.com/flash-og.png'}], 30 | ['meta', {property: 'og:image:type', content: 'image/png'}], 31 | ['meta', {property: 'og:image:width', content: '1280'}], 32 | ['meta', {property: 'og:image:height', content: '640'}], 33 | ['meta', {property: 'og:url', content: 'https://flash.quix-labs.com'}], 34 | ], 35 | 36 | sitemap: { 37 | hostname: 'https://flash.quix-labs.com' 38 | }, 39 | 40 | themeConfig: { 41 | outline: [2, 3], 42 | logo: '/logo.svg', 43 | siteTitle: "Flash", 44 | nav: [ 45 | {text: 'Guide', link: '/guide/what-is-flash', activeMatch: '/guide/'}, 46 | {text: 'Team', link: '/team', activeMatch: '/team/'}, 47 | ], 48 | 49 | socialLinks: [ 50 | {icon: 'github', link: 'https://github.com/quix-labs/flash'} 51 | ], 52 | 53 | sidebar: { 54 | '/guide/': {base: '/guide/', items: sidebarGuide()}, 55 | }, 56 | 57 | editLink: { 58 | pattern: 'https://github.com/quix-labs/flash/edit/main/docs/:path', 59 | text: 'Edit this page on GitHub' 60 | }, 61 | 62 | search: { 63 | provider: 'local', 64 | }, 65 | 66 | footer: { 67 | message: 'Released under the MIT License.', 68 | copyright: `Copyright © ${new Date().getFullYear()} - Quix Labs` 69 | } 70 | } 71 | })) 72 | 73 | 74 | function sidebarGuide(): DefaultTheme.SidebarItem[] { 75 | return [ 76 | { 77 | text: 'Getting Started', 78 | collapsed: false, 79 | items: [ 80 | {text: 'Introduction', link: 'what-is-flash'}, 81 | {text: 'Installation', link: 'installation'}, 82 | ] 83 | }, 84 | 85 | { 86 | text: 'Usage', 87 | collapsed: false, 88 | items: [ 89 | {text: 'Start listening', link: 'start-listening'}, 90 | {text: 'Advanced Features', link: 'advanced-features'}, 91 | {text: 'Drivers Overview', link: 'drivers/'}, 92 | ] 93 | }, 94 | 95 | { 96 | text: 'Drivers', 97 | collapsed: false, 98 | base: '/guide/drivers/', 99 | items: [ 100 | {text: 'Trigger', link: 'trigger/'}, 101 | {text: 'WAL Logical', link: 'wal_logical/',}, 102 | ] 103 | }, 104 | 105 | { 106 | text: "Additional Resources", 107 | collapsed: false, 108 | items: [ 109 | {text: 'Planned Features', link: 'planned-features'}, 110 | {text: 'Upgrade', link: 'upgrade'}, 111 | {text: 'Contributing Guide', link: 'contributing'}, 112 | ] 113 | }, 114 | ] 115 | } -------------------------------------------------------------------------------- /drivers/wal_logical/querying.go: -------------------------------------------------------------------------------- 1 | package wal_logical 2 | 3 | import ( 4 | "context" 5 | "github.com/jackc/pgx/v5/pgconn" 6 | "github.com/quix-labs/flash" 7 | ) 8 | 9 | type subscriptionClaim struct { 10 | listenerUid string 11 | listenerConfig *flash.ListenerConfig 12 | operation *flash.Operation 13 | } 14 | 15 | type activePublication struct { 16 | listenerConfig *flash.ListenerConfig 17 | slotName string 18 | operations *flash.Operation // Use with bitwise to handle combined operations 19 | } 20 | 21 | // Key -> listenerUid 22 | type subscriptionState struct { 23 | subChan chan *subscriptionClaim 24 | unsubChan chan *subscriptionClaim 25 | currentSubscriptions map[string]*activePublication 26 | } 27 | 28 | func (d *Driver) initQuerying() error { 29 | d.subscriptionState = &subscriptionState{ 30 | subChan: make(chan *subscriptionClaim), 31 | unsubChan: make(chan *subscriptionClaim), 32 | currentSubscriptions: make(map[string]*activePublication), 33 | } 34 | 35 | // Bootstrap/Start listening TODO USELESS 36 | d.activePublications = make(map[string]bool) 37 | 38 | return nil 39 | } 40 | 41 | func (d *Driver) startQuerying(readyChan *chan struct{}) error { 42 | // Create connection 43 | config, err := pgconn.ParseConfig(d._clientConfig.DatabaseCnx) 44 | if err != nil { 45 | return err 46 | } 47 | config.RuntimeParams["application_name"] = "Flash: replication (querying)" 48 | if d.queryConn, err = pgconn.ConnectConfig(context.Background(), config); err != nil { 49 | return err 50 | } 51 | 52 | *readyChan <- struct{}{} 53 | for { 54 | select { 55 | 56 | case claimSub := <-d.subscriptionState.unsubChan: 57 | currentSub, exists := d.subscriptionState.currentSubscriptions[claimSub.listenerUid] 58 | if !exists { 59 | continue 60 | } 61 | 62 | // TODO Operation.Remove() 63 | prevEvents := *currentSub.operations 64 | *currentSub.operations &= ^(*claimSub.operation) // Remove operation from listened 65 | 66 | // Bypass if no changes 67 | if *currentSub.operations == prevEvents { 68 | return nil 69 | } 70 | 71 | if len(currentSub.operations.GetAtomics()) > 0 { 72 | alterSql, err := d.getAlterPublicationEventsSql(currentSub) 73 | if err != nil { 74 | return err 75 | } 76 | if _, err := d.sqlExec(d.queryConn, alterSql); err != nil { 77 | return err 78 | } 79 | } else { 80 | if _, err := d.sqlExec(d.queryConn, d.getDropPublicationSlotSql(currentSub.slotName)); err != nil { 81 | return err 82 | } 83 | delete(d.activePublications, currentSub.slotName) 84 | delete(d.subscriptionState.currentSubscriptions, claimSub.listenerUid) 85 | } 86 | 87 | case claimSub := <-d.subscriptionState.subChan: 88 | currentSub, exists := d.subscriptionState.currentSubscriptions[claimSub.listenerUid] 89 | if !exists { 90 | currentSub = &activePublication{ 91 | listenerConfig: claimSub.listenerConfig, 92 | slotName: d.getFullSlotName(claimSub.listenerUid), 93 | operations: claimSub.operation, 94 | } 95 | 96 | slotName := d.getFullSlotName(claimSub.listenerUid) 97 | rawSql, err := d.getCreatePublicationSlotSql(slotName, claimSub.listenerConfig, claimSub.operation) 98 | if err != nil { 99 | return err 100 | } 101 | if _, err := d.sqlExec(d.queryConn, rawSql); err != nil { 102 | return err 103 | } 104 | 105 | d.subscriptionState.currentSubscriptions[claimSub.listenerUid] = currentSub 106 | d.activePublications[slotName] = true 107 | d.replicationState.restartChan <- struct{}{} // Send restart signal 108 | 109 | } else { 110 | prevEvents := *currentSub.operations 111 | 112 | // TODO Operation.Append() or Operation.Merge() 113 | *currentSub.operations |= *claimSub.operation //Append operation to listened 114 | 115 | // Bypass if no changes 116 | if prevEvents == *currentSub.operations { 117 | return nil 118 | } 119 | 120 | alterSql, err := d.getAlterPublicationEventsSql(currentSub) 121 | if err != nil { 122 | return err 123 | } 124 | if _, err := d.sqlExec(d.queryConn, alterSql); err != nil { 125 | return err 126 | } 127 | } 128 | } 129 | } 130 | } 131 | 132 | func (d *Driver) closeQuerying() error { 133 | if d.queryConn != nil { 134 | for publication, _ := range d.activePublications { 135 | if _, err := d.sqlExec(d.queryConn, d.getDropPublicationSlotSql(publication)); err != nil { 136 | return err 137 | } 138 | } 139 | err := d.queryConn.Close(context.Background()) 140 | if err != nil { 141 | return err 142 | } 143 | d.queryConn = nil 144 | } 145 | return nil 146 | } 147 | -------------------------------------------------------------------------------- /docs/guide/drivers/wal_logical/WORKFLOW.md: -------------------------------------------------------------------------------- 1 | ```mermaid 2 | --- 3 | title: Interaction workflow for WAL Logical driver 4 | --- 5 | sequenceDiagram 6 | participant Your App 7 | participant Listener 8 | participant Client 9 | participant Driver 10 | participant Database 11 | participant External 12 | rect rgba(34,211,238,0.5) 13 | note over Your App, External: Bootstrapping 14 | Your App ->> Listener: on(eventUpdate^EventInsert) 15 | Your App ->> Client: AddListener(listener) 16 | end 17 | rect rgba(250,204,21,0.5) 18 | note over Your App, External: Starting 19 | Your App ->> Client: start() 20 | Client ->> Driver: driver.Init() 21 | Client ->> Driver: driver.Start() 22 | Driver ->> Database: CREATE PUBLICATION "...-init" 23 | Driver ->> Database: CREATE REPLICATION_SLOT "...-slot" TEMPORARY 24 | loop For each active listener 25 | Client ->> Listener: Listener.Init() 26 | loop For each listened operation 27 | Listener ->> Client: start listening for operation 28 | Client ->> Driver: send start listening signal for operation 29 | Driver ->> Database: CREATE PUBLICATION SLOT ... 30 | Driver -->> Driver: Restart connection to handle new slot 31 | Driver -->> Driver: Wait for connection restart 32 | end 33 | end 34 | end 35 | rect rgba(45,212,191,0.5) 36 | par 37 | note over Your App, External: Change listeners during runtime 38 | loop 39 | Your App ->> Listener: on(eventDelete) 40 | Listener ->> Client: start listening for delete 41 | Client ->> Driver: send listen for delete signal 42 | Driver ->> Database: ALTER PUBLICATION ... 43 | end 44 | 45 | and 46 | note over Your App, External: Handle KeepAlive 47 | loop Handle KeepAlive 48 | par 49 | Database --) Driver: claim keepalive 50 | and x seconds since last send 51 | Driver -->> Driver: Wait x seconds 52 | end 53 | 54 | Driver ->> Database: send keepalive 55 | end 56 | and 57 | note over Your App, External: Handle XLogData (not prevented) 58 | loop 59 | External -->> Database: DELETE FROM ... 60 | Database --) Driver: Write WAL 61 | activate Driver 62 | loop For each concerned listener 63 | Driver -->> Client: Parse data and send event 64 | Client -->> Listener: Notify listener 65 | Listener -->> Your App: Event processed 66 | end 67 | Driver ->> Database: FLUSH POSITION 68 | deactivate Driver 69 | end 70 | and 71 | note over Your App, External: Handle StreamStart 72 | External -->> Database: BEGIN TRANSACTION 73 | Database --) Driver: send stream start 74 | Driver ->> Driver: Preventing XLogData processing 75 | and 76 | note over Your App, External: Handle StreamStop 77 | Driver ->> Driver: Stop preventing XLogData processing 78 | and 79 | note over Your App, External: Handle XLogData (prevented) 80 | External -->> Database: DELETE FROM ... 81 | External -->> Database: UPDATE SET ... 82 | External -->> Database: INSERT INTO ... 83 | loop 84 | Database --) Driver: send XLogData 85 | Driver ->> Driver: Parse data and stack in queue 86 | end 87 | and 88 | note over Your App, External: Handle StreamAbort 89 | External -->> Database: ROLLBACK 90 | Database --) Driver: send stream rollback 91 | Driver ->> Driver: remove queue 92 | Driver ->> Database: FLUSH POSITION 93 | and 94 | note over Your App, External: Handle StreamCommit 95 | External -->> Database: COMMIT 96 | Database --) Driver: send stream commit 97 | activate Driver 98 | loop For each queued event 99 | loop For each concerned listener 100 | Driver -->> Client: Send event 101 | Client -->> Listener: Notify listener 102 | Listener -->> Your App: Event processed 103 | end 104 | end 105 | Driver ->> Database: FLUSH POSITION 106 | deactivate Driver 107 | end 108 | end 109 | rect rgba(248,113,113,0.5) 110 | Note over Your App, External: Application Shutdown 111 | Your App ->> Client: stop() 112 | loop For each active listener 113 | Client ->> Listener: Listener.Close() 114 | loop For each listened operation 115 | Listener ->> Client: stop listening for operation 116 | Client ->> Driver: send stop listening signal for operation 117 | Driver ->> Database: DROP PUBLICATION ... 118 | end 119 | end 120 | Client ->> Driver: Driver.Close() 121 | Driver ->> Database: close connection ... 122 | Database ->> Database: DROP TEMPORARY REPLICATION SLOT 123 | end 124 | ``` -------------------------------------------------------------------------------- /listener.go: -------------------------------------------------------------------------------- 1 | package flash 2 | 3 | import ( 4 | "errors" 5 | "sync" 6 | ) 7 | 8 | // TODO SORTIR VERIFICATION AU NIVEAU LISTENER, PBM oblige à envoyer les columns dans l'event 9 | type ListenerCondition struct { 10 | Column string 11 | //Operator string //TODO actually only equals are implemented 12 | Value any 13 | } 14 | 15 | type ListenerConfig struct { 16 | Table string // Can be prefixed by schema - e.g: public.posts 17 | Fields []string // Empty fields means all ( SELECT * ) 18 | MaxParallelProcess int // Default to 1 (not parallel) -> use -1 for Infinity 19 | 20 | Conditions []*ListenerCondition 21 | } 22 | 23 | type CreateEventCallback func(event Operation) error 24 | type DeleteEventCallback func(event Operation) error 25 | type EventCallback func(event Event) 26 | 27 | type Listener struct { 28 | Config *ListenerConfig 29 | 30 | // Internals 31 | sync.Mutex 32 | callbacks map[*EventCallback]Operation 33 | listenedOperations Operation // Use bitwise comparison to check for listened events 34 | semaphore chan struct{} 35 | 36 | // Trigger client 37 | _clientCreateEventCallback CreateEventCallback 38 | _clientDeleteEventCallback DeleteEventCallback 39 | _clientInitialized bool 40 | } 41 | 42 | func NewListener(config *ListenerConfig) (*Listener, error) { 43 | if config == nil { 44 | return nil, errors.New("config cannot be nil") 45 | } 46 | if config.MaxParallelProcess == 0 { 47 | config.MaxParallelProcess = 1 48 | } 49 | 50 | var semaphore chan struct{} = nil 51 | if config.MaxParallelProcess != -1 { 52 | semaphore = make(chan struct{}, config.MaxParallelProcess) 53 | } 54 | 55 | return &Listener{ 56 | Config: config, 57 | callbacks: make(map[*EventCallback]Operation), 58 | semaphore: semaphore, 59 | }, nil 60 | } 61 | 62 | /* Callback management */ 63 | 64 | func (l *Listener) On(operation Operation, callback EventCallback) (func() error, error) { 65 | if callback == nil { 66 | return nil, errors.New("callback cannot be nil") 67 | } 68 | 69 | // TODO NOTIFY CLIENT FROM UPDATE BUT DO NOT SEND INSERT/DELETE 70 | if err := l.addListenedEventIfNeeded(operation); err != nil { 71 | return nil, err 72 | } 73 | 74 | l.callbacks[&callback] = operation 75 | 76 | removeFunc := func() error { 77 | delete(l.callbacks, &callback) // Important keep before removeListenedOperationIfNeeded 78 | if err := l.removeListenedOperationIfNeeded(operation); err != nil { 79 | return err 80 | } 81 | callback = nil 82 | return nil 83 | } 84 | 85 | return removeFunc, nil 86 | } 87 | 88 | func (l *Listener) Dispatch(event *Event) { 89 | for callback, listenedOperations := range l.callbacks { 90 | if listenedOperations.IncludeOne((*event).GetOperation()) { 91 | if l.Config.MaxParallelProcess == -1 { 92 | go (*callback)(*event) 93 | continue 94 | } 95 | 96 | // Acquire semaphore 97 | l.semaphore <- struct{}{} 98 | if l.Config.MaxParallelProcess == 1 { 99 | (*callback)(*event) 100 | <-l.semaphore 101 | continue 102 | } 103 | 104 | go func() { 105 | (*callback)(*event) 106 | <-l.semaphore 107 | }() 108 | } 109 | } 110 | 111 | } 112 | 113 | // Init emit all event for first boot */ 114 | func (l *Listener) Init(_createCallback CreateEventCallback, _deleteCallback DeleteEventCallback) error { 115 | l.Lock() 116 | defer l.Unlock() 117 | 118 | l._clientCreateEventCallback = _createCallback 119 | l._clientDeleteEventCallback = _deleteCallback 120 | 121 | // Emit all events for initialization 122 | for targetEvent := Operation(1); targetEvent != 0 && targetEvent <= OperationAll; targetEvent <<= 1 { 123 | if l.listenedOperations&targetEvent == 0 { 124 | continue 125 | } 126 | if err := _createCallback(targetEvent); err != nil { 127 | return err 128 | } 129 | } 130 | 131 | l._clientInitialized = true 132 | return nil 133 | } 134 | 135 | func (l *Listener) addListenedEventIfNeeded(event Operation) error { 136 | 137 | initialEvents := l.listenedOperations 138 | l.listenedOperations |= event 139 | 140 | // Trigger event if change appears 141 | diff := initialEvents ^ l.listenedOperations 142 | if diff == 0 { 143 | return nil 144 | } 145 | 146 | for targetEvent := Operation(1); targetEvent != 0 && targetEvent <= OperationAll; targetEvent <<= 1 { 147 | if targetEvent&diff == 0 || targetEvent&event == 0 { 148 | continue 149 | } 150 | l.Lock() 151 | if l._clientInitialized { 152 | if err := l._clientCreateEventCallback(targetEvent); err != nil { 153 | return err 154 | } 155 | } 156 | l.Unlock() 157 | } 158 | 159 | return nil 160 | } 161 | 162 | func (l *Listener) removeListenedOperationIfNeeded(event Operation) error { 163 | 164 | for targetEvent := Operation(1); targetEvent != 0 && targetEvent <= event; targetEvent <<= 1 { 165 | if targetEvent&l.listenedOperations == 0 { 166 | continue 167 | } 168 | if l.hasListenersForEvent(targetEvent) { 169 | continue 170 | } 171 | 172 | l.listenedOperations &= ^targetEvent 173 | if l._clientInitialized { 174 | l.Lock() 175 | if err := l._clientDeleteEventCallback(targetEvent); err != nil { 176 | return err 177 | } 178 | l.Unlock() 179 | } 180 | } 181 | return nil 182 | } 183 | 184 | func (l *Listener) Close() error { 185 | l.Lock() 186 | defer l.Unlock() 187 | l._clientInitialized = false 188 | return nil 189 | } 190 | func (l *Listener) hasListenersForEvent(event Operation) bool { 191 | for _, listens := range l.callbacks { 192 | if listens&event > 0 { 193 | return true 194 | } 195 | } 196 | return false 197 | } 198 | -------------------------------------------------------------------------------- /operations_test.go: -------------------------------------------------------------------------------- 1 | package flash 2 | 3 | import "testing" 4 | 5 | func TestIsAtomic(t *testing.T) { 6 | tests := []struct { 7 | name string 8 | o Operation 9 | expected bool 10 | }{ 11 | {"Atomic Operation", OperationTruncate, true}, 12 | {"Composite Operation", OperationInsert | OperationUpdate, false}, 13 | {"Atomic But Invalid", 32, false}, 14 | {"Empty Operation", 0, false}, 15 | } 16 | 17 | for _, test := range tests { 18 | t.Run(test.name, func(t *testing.T) { 19 | if test.o.IsAtomic() != test.expected { 20 | t.Errorf("IsAtomic() failed for %v: expected %v, got %v", test.o, test.expected, test.o.IsAtomic()) 21 | } 22 | }) 23 | } 24 | } 25 | 26 | func TestGetAtomics(t *testing.T) { 27 | tests := []struct { 28 | name string 29 | o Operation 30 | expected []Operation 31 | }{ 32 | {"Atomic Operation", OperationTruncate, []Operation{OperationTruncate}}, 33 | {"Composite Operation", OperationInsert | OperationUpdate, []Operation{OperationInsert, OperationUpdate}}, 34 | {"Composite All Operation", OperationAll, []Operation{OperationInsert, OperationUpdate, OperationDelete, OperationTruncate}}, 35 | {"Empty Operation", 0, []Operation{}}, 36 | {"Unknown Atomic", 32, []Operation{}}, 37 | } 38 | 39 | for _, test := range tests { 40 | t.Run(test.name, func(t *testing.T) { 41 | atomics := test.o.GetAtomics() 42 | if len(atomics) != len(test.expected) { 43 | t.Errorf("GetAtomics() failed for %v: expected length %v, got length %v", test.o, len(test.expected), len(atomics)) 44 | } else { 45 | for i, op := range atomics { 46 | if op != test.expected[i] { 47 | t.Errorf("GetAtomics() failed for %v: expected %v at index %d, got %v", test.o, test.expected[i], i, op) 48 | } 49 | } 50 | } 51 | }) 52 | } 53 | } 54 | 55 | func TestIncludeAll(t *testing.T) { 56 | tests := []struct { 57 | name string 58 | o Operation 59 | mask Operation 60 | expected bool 61 | }{ 62 | {"IncludeAll - true", OperationInsert | OperationUpdate | OperationDelete, OperationInsert | OperationUpdate, true}, 63 | {"IncludeAll - false", OperationInsert | OperationUpdate, OperationInsert | OperationUpdate | OperationDelete, false}, 64 | {"IncludeAll - empty operation", 0, OperationAll, false}, 65 | {"IncludeAll - unknown", 32, OperationAll, false}, 66 | {"IncludeAll - unknown", OperationAll, 32, false}, 67 | } 68 | 69 | for _, test := range tests { 70 | t.Run(test.name, func(t *testing.T) { 71 | if test.o.IncludeAll(test.mask) != test.expected { 72 | t.Errorf("IncludeAll() failed for %v with mask %v: expected %v, got %v", test.o, test.mask, test.expected, test.o.IncludeAll(test.mask)) 73 | } 74 | }) 75 | } 76 | } 77 | 78 | func TestIncludeOne(t *testing.T) { 79 | tests := []struct { 80 | name string 81 | o Operation 82 | mask Operation 83 | expected bool 84 | }{ 85 | {"IncludeOne - true", OperationInsert | OperationUpdate | OperationDelete, OperationDelete, true}, 86 | {"IncludeOne - false", OperationInsert | OperationUpdate, OperationTruncate, false}, 87 | {"IncludeOne - empty operation", 0, OperationInsert, false}, 88 | {"IncludeOne - Atomic same", OperationUpdate, OperationUpdate, true}, 89 | {"IncludeOne - Atomic different", OperationUpdate, OperationDelete, false}, 90 | } 91 | 92 | for _, test := range tests { 93 | t.Run(test.name, func(t *testing.T) { 94 | if test.o.IncludeOne(test.mask) != test.expected { 95 | t.Errorf("IncludeOne() failed for %v with mask %v: expected %v, got %v", test.o, test.mask, test.expected, test.o.IncludeOne(test.mask)) 96 | } 97 | }) 98 | } 99 | } 100 | 101 | func TestStrictName(t *testing.T) { 102 | tests := []struct { 103 | name string 104 | o Operation 105 | expectedName string 106 | expectedErr bool 107 | }{ 108 | {"Insert Operation", OperationInsert, "INSERT", false}, 109 | {"Update Operation", OperationUpdate, "UPDATE", false}, 110 | {"Delete Operation", OperationDelete, "DELETE", false}, 111 | {"Truncate Operation", OperationTruncate, "TRUNCATE", false}, 112 | {"Unknown Operation", Operation(32), "UNKNOWN", true}, 113 | {"Composite Operation", OperationInsert | OperationUpdate, "UNKNOWN", true}, 114 | } 115 | 116 | for _, test := range tests { 117 | t.Run(test.name, func(t *testing.T) { 118 | name, err := test.o.StrictName() 119 | 120 | // Check name correctness 121 | if name != test.expectedName { 122 | t.Errorf("StrictName() failed for %v: expected name %v, got %v", test.o, test.expectedName, name) 123 | } 124 | 125 | // Check error correctness 126 | if (err != nil) != test.expectedErr { 127 | t.Errorf("StrictName() failed for %v: expected error %v, got error %v", test.o, test.expectedErr, err) 128 | } 129 | }) 130 | } 131 | } 132 | 133 | func TestString(t *testing.T) { 134 | tests := []struct { 135 | name string 136 | o Operation 137 | expected string 138 | }{ 139 | {"Single Atomic Operation", OperationInsert, "INSERT"}, 140 | {"Multiple Atomic Operations", OperationInsert | OperationUpdate | OperationTruncate, "INSERT | UPDATE | TRUNCATE"}, 141 | {"Empty Operation", 0, "UNKNOWN"}, 142 | {"Unknown Operation", 32, "UNKNOWN"}, 143 | } 144 | 145 | for _, test := range tests { 146 | t.Run(test.name, func(t *testing.T) { 147 | result := test.o.String() 148 | 149 | if result != test.expected { 150 | t.Errorf("String() failed for %v: expected '%v', got '%v'", test.o, test.expected, result) 151 | } 152 | }) 153 | } 154 | } 155 | 156 | func TestOperationFromName(t *testing.T) { 157 | tests := []struct { 158 | name string 159 | input string 160 | expected Operation 161 | expectedErr bool 162 | }{ 163 | {"Insert", "insert", OperationInsert, false}, 164 | {"Update", "update", OperationUpdate, false}, 165 | {"Delete", "delete", OperationDelete, false}, 166 | {"Truncate", "truncate", OperationTruncate, false}, 167 | {"Truncate", "INSERT", OperationInsert, false}, 168 | {"Truncate", "UPDATE", OperationUpdate, false}, 169 | {"Truncate", "DELETE", OperationDelete, false}, 170 | {"Truncate", "TRUNCATE", OperationTruncate, false}, 171 | {"Unknown", "unknown", 0, true}, 172 | {"Empty String", "", 0, true}, 173 | } 174 | 175 | for _, test := range tests { 176 | t.Run(test.name, func(t *testing.T) { 177 | result, err := OperationFromName(test.input) 178 | 179 | if (err != nil) != test.expectedErr { 180 | t.Errorf("OperationFromName() error = %v, expected error = %v", err, test.expectedErr) 181 | return 182 | } 183 | 184 | if result != test.expected { 185 | t.Errorf("OperationFromName() = %v, expected %v", result, test.expected) 186 | } 187 | }) 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /drivers/trigger/driver.go: -------------------------------------------------------------------------------- 1 | package trigger 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/lib/pq" 8 | "github.com/quix-labs/flash" 9 | "net/url" 10 | "time" 11 | ) 12 | 13 | type DriverConfig struct { 14 | Schema string // The schema name, which should be unique across all instances 15 | } 16 | 17 | var ( 18 | _ flash.Driver = (*Driver)(nil) // Interface implementation 19 | ) 20 | 21 | func NewDriver(config *DriverConfig) *Driver { 22 | if config == nil { 23 | config = &DriverConfig{} 24 | } 25 | if config.Schema == "" { 26 | config.Schema = "flash" 27 | } 28 | return &Driver{ 29 | Config: config, 30 | activeEvents: make(map[string]bool), 31 | } 32 | } 33 | 34 | type Driver struct { 35 | Config *DriverConfig 36 | 37 | conn *sql.DB 38 | pgListener *pq.Listener 39 | 40 | subChan chan string 41 | unsubChan chan string 42 | shutdown chan bool 43 | 44 | activeEvents map[string]bool 45 | _clientConfig *flash.ClientConfig 46 | } 47 | 48 | func (d *Driver) HandleOperationListenStart(listenerUid string, lc *flash.ListenerConfig, operation flash.Operation) error { 49 | createTriggerSql, eventName, err := d.getCreateTriggerSqlForOperation(listenerUid, lc, &operation) 50 | if err != nil { 51 | return err 52 | } 53 | _, err = d.sqlExec(d.conn, createTriggerSql) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | return d.addEventToListened(eventName) 59 | } 60 | 61 | func (d *Driver) HandleOperationListenStop(listenerUid string, lc *flash.ListenerConfig, event flash.Operation) error { 62 | createTriggerSql, eventName, err := d.getDeleteTriggerSqlForEvent(listenerUid, lc, &event) 63 | if err != nil { 64 | return err 65 | } 66 | _, err = d.sqlExec(d.conn, createTriggerSql) 67 | if err != nil { 68 | return err 69 | } 70 | 71 | return d.removeEventToListened(eventName) 72 | } 73 | 74 | func (d *Driver) Init(_clientConfig *flash.ClientConfig) error { 75 | d._clientConfig = _clientConfig 76 | 77 | parsedCnx, err := url.Parse(d._clientConfig.DatabaseCnx) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | query := parsedCnx.Query() 83 | query.Set("application_name", "test") 84 | parsedCnx.RawQuery = query.Encode() 85 | 86 | connector, err := pq.NewConnector(parsedCnx.String()) 87 | if err != nil { 88 | return err 89 | } 90 | 91 | d.conn = sql.OpenDB(connector) 92 | // Create schema if not exists 93 | if _, err := d.sqlExec(d.conn, "CREATE SCHEMA IF NOT EXISTS \""+d.Config.Schema+"\";"); err != nil { 94 | return err 95 | } 96 | return nil 97 | } 98 | 99 | func (d *Driver) Listen(eventsChan *flash.DatabaseEventsChan) error { 100 | errChan := make(chan error) 101 | d.subChan = make(chan string, len(d.activeEvents)) 102 | d.unsubChan = make(chan string, 1) 103 | d.shutdown = make(chan bool) 104 | 105 | reportProblem := func(ev pq.ListenerEventType, err error) { 106 | if err != nil { 107 | errChan <- err 108 | } 109 | } 110 | 111 | parsedCnx, err := url.Parse(d._clientConfig.DatabaseCnx) 112 | if err != nil { 113 | return err 114 | } 115 | 116 | query := parsedCnx.Query() 117 | query.Set("application_name", "test_listen") 118 | parsedCnx.RawQuery = query.Encode() 119 | 120 | d.pgListener = pq.NewListener(parsedCnx.String(), 1*time.Second, time.Minute, reportProblem) 121 | 122 | // Initialize subChan with activeEvents in queue 123 | go func() { 124 | for eventName := range d.activeEvents { 125 | d.subChan <- eventName 126 | } 127 | }() 128 | 129 | for { 130 | select { 131 | 132 | case <-d.shutdown: 133 | return d.pgListener.Close() 134 | 135 | case err := <-errChan: 136 | return err 137 | 138 | case eventName := <-d.unsubChan: 139 | d._clientConfig.Logger.Trace().Str("query", fmt.Sprintf(`UNLISTEN "%s"`, eventName)).Msg("sending sql request") 140 | if err := d.pgListener.Unlisten(eventName); err != nil { 141 | return err 142 | } 143 | continue 144 | 145 | case eventName := <-d.subChan: 146 | d._clientConfig.Logger.Trace().Str("query", fmt.Sprintf(`LISTEN "%s"`, eventName)).Msg("sending sql request") 147 | if err := d.pgListener.Listen(eventName); err != nil { 148 | return err 149 | } 150 | continue 151 | 152 | case notification := <-d.pgListener.Notify: 153 | listenerUid, operation, err := d.parseEventName(notification.Channel) 154 | if err != nil { 155 | errChan <- err 156 | continue 157 | } 158 | 159 | var data map[string]any 160 | if notification.Extra != "" { 161 | data = make(map[string]any) 162 | if err := json.Unmarshal([]byte(notification.Extra), &data); err != nil { 163 | errChan <- err 164 | continue 165 | } 166 | } 167 | var newData, oldData *flash.EventData = nil, nil 168 | if data != nil { 169 | if nd, exists := data["new"]; exists && nd != nil { 170 | typedData := flash.EventData(nd.(map[string]any)) 171 | newData = &typedData 172 | } 173 | if od, exists := data["old"]; exists && od != nil { 174 | typedData := flash.EventData(od.(map[string]any)) 175 | oldData = &typedData 176 | } 177 | } 178 | 179 | // Custom conditions if update to handle soft deletes 180 | if operation == flash.OperationUpdate { 181 | var previouslyMatch, newlyMatch bool = true, true 182 | /* Extract condition match */ 183 | if nc, exists := data["new_condition"]; exists && nc != nil { 184 | newlyMatch = nc.(bool) 185 | } 186 | if oc, exists := data["old_condition"]; exists && oc != nil { 187 | previouslyMatch = oc.(bool) 188 | } 189 | 190 | // Send insert signal 191 | if !previouslyMatch && newlyMatch { 192 | *eventsChan <- &flash.DatabaseEvent{ 193 | ListenerUid: listenerUid, 194 | Event: &flash.InsertEvent{New: newData}, 195 | } 196 | } else if previouslyMatch && !newlyMatch { 197 | *eventsChan <- &flash.DatabaseEvent{ 198 | ListenerUid: listenerUid, 199 | Event: &flash.DeleteEvent{Old: oldData}, 200 | } 201 | } else if previouslyMatch && newlyMatch { 202 | *eventsChan <- &flash.DatabaseEvent{ 203 | ListenerUid: listenerUid, 204 | Event: &flash.UpdateEvent{New: newData, Old: oldData}, 205 | } 206 | } 207 | continue 208 | } 209 | 210 | switch operation { 211 | case flash.OperationInsert: 212 | *eventsChan <- &flash.DatabaseEvent{ 213 | ListenerUid: listenerUid, 214 | Event: &flash.InsertEvent{New: newData}, 215 | } 216 | case flash.OperationUpdate: 217 | *eventsChan <- &flash.DatabaseEvent{ 218 | ListenerUid: listenerUid, 219 | Event: &flash.UpdateEvent{New: newData, Old: oldData}, 220 | } 221 | case flash.OperationDelete: 222 | *eventsChan <- &flash.DatabaseEvent{ 223 | ListenerUid: listenerUid, 224 | Event: &flash.DeleteEvent{Old: oldData}, 225 | } 226 | case flash.OperationTruncate: 227 | *eventsChan <- &flash.DatabaseEvent{ 228 | ListenerUid: listenerUid, 229 | Event: &flash.TruncateEvent{}, 230 | } 231 | default: 232 | return fmt.Errorf("unknown operation: %d", operation) 233 | } 234 | } 235 | } 236 | } 237 | 238 | func (d *Driver) addEventToListened(eventName string) error { 239 | d.activeEvents[eventName] = true 240 | 241 | if d.pgListener == nil { 242 | return nil 243 | } 244 | 245 | d.subChan <- eventName 246 | 247 | return nil 248 | } 249 | 250 | func (d *Driver) removeEventToListened(eventName string) error { 251 | delete(d.activeEvents, eventName) 252 | 253 | if d.pgListener == nil { 254 | return nil 255 | } 256 | d.unsubChan <- eventName 257 | 258 | return nil 259 | } 260 | 261 | func (d *Driver) Close() error { 262 | if d.pgListener != nil { 263 | d.shutdown <- true 264 | } 265 | 266 | // Drop created schema 267 | if _, err := d.sqlExec(d.conn, "DROP SCHEMA IF EXISTS \""+d.Config.Schema+"\" CASCADE;"); err != nil { 268 | return err 269 | } 270 | 271 | // Close active connection 272 | if d.conn != nil { 273 | if err := d.conn.Close(); err != nil { 274 | return err 275 | } 276 | } 277 | return nil 278 | } 279 | -------------------------------------------------------------------------------- /drivers/trigger/queries.go: -------------------------------------------------------------------------------- 1 | package trigger 2 | 3 | import ( 4 | "database/sql" 5 | "errors" 6 | "fmt" 7 | "github.com/quix-labs/flash" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | func (d *Driver) getCreateTriggerSqlForOperation(listenerUid string, l *flash.ListenerConfig, e *flash.Operation) (string, string, error) { 13 | uniqueName, err := d.getUniqueIdentifierForListenerEvent(listenerUid, e) 14 | if err != nil { 15 | return "", "", err 16 | } 17 | 18 | operation, err := e.StrictName() 19 | if err != nil { 20 | return "", "", err 21 | } 22 | 23 | triggerName := uniqueName + "_trigger" 24 | triggerFnName := uniqueName + "_fn" 25 | eventName := uniqueName + "_event" 26 | 27 | var statement string 28 | if len(l.Fields) == 0 { 29 | statement = fmt.Sprintf(` 30 | CREATE OR REPLACE FUNCTION "%s"."%s"() RETURNS trigger AS $trigger$ 31 | BEGIN 32 | PERFORM pg_notify('%s', JSONB_BUILD_OBJECT('old',to_jsonb(OLD),'new',to_jsonb(NEW))::TEXT); 33 | RETURN COALESCE(NEW, OLD); 34 | END; 35 | $trigger$ LANGUAGE plpgsql VOLATILE;`, 36 | d.Config.Schema, triggerFnName, eventName) 37 | } else { 38 | var rawFields, rawConditionSql string 39 | 40 | switch operation { 41 | case "TRUNCATE": 42 | rawFields = "null" 43 | case "DELETE": 44 | 45 | if len(l.Conditions) > 0 { 46 | rawConditionSql, err = d.getConditionsSql(l.Conditions, "OLD") 47 | if err != nil { 48 | return "", "", err 49 | } 50 | } 51 | 52 | jsonFields := make([]string, len(l.Fields)) 53 | for i, field := range l.Fields { 54 | jsonFields[i] = fmt.Sprintf(`'%s', OLD."%s"`, field, field) 55 | } 56 | rawFields = fmt.Sprintf(`JSONB_BUILD_OBJECT('old',JSONB_BUILD_OBJECT(%s))::TEXT`, strings.Join(jsonFields, ",")) 57 | case "INSERT": 58 | 59 | if len(l.Conditions) > 0 { 60 | rawConditionSql, err = d.getConditionsSql(l.Conditions, "NEW") 61 | if err != nil { 62 | return "", "", err 63 | } 64 | } 65 | 66 | jsonFields := make([]string, len(l.Fields)) 67 | for i, field := range l.Fields { 68 | jsonFields[i] = fmt.Sprintf(`'%s', NEW."%s"`, field, field) 69 | } 70 | rawFields = fmt.Sprintf(`JSONB_BUILD_OBJECT('new',JSONB_BUILD_OBJECT(%s))::TEXT`, strings.Join(jsonFields, ",")) 71 | case "UPDATE": 72 | oldJsonFields := make([]string, len(l.Fields)) 73 | newJsonFields := make([]string, len(l.Fields)) 74 | for i, field := range l.Fields { 75 | oldJsonFields[i] = fmt.Sprintf(`'%s', OLD."%s"`, field, field) 76 | newJsonFields[i] = fmt.Sprintf(`'%s', NEW."%s"`, field, field) 77 | } 78 | 79 | // Build raw conditions for field updates 80 | rawConditions := make([]string, len(l.Fields)) 81 | for i, field := range l.Fields { 82 | rawConditions[i] = fmt.Sprintf(`(OLD."%s" IS DISTINCT FROM NEW."%s")`, field, field) 83 | } 84 | rawConditionSql = strings.Join(rawConditions, " OR ") 85 | 86 | // Build conditions for soft delete check 87 | var oldConditionsSql, newConditionsSql string = "null", "null" 88 | if len(l.Conditions) > 0 { 89 | oldConditionsSql, err = d.getConditionsSql(l.Conditions, "OLD") 90 | if err != nil { 91 | return "", "", err 92 | } 93 | newConditionsSql, err = d.getConditionsSql(l.Conditions, "NEW") 94 | if err != nil { 95 | return "", "", err 96 | } 97 | 98 | // Combine update conditions with soft delete conditions 99 | rawConditionSql = fmt.Sprintf(`((%s)!=(%s)) OR (%s)`, oldConditionsSql, newConditionsSql, rawConditionSql) 100 | } 101 | 102 | rawFields = fmt.Sprintf( 103 | `JSONB_BUILD_OBJECT('old',JSONB_BUILD_OBJECT(%s),'new',JSONB_BUILD_OBJECT(%s),'old_condition',%s,'new_condition',%s)::TEXT`, 104 | strings.Join(oldJsonFields, ","), 105 | strings.Join(newJsonFields, ","), 106 | oldConditionsSql, 107 | newConditionsSql, 108 | ) 109 | } 110 | 111 | if rawConditionSql == "" { 112 | 113 | statement = fmt.Sprintf(` 114 | CREATE OR REPLACE FUNCTION "%s"."%s"() RETURNS trigger AS $trigger$ 115 | BEGIN 116 | PERFORM pg_notify('%s', %s); 117 | RETURN COALESCE(NEW, OLD); 118 | END; 119 | $trigger$ LANGUAGE plpgsql VOLATILE;`, 120 | d.Config.Schema, triggerFnName, eventName, rawFields) 121 | } else { 122 | statement = fmt.Sprintf(` 123 | CREATE OR REPLACE FUNCTION "%s"."%s"() RETURNS trigger AS $trigger$ 124 | BEGIN 125 | IF %s THEN 126 | PERFORM pg_notify('%s', %s); 127 | END IF; 128 | RETURN COALESCE(NEW, OLD); 129 | END; 130 | $trigger$ LANGUAGE plpgsql VOLATILE;`, 131 | d.Config.Schema, triggerFnName, rawConditionSql, eventName, rawFields) 132 | } 133 | } 134 | 135 | if operation != "TRUNCATE" { 136 | // Keep drop + create instead of 'create or replace' for Pgsql13 compatibility 137 | statement += fmt.Sprintf(` 138 | DROP TRIGGER IF EXISTS "%s" ON %s; 139 | CREATE TRIGGER "%s" AFTER %s ON %s FOR EACH ROW EXECUTE PROCEDURE "%s"."%s"();`, 140 | triggerName, d.sanitizeTableName(l.Table), triggerName, operation, d.sanitizeTableName(l.Table), d.Config.Schema, triggerFnName) 141 | } else { 142 | // Keep drop + create instead of 'create or replace' for Pgsql13 compatibility 143 | statement += fmt.Sprintf(` 144 | DROP TRIGGER IF EXISTS "%s" ON %s; 145 | CREATE TRIGGER "%s" BEFORE TRUNCATE ON %s FOR EACH STATEMENT EXECUTE PROCEDURE "%s"."%s"();`, 146 | triggerName, d.sanitizeTableName(l.Table), triggerName, d.sanitizeTableName(l.Table), d.Config.Schema, triggerFnName) 147 | } 148 | 149 | return statement, eventName, nil 150 | } 151 | 152 | func (d *Driver) getDeleteTriggerSqlForEvent(listenerUid string, l *flash.ListenerConfig, e *flash.Operation) (string, string, error) { 153 | uniqueName, err := d.getUniqueIdentifierForListenerEvent(listenerUid, e) 154 | if err != nil { 155 | return "", "", err 156 | } 157 | 158 | triggerFnName := uniqueName + "_fn" 159 | eventName := uniqueName + "_event" 160 | 161 | return fmt.Sprintf(`DROP FUNCTION IF EXISTS "%s"."%s" CASCADE;`, d.Config.Schema, triggerFnName), eventName, nil 162 | } 163 | 164 | func (d *Driver) getUniqueIdentifierForListenerEvent(listenerUid string, e *flash.Operation) (string, error) { 165 | operationName, err := e.StrictName() 166 | if err != nil { 167 | return "", err 168 | } 169 | return strings.Join([]string{ 170 | d.Config.Schema, 171 | listenerUid, 172 | strings.ToLower(operationName), 173 | }, "_"), nil 174 | } 175 | func (d *Driver) parseEventName(channel string) (string, flash.Operation, error) { 176 | parts := strings.Split(channel, "_") 177 | if len(parts) != 4 { 178 | return "", 0, errors.New("could not determine unique identifier") 179 | } 180 | 181 | listenerUid := parts[1] 182 | operation, err := flash.OperationFromName(parts[2]) 183 | if err != nil { 184 | return "", 0, err 185 | } 186 | 187 | return listenerUid, operation, nil 188 | 189 | } 190 | func (d *Driver) sanitizeTableName(tableName string) string { 191 | segments := strings.Split(tableName, ".") 192 | for i, segment := range segments { 193 | segments[i] = `"` + segment + `"` 194 | } 195 | return strings.Join(segments, ".") 196 | } 197 | func (d *Driver) sqlExec(conn *sql.DB, query string) (sql.Result, error) { 198 | d._clientConfig.Logger.Trace().Str("query", query).Msg("sending sql request") 199 | return conn.Exec(query) 200 | } 201 | 202 | func (d *Driver) getConditionsSql(conditions []*flash.ListenerCondition, table string) (string, error) { 203 | rawConditions := make([]string, len(conditions)) 204 | 205 | for i, condition := range conditions { 206 | operator := " IS " 207 | valueRepr := "" 208 | // TODO MULTI OPERATOR 209 | 210 | switch condition.Value.(type) { 211 | case nil: 212 | valueRepr = "NULL" 213 | case bool: 214 | if condition.Value.(bool) == true { 215 | valueRepr = "TRUE" 216 | } else { 217 | valueRepr = "FALSE" 218 | } 219 | case string, time.Time: 220 | valueRepr = fmt.Sprintf(`'%s'`, condition.Value) 221 | case float32, float64: 222 | valueRepr = fmt.Sprintf(`%f`, condition.Value) 223 | case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: 224 | valueRepr = fmt.Sprintf(`%d`, condition.Value) 225 | default: 226 | return "", errors.New("could not convert condition value to sql") 227 | } 228 | 229 | rawConditions[i] = fmt.Sprintf(`%s."%s"%s%s`, table, condition.Column, operator, valueRepr) 230 | 231 | } 232 | return strings.Join(rawConditions, " AND "), nil 233 | } 234 | -------------------------------------------------------------------------------- /drivers/wal_logical/replicator.go: -------------------------------------------------------------------------------- 1 | package wal_logical 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "github.com/jackc/pglogrepl" 8 | "github.com/jackc/pgx/v5/pgconn" 9 | "github.com/jackc/pgx/v5/pgproto3" 10 | "github.com/jackc/pgx/v5/pgtype" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | type replicationState struct { 16 | lastReceivedLSN pglogrepl.LSN 17 | currentTransactionLSN pglogrepl.LSN 18 | lastWrittenLSN pglogrepl.LSN 19 | 20 | typeMap *pgtype.Map 21 | relations map[uint32]*pglogrepl.RelationMessageV2 22 | 23 | processMessages bool 24 | inStream bool 25 | streamQueues map[uint32][]*pglogrepl.Message 26 | 27 | restartChan chan struct{} 28 | } 29 | 30 | func (d *Driver) initReplicator() error { 31 | d.replicationState = &replicationState{ 32 | lastWrittenLSN: pglogrepl.LSN(0), //TODO KEEP IN FILE OR IGNORE 33 | relations: make(map[uint32]*pglogrepl.RelationMessageV2), 34 | typeMap: pgtype.NewMap(), 35 | streamQueues: make(map[uint32][]*pglogrepl.Message), 36 | restartChan: make(chan struct{}), 37 | } 38 | return nil 39 | } 40 | 41 | func (d *Driver) startReplicator() error { 42 | if err := d.startConn(); err != nil { 43 | d._clientConfig.Logger.Error().Err(err).Msgf("received err: %s", err) 44 | return err 45 | } 46 | if err := d.startReplication(); err != nil { 47 | d._clientConfig.Logger.Error().Err(err).Msgf("received err: %s", err) 48 | return err 49 | } 50 | 51 | /* LISTENING */ 52 | standbyMessageTimeout := time.Second * 10 53 | nextStandbyMessageDeadline := time.Now().Add(standbyMessageTimeout) 54 | 55 | for { 56 | select { 57 | case <-d.replicationState.restartChan: 58 | 59 | if d.replicationConn == nil { 60 | continue 61 | } 62 | if err := d.replicationConn.Close(context.Background()); err != nil { 63 | d._clientConfig.Logger.Error().Err(err).Msgf("received err: %s", err) 64 | return err 65 | } 66 | if err := d.startConn(); err != nil { 67 | d._clientConfig.Logger.Error().Err(err).Msgf("received err: %s", err) 68 | return err 69 | } 70 | if err := d.startReplication(); err != nil { 71 | d._clientConfig.Logger.Error().Err(err).Msgf("received err: %s", err) 72 | return err 73 | } 74 | 75 | continue 76 | 77 | default: 78 | if d.replicationConn == nil { 79 | time.Sleep(time.Millisecond * 100) 80 | continue 81 | } 82 | 83 | if time.Now().After(nextStandbyMessageDeadline) && d.replicationState.lastReceivedLSN > 0 { 84 | err := pglogrepl.SendStandbyStatusUpdate(context.Background(), d.replicationConn, pglogrepl.StandbyStatusUpdate{ 85 | WALWritePosition: d.replicationState.lastWrittenLSN + 1, 86 | WALFlushPosition: d.replicationState.lastWrittenLSN + 1, 87 | WALApplyPosition: d.replicationState.lastReceivedLSN + 1, 88 | }) 89 | if err != nil { 90 | d._clientConfig.Logger.Error().Err(err).Msgf("received err: %s", err) 91 | return err 92 | } 93 | d._clientConfig.Logger.Trace().Msg("Sent Standby status message at " + (d.replicationState.lastWrittenLSN + 1).String()) 94 | nextStandbyMessageDeadline = time.Now().Add(standbyMessageTimeout) 95 | } 96 | 97 | ctx, cancel := context.WithDeadline(context.Background(), nextStandbyMessageDeadline) 98 | rawMsg, err := d.replicationConn.ReceiveMessage(ctx) 99 | cancel() 100 | 101 | if err != nil { 102 | if pgconn.Timeout(err) { 103 | continue 104 | } 105 | d._clientConfig.Logger.Warn().Err(err).Msgf("received err: %s", err) 106 | time.Sleep(time.Millisecond * 100) 107 | continue // CLOSED CONNECTION TODO handle and return err when needed 108 | } 109 | 110 | if errMsg, ok := rawMsg.(*pgproto3.ErrorResponse); ok { 111 | return errors.New(errMsg.Message) 112 | } 113 | 114 | msg, ok := rawMsg.(*pgproto3.CopyData) 115 | if !ok { 116 | d._clientConfig.Logger.Warn().Msg(fmt.Sprintf("Received unexpected message: %T", rawMsg)) 117 | continue 118 | } 119 | 120 | switch msg.Data[0] { 121 | case pglogrepl.PrimaryKeepaliveMessageByteID: 122 | pkm, err := pglogrepl.ParsePrimaryKeepaliveMessage(msg.Data[1:]) 123 | if err != nil { 124 | return err 125 | } 126 | d._clientConfig.Logger.Trace().Msg(fmt.Sprintf("Primary Keepalive Message => ServerWALEnd: %s ServerTime: %s ReplyRequested: %t", pkm.ServerWALEnd, pkm.ServerTime, pkm.ReplyRequested)) 127 | 128 | d.replicationState.lastReceivedLSN = pkm.ServerWALEnd 129 | 130 | if pkm.ReplyRequested { 131 | nextStandbyMessageDeadline = time.Time{} 132 | } 133 | 134 | case pglogrepl.XLogDataByteID: 135 | xld, err := pglogrepl.ParseXLogData(msg.Data[1:]) 136 | if err != nil { 137 | return err 138 | } 139 | //d._clientConfig.Logger.Trace().Msg(fmt.Sprintf("XLogData => WALStart %s ServerWALEnd %s ServerTime %s WALData: %s", xld.WALStart, xld.ServerWALEnd, xld.ServerTime, rawMsg)) 140 | 141 | updateLsn, err := d.processXld(&xld) 142 | if err != nil { 143 | return err 144 | } 145 | if updateLsn { 146 | d.replicationState.lastWrittenLSN = xld.ServerWALEnd 147 | // TODO write wal position in file if needed 148 | nextStandbyMessageDeadline = time.Time{} // Force resend standby message 149 | } 150 | } 151 | 152 | } 153 | } 154 | } 155 | 156 | func (d *Driver) closeReplicator() error { 157 | if d.replicationConn != nil { 158 | // CLOSE ACTUAL 159 | if err := d.replicationConn.Close(context.Background()); err != nil { 160 | d._clientConfig.Logger.Error().Err(err).Msgf("received err: %s", err) 161 | return err 162 | } 163 | //REMAKE NEW CONN WITHOUT STARTING REPLICATION 164 | if err := d.startConn(); err != nil { 165 | d._clientConfig.Logger.Error().Err(err).Msgf("received err: %s", err) 166 | return err 167 | } 168 | dropReplicationSql := fmt.Sprintf(`select pg_drop_replication_slot(slot_name) from pg_replication_slots where slot_name = '%s';`, d.Config.ReplicationSlot) 169 | _, err := d.sqlExec(d.replicationConn, dropReplicationSql) 170 | if err != nil { 171 | d._clientConfig.Logger.Error().Err(err).Msgf("received err: %s", err) 172 | return err 173 | } 174 | // CLOSE TEMP 175 | if err := d.replicationConn.Close(context.Background()); err != nil { 176 | d._clientConfig.Logger.Error().Err(err).Msgf("received err: %s", err) 177 | return err 178 | } 179 | 180 | d.replicationConn = nil 181 | } 182 | return nil 183 | } 184 | 185 | func (d *Driver) startConn() error { 186 | // Create querying and listening connections 187 | config, err := pgconn.ParseConfig(d._clientConfig.DatabaseCnx) 188 | if err != nil { 189 | return err 190 | } 191 | config.RuntimeParams["application_name"] = "Flash: replication (replicator)" 192 | config.RuntimeParams["replication"] = "database" 193 | 194 | if d.replicationConn, err = pgconn.ConnectConfig(context.Background(), config); err != nil { 195 | return err 196 | } 197 | 198 | // Create false publication to avoid START_REPLICATION error 199 | initSlotName := d.getFullSlotName("init") 200 | 201 | // DROP OLD 202 | dropPublicationSql := d.getDropPublicationSlotSql(initSlotName) 203 | dropReplicationSql := fmt.Sprintf(`select pg_drop_replication_slot(slot_name) from pg_replication_slots where slot_name = '%s';`, d.Config.ReplicationSlot) 204 | createPublicationSlotSql, err := d.getCreatePublicationSlotSql(initSlotName, nil, nil) 205 | if err != nil { 206 | return err 207 | } 208 | 209 | d.activePublications[initSlotName] = true 210 | 211 | if _, err := d.sqlExec(d.replicationConn, dropPublicationSql+dropReplicationSql+createPublicationSlotSql); err != nil { 212 | return err 213 | } 214 | 215 | return nil 216 | } 217 | 218 | func (d *Driver) startReplication() error { 219 | if _, err := d.sqlExec(d.replicationConn, fmt.Sprintf(`CREATE_REPLICATION_SLOT "%s" TEMPORARY LOGICAL "pgoutput";`, d.Config.ReplicationSlot)); err != nil { 220 | return err 221 | } 222 | 223 | initSlotName := d.getFullSlotName("init") 224 | activePublications := []string{initSlotName} 225 | for publicationName, _ := range d.activePublications { 226 | activePublications = append(activePublications, publicationName) 227 | } 228 | replicationOptions := pglogrepl.StartReplicationOptions{ 229 | Mode: pglogrepl.LogicalReplication, 230 | PluginArgs: []string{ 231 | "proto_version '2'", // Keep as version 2 to compatibility 232 | "publication_names '" + strings.Join(activePublications, ", ") + "'", 233 | "messages 'true'", 234 | }, 235 | } 236 | if d.Config.UseStreaming { 237 | replicationOptions.PluginArgs = append(replicationOptions.PluginArgs, "streaming 'true'") 238 | } 239 | 240 | if err := pglogrepl.StartReplication(context.Background(), d.replicationConn, d.Config.ReplicationSlot, d.replicationState.lastWrittenLSN+1, replicationOptions); err != nil { 241 | return err 242 | } 243 | d._clientConfig.Logger.Debug().Msg("Started replication slot: " + d.Config.ReplicationSlot) 244 | return nil 245 | } 246 | -------------------------------------------------------------------------------- /driver_testcase.go: -------------------------------------------------------------------------------- 1 | package flash 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/rs/zerolog" 7 | "github.com/testcontainers/testcontainers-go" 8 | "github.com/testcontainers/testcontainers-go/modules/postgres" 9 | "io" 10 | "os" 11 | "testing" 12 | "time" 13 | ) 14 | 15 | func runTests(t *testing.T, test TestFn, driver Driver, tc *DriverTestConfig, cc *ClientConfig, execSql ExecSqlFunc) { 16 | 17 | /* ------------------------------------------- INITIALIZATION TEST-------------------------------*/ 18 | test(t, "Can be initialized", func(t *testing.T) { 19 | defer driver.Close() 20 | 21 | err := driver.Init(cc) 22 | if err != nil { 23 | t.Error(err) 24 | } 25 | }, true) 26 | 27 | test(t, "Can be closed", func(t *testing.T) { 28 | _ = driver.Init(cc) 29 | if err := driver.Close(); err != nil { 30 | t.Error(err) 31 | } 32 | }, true) 33 | 34 | test(t, "Listen keep running at least 3 seconds without error when no listeners exists", func(t *testing.T) { 35 | defer driver.Close() 36 | _ = driver.Init(cc) 37 | 38 | errChan := make(chan error, 1) 39 | go func() { 40 | c := make(DatabaseEventsChan) 41 | errChan <- driver.Listen(&c) 42 | }() 43 | 44 | ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 45 | defer cancel() 46 | 47 | select { 48 | case err := <-errChan: 49 | if err != nil { 50 | t.Errorf("Listen returned an error: %v", err) 51 | } 52 | case <-ctx.Done(): 53 | return 54 | } 55 | }, true) 56 | 57 | /* ------------------------------------------- RUNTIME TEST-------------------------------*/ 58 | _ = driver.Init(cc) 59 | go func() { 60 | eventChan := make(DatabaseEventsChan) 61 | _ = driver.Listen(&eventChan) 62 | }() 63 | defer driver.Close() 64 | 65 | type ListenerConfigTestMap struct { 66 | Name string 67 | listenerConfig *ListenerConfig 68 | } 69 | for ti, testEntry := range []ListenerConfigTestMap{ 70 | {Name: "All fields", listenerConfig: &ListenerConfig{Table: "posts"}}, 71 | {Name: "Partial fields", listenerConfig: &ListenerConfig{Table: "posts", Fields: []string{"active", "slug"}}}, 72 | {Name: "All fields with conditions", listenerConfig: &ListenerConfig{ 73 | Table: "posts", 74 | Conditions: []*ListenerCondition{{Column: "active", Value: true}}, 75 | }}, 76 | {Name: "Partial fields with conditions", listenerConfig: &ListenerConfig{ 77 | Table: "posts", 78 | Fields: []string{"id", "active"}, 79 | Conditions: []*ListenerCondition{{Column: "slug", Value: nil}}, 80 | }}, 81 | } { 82 | for _, operation := range []Operation{ 83 | OperationInsert, 84 | OperationUpdate, 85 | OperationDelete, 86 | OperationTruncate, 87 | } { 88 | test(t, "HandleOperationListenStart - "+testEntry.Name+" - "+operation.String(), func(t *testing.T) { 89 | errChan := make(chan error, 1) 90 | go func() { 91 | errChan <- driver.HandleOperationListenStart(fmt.Sprintf(`uid-%d`, ti), testEntry.listenerConfig, operation) 92 | }() 93 | 94 | ctx, cancel := context.WithTimeout(context.Background(), tc.RegistrationTimeout) 95 | defer cancel() 96 | 97 | select { 98 | case err := <-errChan: 99 | if err != nil { 100 | t.Errorf("HandleOperationListenStart returned an error: %v", err) 101 | } 102 | case <-ctx.Done(): 103 | t.Errorf("HandleOperationListenStart timed out") 104 | } 105 | }, false) 106 | test(t, "HandleOperationListenStop - "+testEntry.Name+" - "+operation.String(), func(t *testing.T) { 107 | lc := &ListenerConfig{Table: "posts"} 108 | errChan := make(chan error, 1) 109 | go func() { 110 | errChan <- driver.HandleOperationListenStop(fmt.Sprintf(`uid-%d`, ti), lc, operation) 111 | }() 112 | 113 | ctx, cancel := context.WithTimeout(context.Background(), tc.RegistrationTimeout) 114 | defer cancel() 115 | 116 | select { 117 | case err := <-errChan: 118 | if err != nil { 119 | t.Errorf("HandleOperationListenStop returned an error: %v", err) 120 | } 121 | case <-ctx.Done(): 122 | t.Errorf("HandleOperationListenStop timed out") 123 | } 124 | }, false) 125 | } 126 | } 127 | 128 | } 129 | 130 | type DriverTestConfig struct { 131 | ImagesVersions []string `default:"postgres,flash"` 132 | 133 | Database string 134 | Username string 135 | Password string 136 | 137 | ContainerCustomizers []testcontainers.ContainerCustomizer 138 | 139 | PropagationTimeout time.Duration // Delay for event propagated from the DB to the eventsChan 140 | RegistrationTimeout time.Duration // Delay for OperationListenStart / HandleOperationListenStop 141 | 142 | Parallel bool 143 | } 144 | 145 | var DefaultDriverTestConfig = &DriverTestConfig{ 146 | ImagesVersions: []string{ 147 | // Standard PostgreSQL 148 | "docker.io/postgres:14-alpine", 149 | "docker.io/postgres:15-alpine", 150 | "docker.io/postgres:16-alpine", 151 | 152 | // PgVector 153 | // "docker.io/pgvector/pgvector:pg14", 154 | // "docker.io/pgvector/pgvector:pg15", 155 | // "docker.io/pgvector/pgvector:pg16", 156 | 157 | // PostGIS 158 | // "docker.io/postgis/postgis:14-3.4-alpine", 159 | // "docker.io/postgis/postgis:15-3.4-alpine", 160 | // "docker.io/postgis/postgis:16-3.4-alpine", 161 | 162 | // TimescaleDB 163 | // "docker.io/timescale/timescaledb:latest-pg14", 164 | // "docker.io/timescale/timescaledb:latest-pg15", 165 | // "docker.io/timescale/timescaledb:latest-pg16", 166 | }, 167 | 168 | Database: "testdb", 169 | Username: "testuser", 170 | Password: "testpasword", 171 | 172 | PropagationTimeout: time.Second, 173 | RegistrationTimeout: time.Second, 174 | 175 | Parallel: false, // DO NOT WORK 176 | } 177 | 178 | type TestFn func(t *testing.T, name string, f func(t *testing.T), restore bool) 179 | 180 | func RunFlashDriverTestCase[T Driver](t *testing.T, config *DriverTestConfig, getDriverCb func() T) { 181 | if config == nil { 182 | config = DefaultDriverTestConfig 183 | } 184 | for _, image := range config.ImagesVersions { 185 | t.Run(image, func(t *testing.T) { 186 | 187 | driverInstance := getDriverCb() 188 | 189 | t.Parallel() 190 | 191 | dbCnx, conn, container := startPostgresContainer(t, config, image) 192 | logger := zerolog.New(os.Stdout).Level(zerolog.FatalLevel).With().Caller().Stack().Timestamp().Logger() 193 | clientConfig := &ClientConfig{ 194 | DatabaseCnx: dbCnx, 195 | Driver: driverInstance, 196 | Logger: &logger, 197 | } 198 | 199 | testFn := func(t *testing.T, name string, f func(t *testing.T), restore bool) { 200 | t.Run(name, func(t *testing.T) { 201 | if restore { 202 | t.Cleanup(func() { 203 | restoreSnapshot(t, container) 204 | }) 205 | } 206 | // USE LOCK in parallel to avoid restore snapshot during 207 | 208 | //if config.Parallel { 209 | // t.Parallel() TODO 210 | //} 211 | 212 | f(t) 213 | }) 214 | } 215 | 216 | runTests(t, testFn, driverInstance, config, clientConfig, conn) 217 | }) 218 | } 219 | } 220 | 221 | type ExecSqlFunc func(t *testing.T, sql string) string 222 | 223 | func startPostgresContainer(t *testing.T, config *DriverTestConfig, image string) (string, ExecSqlFunc, *postgres.PostgresContainer) { 224 | ctx := context.Background() 225 | 226 | customizers := config.ContainerCustomizers 227 | customizers = append(customizers, 228 | postgres.WithDatabase(config.Database), 229 | postgres.WithUsername(config.Username), 230 | postgres.WithPassword(config.Password), 231 | postgres.BasicWaitStrategies(), 232 | ) 233 | 234 | container, err := postgres.Run(ctx, image, customizers...) 235 | if err != nil { 236 | t.Error(err) 237 | } 238 | 239 | // Clean up the container after the test is complete 240 | t.Cleanup(func() { 241 | if err := container.Terminate(ctx); err != nil { 242 | t.Errorf("failed to terminate container: %s", err) 243 | } 244 | }) 245 | 246 | // explicitly set sslmode=disable because the container is not configured to use TLS 247 | connStr, err := container.ConnectionString(ctx, "sslmode=disable") 248 | if err != nil { 249 | t.Error(err) 250 | } 251 | 252 | execSql := func(t *testing.T, sql string) string { 253 | code, r, err := container.Exec(context.Background(), []string{"psql", "-U", config.Username, "-d", config.Database, "-c", sql}) 254 | if err != nil { 255 | t.Error(err) 256 | } 257 | bytes, err := io.ReadAll(r) 258 | if err != nil { 259 | t.Error(err) 260 | } 261 | 262 | if code != 0 { 263 | t.Error(string(bytes)) 264 | } 265 | return string(bytes) 266 | } 267 | 268 | // Bootstrap DB with default table 269 | execSql(t, bootstrapSql) 270 | 271 | // Create restore point for later 272 | err = container.Snapshot(context.Background(), postgres.WithSnapshotName("db-snapshot")) 273 | if err != nil { 274 | t.Error(err) 275 | } 276 | 277 | return connStr, execSql, container 278 | } 279 | 280 | const bootstrapSql = ` 281 | CREATE TABLE posts ( 282 | id SERIAL PRIMARY KEY, 283 | slug VARCHAR(255), 284 | active BOOLEAN NOT NULL DEFAULT FALSE 285 | ); 286 | 287 | CREATE INDEX idx_active ON posts (active); 288 | 289 | INSERT INTO posts (slug, active) VALUES 290 | ('slug1', true), 291 | ('slug2', false), 292 | ('slug3', true), 293 | ('slug4', false), 294 | ('slug5', true), 295 | ('slug6', false), 296 | ('slug7', true), 297 | ('slug8', false), 298 | ('slug9', true), 299 | ('slug10', false), 300 | ('slug11', true), 301 | ('slug12', false), 302 | ('slug13', true), 303 | ('slug14', false), 304 | ('slug15', true), 305 | ('slug16', false), 306 | ('slug17', true), 307 | ('slug18', false), 308 | ('slug19', true), 309 | (NULL, false) 310 | ` 311 | 312 | func restoreSnapshot(t *testing.T, container *postgres.PostgresContainer) { 313 | ctx := context.Background() 314 | err := container.Restore(ctx) 315 | if err != nil { 316 | t.Fatalf("failed to restore snapshot: %v", err) 317 | } 318 | } 319 | -------------------------------------------------------------------------------- /drivers/wal_logical/process.go: -------------------------------------------------------------------------------- 1 | package wal_logical 2 | 3 | import ( 4 | "fmt" 5 | "github.com/jackc/pglogrepl" 6 | "github.com/jackc/pgx/v5/pgtype" 7 | "github.com/quix-labs/flash" 8 | "reflect" 9 | ) 10 | 11 | func (d *Driver) processXld(xld *pglogrepl.XLogData) (bool, error) { 12 | logicalMsg, err := pglogrepl.ParseV2(xld.WALData, d.replicationState.inStream) 13 | if err != nil { 14 | return false, err 15 | } 16 | 17 | d.replicationState.lastReceivedLSN = xld.ServerWALEnd 18 | return d.processMessage(logicalMsg, false) 19 | } 20 | 21 | func (d *Driver) processMessage(logicalMsg pglogrepl.Message, fromQueue bool) (bool, error) { 22 | switch typedLogicalMsg := logicalMsg.(type) { 23 | case *pglogrepl.RelationMessageV2: 24 | d.replicationState.relations[typedLogicalMsg.RelationID] = typedLogicalMsg 25 | 26 | case *pglogrepl.BeginMessage: 27 | if d.replicationState.lastWrittenLSN > typedLogicalMsg.FinalLSN { 28 | d._clientConfig.Logger.Trace().Msgf("Received stale message, ignoring. Last written LSN: %s Message LSN: %s", d.replicationState.lastWrittenLSN, typedLogicalMsg.FinalLSN) 29 | d.replicationState.processMessages = false 30 | break 31 | } 32 | 33 | d.replicationState.processMessages = true 34 | d.replicationState.currentTransactionLSN = typedLogicalMsg.FinalLSN 35 | 36 | case *pglogrepl.CommitMessage: 37 | d.replicationState.processMessages = false 38 | return true, nil 39 | 40 | case *pglogrepl.InsertMessageV2: 41 | // If we are in replicationState, append XLogData to memory to run/delete after stream commit/abort 42 | if d.replicationState.inStream && !fromQueue { 43 | d.replicationState.streamQueues[typedLogicalMsg.Xid] = append(d.replicationState.streamQueues[typedLogicalMsg.Xid], &logicalMsg) 44 | break 45 | } 46 | 47 | if !d.replicationState.processMessages && !fromQueue { 48 | // Stale message 49 | break 50 | } 51 | 52 | tableName, _ := d.getRelationTableName(typedLogicalMsg.RelationID) 53 | listeners, exists := d.activeListeners[tableName] 54 | if !exists { 55 | break 56 | } 57 | 58 | newData, err := d.parseTuple(typedLogicalMsg.RelationID, typedLogicalMsg.Tuple) 59 | if err != nil { 60 | return false, err 61 | } 62 | for listenerUid, listenerConfig := range listeners { 63 | 64 | if !d.checkConditions(newData, listenerConfig.Conditions) { 65 | continue 66 | } 67 | 68 | reducedNewData := d.ExtractFields(newData, listenerConfig.Fields) 69 | *d.eventsChan <- &flash.DatabaseEvent{ 70 | ListenerUid: listenerUid, 71 | Event: &flash.InsertEvent{New: reducedNewData}, 72 | } 73 | } 74 | 75 | case *pglogrepl.UpdateMessageV2: 76 | // If we are in replicationState, append XLogData to memory to run/delete after stream commit/abort 77 | if d.replicationState.inStream && !fromQueue { 78 | d.replicationState.streamQueues[typedLogicalMsg.Xid] = append(d.replicationState.streamQueues[typedLogicalMsg.Xid], &logicalMsg) 79 | break 80 | } 81 | 82 | if !d.replicationState.processMessages && !fromQueue { 83 | // Stale message 84 | break 85 | } 86 | 87 | tableName, _ := d.getRelationTableName(typedLogicalMsg.RelationID) 88 | listeners, exists := d.activeListeners[tableName] 89 | if !exists { 90 | break 91 | } 92 | 93 | newData, err := d.parseTuple(typedLogicalMsg.RelationID, typedLogicalMsg.NewTuple) 94 | if err != nil { 95 | return false, err 96 | } 97 | 98 | oldData, err := d.parseTuple(typedLogicalMsg.RelationID, typedLogicalMsg.OldTuple) 99 | if err != nil { 100 | return false, err 101 | } 102 | for listenerUid, listenerConfig := range listeners { 103 | 104 | if len(listenerConfig.Conditions) > 0 { 105 | // HANDLING CONDITIONS - e.g: SOFT DELETE 106 | oldRespectConditions := d.checkConditions(oldData, listenerConfig.Conditions) 107 | newRespectConditions := d.checkConditions(newData, listenerConfig.Conditions) 108 | if !oldRespectConditions && !newRespectConditions { 109 | continue 110 | } 111 | 112 | if !oldRespectConditions && newRespectConditions { 113 | // IN THIS CASE, THIS IS AN INSERT 114 | *d.eventsChan <- &flash.DatabaseEvent{ 115 | ListenerUid: listenerUid, 116 | Event: &flash.InsertEvent{New: d.ExtractFields(newData, listenerConfig.Fields)}, 117 | } 118 | continue 119 | } 120 | 121 | if oldRespectConditions && !newRespectConditions { 122 | // IN THIS CASE, THIS IS A DELETE 123 | *d.eventsChan <- &flash.DatabaseEvent{ 124 | ListenerUid: listenerUid, 125 | Event: &flash.DeleteEvent{Old: d.ExtractFields(oldData, listenerConfig.Fields)}, 126 | } 127 | continue 128 | } 129 | } 130 | 131 | reducedOldData := d.ExtractFields(oldData, listenerConfig.Fields) 132 | reducedNewData := d.ExtractFields(newData, listenerConfig.Fields) 133 | if d.CheckEquals(reducedNewData, reducedOldData) { 134 | continue //Ignore operation if update is not in listener fields 135 | } 136 | *d.eventsChan <- &flash.DatabaseEvent{ 137 | ListenerUid: listenerUid, 138 | Event: &flash.UpdateEvent{Old: reducedOldData, New: reducedNewData}, 139 | } 140 | } 141 | 142 | case *pglogrepl.DeleteMessageV2: 143 | // If we are in replicationState, append XLogData to memory to run/delete after stream commit/abort 144 | if d.replicationState.inStream && !fromQueue { 145 | d.replicationState.streamQueues[typedLogicalMsg.Xid] = append(d.replicationState.streamQueues[typedLogicalMsg.Xid], &logicalMsg) 146 | break 147 | } 148 | 149 | if !d.replicationState.processMessages && !fromQueue { 150 | // Stale message 151 | break 152 | } 153 | 154 | tableName, _ := d.getRelationTableName(typedLogicalMsg.RelationID) 155 | listeners, exists := d.activeListeners[tableName] 156 | if !exists { 157 | break 158 | } 159 | oldData, err := d.parseTuple(typedLogicalMsg.RelationID, typedLogicalMsg.OldTuple) 160 | if err != nil { 161 | return false, err 162 | } 163 | for listenerUid, listenerConfig := range listeners { 164 | 165 | if !d.checkConditions(oldData, listenerConfig.Conditions) { 166 | continue 167 | } 168 | 169 | reducedOldData := d.ExtractFields(oldData, listenerConfig.Fields) 170 | *d.eventsChan <- &flash.DatabaseEvent{ 171 | ListenerUid: listenerUid, 172 | Event: &flash.DeleteEvent{Old: reducedOldData}, 173 | } 174 | } 175 | 176 | case *pglogrepl.TruncateMessageV2: 177 | // If we are in replicationState, append XLogData to memory to run/delete after stream commit/abort 178 | if d.replicationState.inStream && !fromQueue { 179 | d.replicationState.streamQueues[typedLogicalMsg.Xid] = append(d.replicationState.streamQueues[typedLogicalMsg.Xid], &logicalMsg) 180 | break 181 | } 182 | 183 | if !d.replicationState.processMessages && !fromQueue { 184 | // Stale message 185 | break 186 | } 187 | 188 | for _, relId := range typedLogicalMsg.RelationIDs { 189 | tableName, _ := d.getRelationTableName(relId) 190 | listeners, exists := d.activeListeners[tableName] 191 | if !exists { 192 | break 193 | } 194 | for listenerUid, _ := range listeners { 195 | *d.eventsChan <- &flash.DatabaseEvent{ 196 | ListenerUid: listenerUid, 197 | Event: &flash.TruncateEvent{}, 198 | } 199 | } 200 | } 201 | case *pglogrepl.TypeMessageV2: 202 | d._clientConfig.Logger.Trace().Msgf("typeMessage for xid %d\n", typedLogicalMsg.Xid) 203 | case *pglogrepl.OriginMessage: 204 | d._clientConfig.Logger.Trace().Msgf("originMessage for xid %s\n", typedLogicalMsg.Name) 205 | case *pglogrepl.LogicalDecodingMessageV2: 206 | d._clientConfig.Logger.Trace().Msgf("Logical decoding message: %q, %q, %d", typedLogicalMsg.Prefix, typedLogicalMsg.Content, typedLogicalMsg.Xid) 207 | 208 | case *pglogrepl.StreamStartMessageV2: 209 | d.replicationState.inStream = true 210 | // Create dynamic queue if not exists 211 | if _, exists := d.replicationState.streamQueues[typedLogicalMsg.Xid]; !exists { 212 | d.replicationState.streamQueues[typedLogicalMsg.Xid] = []*pglogrepl.Message{} // Dynamic size 213 | } 214 | d._clientConfig.Logger.Trace().Msgf("Stream start message: xid %d, first segment? %d", typedLogicalMsg.Xid, typedLogicalMsg.FirstSegment) 215 | 216 | case *pglogrepl.StreamStopMessageV2: 217 | d.replicationState.inStream = false 218 | d._clientConfig.Logger.Trace().Msgf("Stream stop message") 219 | case *pglogrepl.StreamCommitMessageV2: 220 | d._clientConfig.Logger.Trace().Msgf("Stream commit message: xid %d", typedLogicalMsg.Xid) 221 | 222 | // Process all operations then remove queue 223 | queueLen := len(d.replicationState.streamQueues[typedLogicalMsg.Xid]) 224 | if queueLen > 0 { 225 | d._clientConfig.Logger.Trace().Msgf("Processing %d entries from stream queue: xid %d", queueLen, typedLogicalMsg.Xid) 226 | // ⚠️ Do not use goroutine to handle in parallel, order is very important 227 | for _, message := range d.replicationState.streamQueues[typedLogicalMsg.Xid] { 228 | // Cannot flush position here because return statement can cause loss 229 | _, err := d.processMessage(*message, true) 230 | if err != nil { 231 | return false, err 232 | } 233 | } 234 | } 235 | d._clientConfig.Logger.Trace().Msgf("Delete %d entries from stream queue: xid %d", queueLen, typedLogicalMsg.Xid) 236 | delete(d.replicationState.streamQueues, typedLogicalMsg.Xid) 237 | return true, nil // FLUSH position 238 | 239 | case *pglogrepl.StreamAbortMessageV2: 240 | d._clientConfig.Logger.Trace().Msgf("Stream abort message: xid %d", typedLogicalMsg.Xid) 241 | d._clientConfig.Logger.Trace().Msgf("Delete %d entries from stream queue: xid %d", len(d.replicationState.streamQueues[typedLogicalMsg.Xid]), typedLogicalMsg.Xid) 242 | delete(d.replicationState.streamQueues, typedLogicalMsg.Xid) 243 | default: 244 | d._clientConfig.Logger.Trace().Msgf("Unknown message type in pgoutput stream: %T", typedLogicalMsg) 245 | } 246 | 247 | return false, nil 248 | } 249 | 250 | func (d *Driver) parseTuple(relationID uint32, tuple *pglogrepl.TupleData) (*flash.EventData, error) { 251 | rel, ok := d.replicationState.relations[relationID] 252 | if !ok { 253 | return nil, fmt.Errorf("unknown relation ID %d", relationID) 254 | } 255 | if len(tuple.Columns) == 0 { 256 | return nil, nil 257 | } 258 | values := flash.EventData{} //Initialize as nil and create only on first col 259 | for idx, col := range tuple.Columns { 260 | colName := rel.Columns[idx].Name 261 | switch col.DataType { 262 | case 'n': // null 263 | values[colName] = nil 264 | case 'u': // unchanged toast 265 | // This TOAST value was not changed. TOAST values are not stored in the tuple, and logical replication doesn't want to spend a disk read to fetch its value for you. 266 | case 't': //text 267 | val, err := d.decodeTextColumnData(col.Data, rel.Columns[idx].DataType) 268 | if err != nil { 269 | return nil, err 270 | } 271 | values[colName] = val 272 | } 273 | } 274 | return &values, nil 275 | } 276 | 277 | func (d *Driver) ExtractFields(data *flash.EventData, fields []string) *flash.EventData { 278 | if len(fields) == 0 { // Empty same as SELECT * 279 | return data 280 | } 281 | 282 | reducedData := flash.EventData{} 283 | for _, field := range fields { 284 | reducedData[field] = (*data)[field] 285 | } 286 | return &reducedData 287 | } 288 | func (d *Driver) CheckEquals(source any, target any) bool { 289 | return reflect.DeepEqual(source, target) 290 | } 291 | 292 | func (d *Driver) getRelationTableName(relationID uint32) (string, error) { 293 | rel, ok := d.replicationState.relations[relationID] 294 | if !ok { 295 | return "", fmt.Errorf("unknown relation ID %d", relationID) 296 | } 297 | return rel.Namespace + "." + rel.RelationName, nil 298 | } 299 | 300 | func (d *Driver) decodeTextColumnData(data []byte, dataType uint32) (interface{}, error) { 301 | if dt, ok := d.replicationState.typeMap.TypeForOID(dataType); ok { 302 | return dt.Codec.DecodeValue(d.replicationState.typeMap, dataType, pgtype.TextFormatCode, data) 303 | } 304 | return string(data), nil 305 | } 306 | 307 | func (d *Driver) checkConditions(data *flash.EventData, conditions []*flash.ListenerCondition) bool { 308 | for _, condition := range conditions { 309 | value := (*data)[condition.Column] 310 | if value != condition.Value { 311 | return false 312 | } 313 | } 314 | return true 315 | } 316 | --------------------------------------------------------------------------------