├── .env.dev ├── .gitignore ├── LICENSE ├── README.md ├── db └── migrations │ └── 001-create-organization.sql ├── docker-compose.yaml ├── kv-sync.ts ├── package-lock.json ├── package.json ├── worker.ts └── wrangler.toml /.env.dev: -------------------------------------------------------------------------------- 1 | DATABASE_URL=postgresql://postgres:password@localhost:54321/electric 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | .pnpm-debug.log* 9 | 10 | # Diagnostic reports (https://nodejs.org/api/report.html) 11 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 12 | 13 | # Runtime data 14 | pids 15 | *.pid 16 | *.seed 17 | *.pid.lock 18 | 19 | # Directory for instrumented libs generated by jscoverage/JSCover 20 | lib-cov 21 | 22 | # Coverage directory used by tools like istanbul 23 | coverage 24 | *.lcov 25 | 26 | # nyc test coverage 27 | .nyc_output 28 | 29 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 30 | .grunt 31 | 32 | # Bower dependency directory (https://bower.io/) 33 | bower_components 34 | 35 | # node-waf configuration 36 | .lock-wscript 37 | 38 | # Compiled binary addons (https://nodejs.org/api/addons.html) 39 | build/Release 40 | 41 | # Dependency directories 42 | node_modules/ 43 | jspm_packages/ 44 | 45 | # Snowpack dependency directory (https://snowpack.dev/) 46 | web_modules/ 47 | 48 | # TypeScript cache 49 | *.tsbuildinfo 50 | 51 | # Optional npm cache directory 52 | .npm 53 | 54 | # Optional eslint cache 55 | .eslintcache 56 | 57 | # Optional stylelint cache 58 | .stylelintcache 59 | 60 | # Microbundle cache 61 | .rpt2_cache/ 62 | .rts2_cache_cjs/ 63 | .rts2_cache_es/ 64 | .rts2_cache_umd/ 65 | 66 | # Optional REPL history 67 | .node_repl_history 68 | 69 | # Output of 'npm pack' 70 | *.tgz 71 | 72 | # Yarn Integrity file 73 | .yarn-integrity 74 | 75 | # dotenv environment variable files 76 | .env 77 | .env.development.local 78 | .env.test.local 79 | .env.production.local 80 | .env.local 81 | 82 | # parcel-bundler cache (https://parceljs.org/) 83 | .cache 84 | .parcel-cache 85 | 86 | # Next.js build output 87 | .next 88 | out 89 | 90 | # Nuxt.js build / generate output 91 | .nuxt 92 | dist 93 | 94 | # Gatsby files 95 | .cache/ 96 | # Comment in the public line in if your project uses Gatsby and not Next.js 97 | # https://nextjs.org/blog/next-9-1#public-directory-support 98 | # public 99 | 100 | # vuepress build output 101 | .vuepress/dist 102 | 103 | # vuepress v2.x temp and cache directory 104 | .temp 105 | .cache 106 | 107 | # Docusaurus cache and generated files 108 | .docusaurus 109 | 110 | # Serverless directories 111 | .serverless/ 112 | 113 | # FuseBox cache 114 | .fusebox/ 115 | 116 | # DynamoDB Local files 117 | .dynamodb/ 118 | 119 | # TernJS port file 120 | .tern-port 121 | 122 | # Stores VSCode versions used for testing VSCode extensions 123 | .vscode-test 124 | 125 | # yarn v2 126 | .yarn/cache 127 | .yarn/unplugged 128 | .yarn/build-state.yml 129 | .yarn/install-state.gz 130 | .pnp.* 131 | .wrangler 132 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # electric-cloudflare-kv-sync 2 | 3 | Demo showing how to sync data with ElectricSQL from Postgres to Cloudflare's Workers KV 4 | 5 | Edge Workers often need instant access to Postgres data. 6 | 7 | The typical options are poor: 8 | 9 | - Querying to the db is slow. 10 | - Maintaining a cache isn't ideal as you either need to hook up cache invalidation or accept stale data. 11 | 12 | Sync Engines like [ElectricSQL](https://next.electric-sql.com/) are a systematic fix to these sorts of "I want an up-to-date copy of some Postgres data in this other system". Electric lets you subscribe to what we call a [shape](https://next.electric-sql.com/guides/shapes), which is basically a table with a where clause, and then any changes to the database within that shape will be sent to subscribers 13 | 14 | Check out the demo (sound on). 15 | 16 | https://github.com/user-attachments/assets/cebc1eaf-d8ae-4603-83fa-920b974bc04d 17 | 18 | The sync code is in `kv-sync.ts` and worker code in `worker.ts`. 19 | 20 | To run this locally. 21 | 22 | 1. clone 23 | 2. `npm install` 24 | 3. start the PG/Electric backends (Docker should be installed) `npm run backend:up` 25 | 4. start the worker: `npm run start` 26 | 5. start the syncer: `npm run sync` (note, the port of the worker is hard-coded but might be different on your machine). Open the worker in the browser to see what port it's on. 27 | 6. curl the worker `curl http://localhost:65094/` — you should get back an empty array. 28 | 7. Insert some organizations: 29 | 30 | ```sql 31 | insert into 32 | organizations ( 33 | name, 34 | address, 35 | city, 36 | state, 37 | postal_code, 38 | country, 39 | phone_number, 40 | email, 41 | website 42 | ) 43 | values 44 | ( 45 | 'Tech Innovators Inc.', 46 | '123 Tech Lane', 47 | 'San Francisco', 48 | 'CA', 49 | '94105', 50 | 'USA', 51 | '415-555-1234', 52 | 'info@techinnovators.com', 53 | 'www.techinnovators.com' 54 | ), 55 | ( 56 | 'Green Solutions LLC', 57 | '456 Greenway Blvd', 58 | 'Austin', 59 | 'TX', 60 | '78701', 61 | 'USA', 62 | '512-555-5678', 63 | 'contact@greensolutions.com', 64 | 'www.greensolutions.com' 65 | ), 66 | ( 67 | 'Health First Corp.', 68 | '789 Wellness Ave', 69 | 'New York', 70 | 'NY', 71 | '10001', 72 | 'USA', 73 | '212-555-9012', 74 | 'support@healthfirst.com', 75 | 'www.healthfirst.com' 76 | ); 77 | ``` 78 | 8. Curl again — you should see the data. Then try updating a field or two and deleting an org and curl to see how the data is instantly synced. 79 | -------------------------------------------------------------------------------- /db/migrations/001-create-organization.sql: -------------------------------------------------------------------------------- 1 | create table organizations ( 2 | id bigint primary key generated always as identity, 3 | name text not null, 4 | address text, 5 | city text, 6 | state text, 7 | postal_code text, 8 | country text, 9 | phone_number text, 10 | email text unique, 11 | website text 12 | ); 13 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "3.3" 2 | name: "electric_cloudflare_sync" 3 | 4 | services: 5 | postgres: 6 | image: postgres:16-alpine 7 | environment: 8 | POSTGRES_DB: electric 9 | POSTGRES_USER: postgres 10 | POSTGRES_PASSWORD: password 11 | ports: 12 | - 54321:5432 13 | tmpfs: 14 | - /var/lib/postgresql/data 15 | - /tmp 16 | command: 17 | - -c 18 | - listen_addresses=* 19 | - -c 20 | - wal_level=logical 21 | 22 | electric: 23 | image: electricsql/electric 24 | environment: 25 | DATABASE_URL: postgresql://postgres:password@postgres:5432/electric 26 | ports: 27 | - "3000:3000" 28 | depends_on: 29 | - postgres 30 | -------------------------------------------------------------------------------- /kv-sync.ts: -------------------------------------------------------------------------------- 1 | import { ShapeStream, ChangeMessage, ControlMessage, Message } from "@electric-sql/client"; 2 | 3 | function isChangeMessage(message: Message): message is ControlMessage { 4 | return "key" in message; 5 | } 6 | 7 | const stream = new ShapeStream({ 8 | url: `http://localhost:3000/v1/shape/organizations`, 9 | parser: { 10 | int8: (value) => parseInt(value, 10), 11 | }, 12 | }); 13 | 14 | stream.subscribe(async (messages) => { 15 | // Remove control messages and transform to the id: rowValue form we'll want 16 | // in the Worker KV store. 17 | const transformedMessages = messages 18 | .filter((message) => isChangeMessage(message)) 19 | .map((message: ChangeMessage) => { 20 | return { 21 | [message.value.id]: JSON.stringify(message.value), 22 | operation: message.headers.operation 23 | }; 24 | }); 25 | 26 | await fetch(`http://localhost:65094`, { 27 | method: `POST`, 28 | body: JSON.stringify(transformedMessages), 29 | }); 30 | }); 31 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "electric-cloudflare-kv-sync", 3 | "description": "Demo showing how to sync data with ElectricSQL from Postgres to Cloudflare's Workers KV", 4 | "version": "1.0.0", 5 | "author": "Kyle Mathews ", 6 | "bugs": { 7 | "url": "https://github.com/KyleAMathews/electric-cloudflare-kv-sync/issues" 8 | }, 9 | "dependencies": { 10 | "@cloudflare/workers-types": "^4.20240806.0", 11 | "@databases/pg-migrations": "^5.0.3", 12 | "@electric-sql/client": "^0.3.1", 13 | "dotenv-cli": "^7.4.2", 14 | "wrangler": "^3.72.0" 15 | }, 16 | "homepage": "https://github.com/KyleAMathews/electric-cloudflare-kv-sync#readme", 17 | "keywords": [ 18 | "cloudflare", 19 | "electricsql" 20 | ], 21 | "license": "Unlicense", 22 | "repository": { 23 | "type": "git", 24 | "url": "git+https://github.com/KyleAMathews/electric-cloudflare-kv-sync.git" 25 | }, 26 | "scripts": { 27 | "backend:down": "docker compose --env-file .env.dev -f docker-compose.yaml down --volumes", 28 | "backend:up": "docker compose --env-file .env.dev -f docker-compose.yaml up -d && npm run db:migrate", 29 | "db:migrate": "dotenv -e .env.dev -- npx pg-migrations apply --directory ./db/migrations", 30 | "start": "wrangler dev" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /worker.ts: -------------------------------------------------------------------------------- 1 | import { KVNamespace, ExportedHandler } from '@cloudflare/workers-types' 2 | 3 | export interface Env { 4 | organizations: KVNamespace; 5 | } 6 | 7 | export default { 8 | async fetch(request, env): Promise { 9 | try { 10 | // Write to organizations KV namespace when kv-sync sends updates from Postgres. 11 | if (request.method === `POST`) { 12 | const body: any[] = await request.json() 13 | console.log(body) 14 | for (const entry of body) { 15 | const [[key,value], [_operation, operation]] = Object.entries(entry) 16 | if (operation === `insert`) { 17 | await env.organizations.put(key, value as string); 18 | } else if (operation === `update`) { 19 | // Merge updates into the old value 20 | const oldValue = await env.organizations.get(key) 21 | const mergedValue = {...JSON.parse(oldValue), ...JSON.parse(value as string)} 22 | await env.organizations.put(key, JSON.stringify(mergedValue)); 23 | } else if (operation === `delete`) { 24 | await env.organizations.delete(key) 25 | } 26 | } 27 | return new Response(`ok`) 28 | } else if (request.method === `GET`) { 29 | // Get all organization values from the KV store and return them. 30 | const list = await env.organizations.list() 31 | console.log(list) 32 | const values = await Promise.all(list.keys.map(async ({name}) => { 33 | const value = await env.organizations.get(name) 34 | return JSON.parse(value) 35 | })) 36 | return new Response(JSON.stringify(values, null, 4)); 37 | } 38 | } catch (err) { 39 | console.error(`KV returned error: ${err}`) 40 | return new Response(err, { status: 500 }) 41 | } 42 | }, 43 | } satisfies ExportedHandler; 44 | -------------------------------------------------------------------------------- /wrangler.toml: -------------------------------------------------------------------------------- 1 | name = "electric-sync-kv-example" 2 | main = "worker.ts" 3 | compatibility_date = "2024-08-06" 4 | 5 | kv_namespaces = [ 6 | { binding = "organizations", id = "1e12706189a24be08a1a8446dd75a672" } 7 | ] 8 | --------------------------------------------------------------------------------