├── .env.example ├── .github └── images │ └── readme-illustration.png ├── .gitignore ├── README.md ├── package.json ├── pnpm-lock.yaml ├── src ├── dump.ts ├── index.ts ├── inject.ts ├── schemas │ ├── generatePrismaUtilsTypes.ts │ ├── source.prisma │ └── target.prisma ├── sync.ts └── utils │ ├── fileName.ts │ ├── isEmpty.ts │ └── parseProgressBarFormat.ts └── tsconfig.json /.env.example: -------------------------------------------------------------------------------- 1 | SOURCE_DATABASE_URL= 2 | TARGET_DATABASE_URL= 3 | SYNC_INTERVAL_MS=30000 4 | -------------------------------------------------------------------------------- /.github/images/readme-illustration.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baptisteArno/prisma-database-sync/0cc133321298f0f1467dc38f0376f2ad408b4437/.github/images/readme-illustration.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | node_modules 3 | prisma-clients 4 | snapshots 5 | migrations 6 | .DS_Store -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | Readme illustration 3 |

Prisma Databases Sync

4 |

5 | 6 | I implemented this CLI because I had to migrate from a PostgreSQL database to a MySQL database for my startup [Typebot](https://typebot.io/). Prisma takes care of the data transformation and this allows you to migrate without downtime. 7 | 8 | > I just tested the scripts with my setup. You may have to tweak things for yours. Feel free to create an issue so that I can help 👌 9 | 10 | ## Features 11 | 12 | - Sync data from and to any database (PostgreSQL, MySQL, MongoDB etc...) as long as it is supported by Prisma 13 | - Incremental sync (based on `@updatedAt` or `@createdAt`). To avoid extracting all the data every time. 14 | 15 | You just have to provide 2 prisma schemas: source and target. 16 | 17 | The first data sync will take some time because it will download all the data from the beginning. Then, subsequent syncs will be tiny and will happen in an instant. 18 | 19 | ## Get started 20 | 21 | 1. Clone the repository 22 | 2. Copy `.env.example` to `.env` and fill it with your database URLs (It is recommended to disable the prisma pool timeout using the `?pool_timeout=0` query param). 23 | 3. Edit `src/schemas/source.prisma` and `src/schemas/target.prisma` with your schemas. Make sure it contains: 24 | 25 | ``` 26 | generator utils { 27 | provider = "pnpm tsx src/generatePrismaUtilsTypes.ts" 28 | output = "prisma-clients/target" 29 | } 30 | ``` 31 | 32 | 4. Install dependencies and generate prisma clients: `pnpm install` 33 | 5. Run the CLI: `pnpm start`, it will popup the menu 34 | ``` 35 | ? › - Use arrow-keys. Return to submit. 36 | ❯ Sync - Watch for changes in your source database and inject it in your target database 37 | Dump 38 | Inject 39 | ``` 40 | 41 | ## How to use 42 | 43 | This library has 3 functions `dump`, `inject`, `sync`: 44 | 45 | - `dump` reads your source database and generate timestamped snapshots 46 | - `inject` injects the previously imported snapshots chronologically 47 | - `sync` executes dump and restore with a set interval 48 | 49 | This is perfect if you are planning on migrating to another database in production. The first dump will be quite big as it will pull all the data from the beginning. Subsequent dumps will be tiny. 50 | 51 | Let's say you need to migrate to a new database and can't afford to have application downtime: 52 | 53 | 1. Launch a sync job to sync the target database with the source database. 54 | 2. Deploy your application version that consumes the target database 55 | 3. Once it's deployed, make sure the sync job doesn't detect new data from source database for at least 30 minutes. 56 | 4. 🎉 Congrats, you migrated to a new database with no downtime. 57 | 58 | ## Limitations 59 | 60 | It doesn't detect if a row has been deleted on the source database. 61 | 62 | Foreign keys can make it difficult to sync data. Two options are: 63 | 64 | - Defer constraints (recommended) 65 | - Disable foreign key triggers, which can silently break referential integrity (not recommended) 66 | 67 | If your target schema uses `prisma` relationMode (or if it's a MongoDB database), you need to set the relationMode to `foreignKeys` 68 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "prisma-database-sync", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "generate": "prisma generate --schema src/schemas/source.prisma && prisma generate --schema src/schemas/target.prisma", 8 | "reset:source": "prisma migrate reset --schema src/schemas/source.prisma", 9 | "reset:target": "prisma migrate reset --schema src/schemas/target.prisma", 10 | "start": "tsx src/index.ts", 11 | "postinstall": "pnpm generate" 12 | }, 13 | "keywords": [], 14 | "author": "Baptiste Arnaud", 15 | "license": "ISC", 16 | "dependencies": { 17 | "@prisma/client": "^4.10.0", 18 | "cli-progress": "^3.11.2", 19 | "prompts": "2.4.2", 20 | "stream-chain": "^2.2.5", 21 | "stream-json": "^1.7.5" 22 | }, 23 | "devDependencies": { 24 | "@prisma/generator-helper": "^4.10.0", 25 | "@types/cli-progress": "^3.11.0", 26 | "@types/node": "^18.13.0", 27 | "@types/prompts": "2.4.2", 28 | "@types/stream-json": "^1.7.3", 29 | "dotenv": "^16.0.3", 30 | "prisma": "^4.10.0", 31 | "tsx": "^3.12.3" 32 | }, 33 | "packageManager": "pnpm@7.27.0" 34 | } 35 | -------------------------------------------------------------------------------- /pnpm-lock.yaml: -------------------------------------------------------------------------------- 1 | lockfileVersion: 5.4 2 | 3 | specifiers: 4 | '@prisma/client': ^4.10.0 5 | '@prisma/generator-helper': ^4.10.0 6 | '@types/cli-progress': ^3.11.0 7 | '@types/node': ^18.13.0 8 | '@types/prompts': 2.4.2 9 | '@types/stream-json': ^1.7.3 10 | cli-progress: ^3.11.2 11 | dotenv: ^16.0.3 12 | prisma: ^4.10.0 13 | prompts: 2.4.2 14 | stream-chain: ^2.2.5 15 | stream-json: ^1.7.5 16 | tsx: ^3.12.3 17 | 18 | dependencies: 19 | '@prisma/client': 4.10.0_prisma@4.10.0 20 | cli-progress: 3.11.2 21 | prompts: 2.4.2 22 | stream-chain: 2.2.5 23 | stream-json: 1.7.5 24 | 25 | devDependencies: 26 | '@prisma/generator-helper': 4.10.0 27 | '@types/cli-progress': 3.11.0 28 | '@types/node': 18.13.0 29 | '@types/prompts': 2.4.2 30 | '@types/stream-json': 1.7.3 31 | dotenv: 16.0.3 32 | prisma: 4.10.0 33 | tsx: 3.12.3 34 | 35 | packages: 36 | 37 | /@esbuild-kit/cjs-loader/2.4.2: 38 | resolution: {integrity: sha512-BDXFbYOJzT/NBEtp71cvsrGPwGAMGRB/349rwKuoxNSiKjPraNNnlK6MIIabViCjqZugu6j+xeMDlEkWdHHJSg==} 39 | dependencies: 40 | '@esbuild-kit/core-utils': 3.0.0 41 | get-tsconfig: 4.4.0 42 | dev: true 43 | 44 | /@esbuild-kit/core-utils/3.0.0: 45 | resolution: {integrity: sha512-TXmwH9EFS3DC2sI2YJWJBgHGhlteK0Xyu1VabwetMULfm3oYhbrsWV5yaSr2NTWZIgDGVLHbRf0inxbjXqAcmQ==} 46 | dependencies: 47 | esbuild: 0.15.18 48 | source-map-support: 0.5.21 49 | dev: true 50 | 51 | /@esbuild-kit/esm-loader/2.5.5: 52 | resolution: {integrity: sha512-Qwfvj/qoPbClxCRNuac1Du01r9gvNOT+pMYtJDapfB1eoGN1YlJ1BixLyL9WVENRx5RXgNLdfYdx/CuswlGhMw==} 53 | dependencies: 54 | '@esbuild-kit/core-utils': 3.0.0 55 | get-tsconfig: 4.4.0 56 | dev: true 57 | 58 | /@esbuild/android-arm/0.15.18: 59 | resolution: {integrity: sha512-5GT+kcs2WVGjVs7+boataCkO5Fg0y4kCjzkB5bAip7H4jfnOS3dA6KPiww9W1OEKTKeAcUVhdZGvgI65OXmUnw==} 60 | engines: {node: '>=12'} 61 | cpu: [arm] 62 | os: [android] 63 | requiresBuild: true 64 | dev: true 65 | optional: true 66 | 67 | /@esbuild/linux-loong64/0.15.18: 68 | resolution: {integrity: sha512-L4jVKS82XVhw2nvzLg/19ClLWg0y27ulRwuP7lcyL6AbUWB5aPglXY3M21mauDQMDfRLs8cQmeT03r/+X3cZYQ==} 69 | engines: {node: '>=12'} 70 | cpu: [loong64] 71 | os: [linux] 72 | requiresBuild: true 73 | dev: true 74 | optional: true 75 | 76 | /@prisma/client/4.10.0_prisma@4.10.0: 77 | resolution: {integrity: sha512-sBmYb1S6SMKFIESaLMfKqWSalv3pH73cMCsFt9HslJvYjIIcKQCA6PDL2O4SZGWvc4JBef9cg5Gd7d9x3AtKjw==} 78 | engines: {node: '>=14.17'} 79 | requiresBuild: true 80 | peerDependencies: 81 | prisma: '*' 82 | peerDependenciesMeta: 83 | prisma: 84 | optional: true 85 | dependencies: 86 | '@prisma/engines-version': 4.10.0-84.ca7fcef713137fa11029d519a9780db130cca91d 87 | prisma: 4.10.0 88 | dev: false 89 | 90 | /@prisma/debug/4.10.0: 91 | resolution: {integrity: sha512-rxVOZKsEyjlQCwN/pkkJO7wEdARt1yRyukSjLa+BF2QTvy2+VgtBmrfys4WDQSnj3jVWeHMpi5GeAoJjKkSKyA==} 92 | dependencies: 93 | '@types/debug': 4.1.7 94 | debug: 4.3.4 95 | strip-ansi: 6.0.1 96 | transitivePeerDependencies: 97 | - supports-color 98 | dev: true 99 | 100 | /@prisma/engines-version/4.10.0-84.ca7fcef713137fa11029d519a9780db130cca91d: 101 | resolution: {integrity: sha512-UVpmVlvSaGfY4ue+hh8CTkIesbuXCFUfrr8zk//+u85WwkKfWMtt6nLB2tNSzR1YO8eAA8+HqNf8LM7mnXIq5w==} 102 | dev: false 103 | 104 | /@prisma/engines/4.10.0: 105 | resolution: {integrity: sha512-ZPPo7q+nQZdTlPFedS7mFXPE3oZ2kWtTh3GO4sku0XQ8ikLqEyinuTPJbQCw/8qel2xglIEQicsK6yI4Jgh20A==} 106 | requiresBuild: true 107 | 108 | /@prisma/generator-helper/4.10.0: 109 | resolution: {integrity: sha512-NkQOfZpHUjVjqJ7NN2FymHSLkGd/E0fz5c3RkyESKvQqBy2sFBxt+aFxGsUbUy3FfwvkckC04HdQOXpisAko0A==} 110 | dependencies: 111 | '@prisma/debug': 4.10.0 112 | '@types/cross-spawn': 6.0.2 113 | chalk: 4.1.2 114 | cross-spawn: 7.0.3 115 | transitivePeerDependencies: 116 | - supports-color 117 | dev: true 118 | 119 | /@types/cli-progress/3.11.0: 120 | resolution: {integrity: sha512-XhXhBv1R/q2ahF3BM7qT5HLzJNlIL0wbcGyZVjqOTqAybAnsLisd7gy1UCyIqpL+5Iv6XhlSyzjLCnI2sIdbCg==} 121 | dependencies: 122 | '@types/node': 18.13.0 123 | dev: true 124 | 125 | /@types/cross-spawn/6.0.2: 126 | resolution: {integrity: sha512-KuwNhp3eza+Rhu8IFI5HUXRP0LIhqH5cAjubUvGXXthh4YYBuP2ntwEX+Cz8GJoZUHlKo247wPWOfA9LYEq4cw==} 127 | dependencies: 128 | '@types/node': 18.13.0 129 | dev: true 130 | 131 | /@types/debug/4.1.7: 132 | resolution: {integrity: sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg==} 133 | dependencies: 134 | '@types/ms': 0.7.31 135 | dev: true 136 | 137 | /@types/ms/0.7.31: 138 | resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==} 139 | dev: true 140 | 141 | /@types/node/18.13.0: 142 | resolution: {integrity: sha512-gC3TazRzGoOnoKAhUx+Q0t8S9Tzs74z7m0ipwGpSqQrleP14hKxP4/JUeEQcD3W1/aIpnWl8pHowI7WokuZpXg==} 143 | dev: true 144 | 145 | /@types/prompts/2.4.2: 146 | resolution: {integrity: sha512-TwNx7qsjvRIUv/BCx583tqF5IINEVjCNqg9ofKHRlSoUHE62WBHrem4B1HGXcIrG511v29d1kJ9a/t2Esz7MIg==} 147 | dependencies: 148 | '@types/node': 18.13.0 149 | kleur: 3.0.3 150 | dev: true 151 | 152 | /@types/stream-chain/2.0.1: 153 | resolution: {integrity: sha512-D+Id9XpcBpampptkegH7WMsEk6fUdf9LlCIX7UhLydILsqDin4L0QT7ryJR0oycwC7OqohIzdfcMHVZ34ezNGg==} 154 | dependencies: 155 | '@types/node': 18.13.0 156 | dev: true 157 | 158 | /@types/stream-json/1.7.3: 159 | resolution: {integrity: sha512-Jqsyq5VPOTWorvEmzWhEWH5tJnHA+bB8vt/Zzb11vSDj8esfSHDMj2rbVjP0mfJQzl3YBJSXBBq08iiyaBK3KA==} 160 | dependencies: 161 | '@types/node': 18.13.0 162 | '@types/stream-chain': 2.0.1 163 | dev: true 164 | 165 | /ansi-regex/5.0.1: 166 | resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} 167 | engines: {node: '>=8'} 168 | 169 | /ansi-styles/4.3.0: 170 | resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} 171 | engines: {node: '>=8'} 172 | dependencies: 173 | color-convert: 2.0.1 174 | dev: true 175 | 176 | /buffer-from/1.1.2: 177 | resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} 178 | dev: true 179 | 180 | /chalk/4.1.2: 181 | resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} 182 | engines: {node: '>=10'} 183 | dependencies: 184 | ansi-styles: 4.3.0 185 | supports-color: 7.2.0 186 | dev: true 187 | 188 | /cli-progress/3.11.2: 189 | resolution: {integrity: sha512-lCPoS6ncgX4+rJu5bS3F/iCz17kZ9MPZ6dpuTtI0KXKABkhyXIdYB3Inby1OpaGti3YlI3EeEkM9AuWpelJrVA==} 190 | engines: {node: '>=4'} 191 | dependencies: 192 | string-width: 4.2.3 193 | dev: false 194 | 195 | /color-convert/2.0.1: 196 | resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} 197 | engines: {node: '>=7.0.0'} 198 | dependencies: 199 | color-name: 1.1.4 200 | dev: true 201 | 202 | /color-name/1.1.4: 203 | resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} 204 | dev: true 205 | 206 | /cross-spawn/7.0.3: 207 | resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} 208 | engines: {node: '>= 8'} 209 | dependencies: 210 | path-key: 3.1.1 211 | shebang-command: 2.0.0 212 | which: 2.0.2 213 | dev: true 214 | 215 | /debug/4.3.4: 216 | resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} 217 | engines: {node: '>=6.0'} 218 | peerDependencies: 219 | supports-color: '*' 220 | peerDependenciesMeta: 221 | supports-color: 222 | optional: true 223 | dependencies: 224 | ms: 2.1.2 225 | dev: true 226 | 227 | /dotenv/16.0.3: 228 | resolution: {integrity: sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==} 229 | engines: {node: '>=12'} 230 | dev: true 231 | 232 | /emoji-regex/8.0.0: 233 | resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} 234 | dev: false 235 | 236 | /esbuild-android-64/0.15.18: 237 | resolution: {integrity: sha512-wnpt3OXRhcjfIDSZu9bnzT4/TNTDsOUvip0foZOUBG7QbSt//w3QV4FInVJxNhKc/ErhUxc5z4QjHtMi7/TbgA==} 238 | engines: {node: '>=12'} 239 | cpu: [x64] 240 | os: [android] 241 | requiresBuild: true 242 | dev: true 243 | optional: true 244 | 245 | /esbuild-android-arm64/0.15.18: 246 | resolution: {integrity: sha512-G4xu89B8FCzav9XU8EjsXacCKSG2FT7wW9J6hOc18soEHJdtWu03L3TQDGf0geNxfLTtxENKBzMSq9LlbjS8OQ==} 247 | engines: {node: '>=12'} 248 | cpu: [arm64] 249 | os: [android] 250 | requiresBuild: true 251 | dev: true 252 | optional: true 253 | 254 | /esbuild-darwin-64/0.15.18: 255 | resolution: {integrity: sha512-2WAvs95uPnVJPuYKP0Eqx+Dl/jaYseZEUUT1sjg97TJa4oBtbAKnPnl3b5M9l51/nbx7+QAEtuummJZW0sBEmg==} 256 | engines: {node: '>=12'} 257 | cpu: [x64] 258 | os: [darwin] 259 | requiresBuild: true 260 | dev: true 261 | optional: true 262 | 263 | /esbuild-darwin-arm64/0.15.18: 264 | resolution: {integrity: sha512-tKPSxcTJ5OmNb1btVikATJ8NftlyNlc8BVNtyT/UAr62JFOhwHlnoPrhYWz09akBLHI9nElFVfWSTSRsrZiDUA==} 265 | engines: {node: '>=12'} 266 | cpu: [arm64] 267 | os: [darwin] 268 | requiresBuild: true 269 | dev: true 270 | optional: true 271 | 272 | /esbuild-freebsd-64/0.15.18: 273 | resolution: {integrity: sha512-TT3uBUxkteAjR1QbsmvSsjpKjOX6UkCstr8nMr+q7zi3NuZ1oIpa8U41Y8I8dJH2fJgdC3Dj3CXO5biLQpfdZA==} 274 | engines: {node: '>=12'} 275 | cpu: [x64] 276 | os: [freebsd] 277 | requiresBuild: true 278 | dev: true 279 | optional: true 280 | 281 | /esbuild-freebsd-arm64/0.15.18: 282 | resolution: {integrity: sha512-R/oVr+X3Tkh+S0+tL41wRMbdWtpWB8hEAMsOXDumSSa6qJR89U0S/PpLXrGF7Wk/JykfpWNokERUpCeHDl47wA==} 283 | engines: {node: '>=12'} 284 | cpu: [arm64] 285 | os: [freebsd] 286 | requiresBuild: true 287 | dev: true 288 | optional: true 289 | 290 | /esbuild-linux-32/0.15.18: 291 | resolution: {integrity: sha512-lphF3HiCSYtaa9p1DtXndiQEeQDKPl9eN/XNoBf2amEghugNuqXNZA/ZovthNE2aa4EN43WroO0B85xVSjYkbg==} 292 | engines: {node: '>=12'} 293 | cpu: [ia32] 294 | os: [linux] 295 | requiresBuild: true 296 | dev: true 297 | optional: true 298 | 299 | /esbuild-linux-64/0.15.18: 300 | resolution: {integrity: sha512-hNSeP97IviD7oxLKFuii5sDPJ+QHeiFTFLoLm7NZQligur8poNOWGIgpQ7Qf8Balb69hptMZzyOBIPtY09GZYw==} 301 | engines: {node: '>=12'} 302 | cpu: [x64] 303 | os: [linux] 304 | requiresBuild: true 305 | dev: true 306 | optional: true 307 | 308 | /esbuild-linux-arm/0.15.18: 309 | resolution: {integrity: sha512-UH779gstRblS4aoS2qpMl3wjg7U0j+ygu3GjIeTonCcN79ZvpPee12Qun3vcdxX+37O5LFxz39XeW2I9bybMVA==} 310 | engines: {node: '>=12'} 311 | cpu: [arm] 312 | os: [linux] 313 | requiresBuild: true 314 | dev: true 315 | optional: true 316 | 317 | /esbuild-linux-arm64/0.15.18: 318 | resolution: {integrity: sha512-54qr8kg/6ilcxd+0V3h9rjT4qmjc0CccMVWrjOEM/pEcUzt8X62HfBSeZfT2ECpM7104mk4yfQXkosY8Quptug==} 319 | engines: {node: '>=12'} 320 | cpu: [arm64] 321 | os: [linux] 322 | requiresBuild: true 323 | dev: true 324 | optional: true 325 | 326 | /esbuild-linux-mips64le/0.15.18: 327 | resolution: {integrity: sha512-Mk6Ppwzzz3YbMl/ZZL2P0q1tnYqh/trYZ1VfNP47C31yT0K8t9s7Z077QrDA/guU60tGNp2GOwCQnp+DYv7bxQ==} 328 | engines: {node: '>=12'} 329 | cpu: [mips64el] 330 | os: [linux] 331 | requiresBuild: true 332 | dev: true 333 | optional: true 334 | 335 | /esbuild-linux-ppc64le/0.15.18: 336 | resolution: {integrity: sha512-b0XkN4pL9WUulPTa/VKHx2wLCgvIAbgwABGnKMY19WhKZPT+8BxhZdqz6EgkqCLld7X5qiCY2F/bfpUUlnFZ9w==} 337 | engines: {node: '>=12'} 338 | cpu: [ppc64] 339 | os: [linux] 340 | requiresBuild: true 341 | dev: true 342 | optional: true 343 | 344 | /esbuild-linux-riscv64/0.15.18: 345 | resolution: {integrity: sha512-ba2COaoF5wL6VLZWn04k+ACZjZ6NYniMSQStodFKH/Pu6RxzQqzsmjR1t9QC89VYJxBeyVPTaHuBMCejl3O/xg==} 346 | engines: {node: '>=12'} 347 | cpu: [riscv64] 348 | os: [linux] 349 | requiresBuild: true 350 | dev: true 351 | optional: true 352 | 353 | /esbuild-linux-s390x/0.15.18: 354 | resolution: {integrity: sha512-VbpGuXEl5FCs1wDVp93O8UIzl3ZrglgnSQ+Hu79g7hZu6te6/YHgVJxCM2SqfIila0J3k0csfnf8VD2W7u2kzQ==} 355 | engines: {node: '>=12'} 356 | cpu: [s390x] 357 | os: [linux] 358 | requiresBuild: true 359 | dev: true 360 | optional: true 361 | 362 | /esbuild-netbsd-64/0.15.18: 363 | resolution: {integrity: sha512-98ukeCdvdX7wr1vUYQzKo4kQ0N2p27H7I11maINv73fVEXt2kyh4K4m9f35U1K43Xc2QGXlzAw0K9yoU7JUjOg==} 364 | engines: {node: '>=12'} 365 | cpu: [x64] 366 | os: [netbsd] 367 | requiresBuild: true 368 | dev: true 369 | optional: true 370 | 371 | /esbuild-openbsd-64/0.15.18: 372 | resolution: {integrity: sha512-yK5NCcH31Uae076AyQAXeJzt/vxIo9+omZRKj1pauhk3ITuADzuOx5N2fdHrAKPxN+zH3w96uFKlY7yIn490xQ==} 373 | engines: {node: '>=12'} 374 | cpu: [x64] 375 | os: [openbsd] 376 | requiresBuild: true 377 | dev: true 378 | optional: true 379 | 380 | /esbuild-sunos-64/0.15.18: 381 | resolution: {integrity: sha512-On22LLFlBeLNj/YF3FT+cXcyKPEI263nflYlAhz5crxtp3yRG1Ugfr7ITyxmCmjm4vbN/dGrb/B7w7U8yJR9yw==} 382 | engines: {node: '>=12'} 383 | cpu: [x64] 384 | os: [sunos] 385 | requiresBuild: true 386 | dev: true 387 | optional: true 388 | 389 | /esbuild-windows-32/0.15.18: 390 | resolution: {integrity: sha512-o+eyLu2MjVny/nt+E0uPnBxYuJHBvho8vWsC2lV61A7wwTWC3jkN2w36jtA+yv1UgYkHRihPuQsL23hsCYGcOQ==} 391 | engines: {node: '>=12'} 392 | cpu: [ia32] 393 | os: [win32] 394 | requiresBuild: true 395 | dev: true 396 | optional: true 397 | 398 | /esbuild-windows-64/0.15.18: 399 | resolution: {integrity: sha512-qinug1iTTaIIrCorAUjR0fcBk24fjzEedFYhhispP8Oc7SFvs+XeW3YpAKiKp8dRpizl4YYAhxMjlftAMJiaUw==} 400 | engines: {node: '>=12'} 401 | cpu: [x64] 402 | os: [win32] 403 | requiresBuild: true 404 | dev: true 405 | optional: true 406 | 407 | /esbuild-windows-arm64/0.15.18: 408 | resolution: {integrity: sha512-q9bsYzegpZcLziq0zgUi5KqGVtfhjxGbnksaBFYmWLxeV/S1fK4OLdq2DFYnXcLMjlZw2L0jLsk1eGoB522WXQ==} 409 | engines: {node: '>=12'} 410 | cpu: [arm64] 411 | os: [win32] 412 | requiresBuild: true 413 | dev: true 414 | optional: true 415 | 416 | /esbuild/0.15.18: 417 | resolution: {integrity: sha512-x/R72SmW3sSFRm5zrrIjAhCeQSAWoni3CmHEqfQrZIQTM3lVCdehdwuIqaOtfC2slvpdlLa62GYoN8SxT23m6Q==} 418 | engines: {node: '>=12'} 419 | hasBin: true 420 | requiresBuild: true 421 | optionalDependencies: 422 | '@esbuild/android-arm': 0.15.18 423 | '@esbuild/linux-loong64': 0.15.18 424 | esbuild-android-64: 0.15.18 425 | esbuild-android-arm64: 0.15.18 426 | esbuild-darwin-64: 0.15.18 427 | esbuild-darwin-arm64: 0.15.18 428 | esbuild-freebsd-64: 0.15.18 429 | esbuild-freebsd-arm64: 0.15.18 430 | esbuild-linux-32: 0.15.18 431 | esbuild-linux-64: 0.15.18 432 | esbuild-linux-arm: 0.15.18 433 | esbuild-linux-arm64: 0.15.18 434 | esbuild-linux-mips64le: 0.15.18 435 | esbuild-linux-ppc64le: 0.15.18 436 | esbuild-linux-riscv64: 0.15.18 437 | esbuild-linux-s390x: 0.15.18 438 | esbuild-netbsd-64: 0.15.18 439 | esbuild-openbsd-64: 0.15.18 440 | esbuild-sunos-64: 0.15.18 441 | esbuild-windows-32: 0.15.18 442 | esbuild-windows-64: 0.15.18 443 | esbuild-windows-arm64: 0.15.18 444 | dev: true 445 | 446 | /fsevents/2.3.2: 447 | resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} 448 | engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} 449 | os: [darwin] 450 | requiresBuild: true 451 | dev: true 452 | optional: true 453 | 454 | /get-tsconfig/4.4.0: 455 | resolution: {integrity: sha512-0Gdjo/9+FzsYhXCEFueo2aY1z1tpXrxWZzP7k8ul9qt1U5o8rYJwTJYmaeHdrVosYIVYkOy2iwCJ9FdpocJhPQ==} 456 | dev: true 457 | 458 | /has-flag/4.0.0: 459 | resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} 460 | engines: {node: '>=8'} 461 | dev: true 462 | 463 | /is-fullwidth-code-point/3.0.0: 464 | resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} 465 | engines: {node: '>=8'} 466 | dev: false 467 | 468 | /isexe/2.0.0: 469 | resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} 470 | dev: true 471 | 472 | /kleur/3.0.3: 473 | resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==} 474 | engines: {node: '>=6'} 475 | 476 | /ms/2.1.2: 477 | resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} 478 | dev: true 479 | 480 | /path-key/3.1.1: 481 | resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} 482 | engines: {node: '>=8'} 483 | dev: true 484 | 485 | /prisma/4.10.0: 486 | resolution: {integrity: sha512-xUHcF3Glc8QGgW8x0rfPITvyyTo04fskUdG7pI4kQbvDX/rhzDP4046x/FvazYqYHXMLR5/KTIi2p2Gth5vKOQ==} 487 | engines: {node: '>=14.17'} 488 | hasBin: true 489 | requiresBuild: true 490 | dependencies: 491 | '@prisma/engines': 4.10.0 492 | 493 | /prompts/2.4.2: 494 | resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==} 495 | engines: {node: '>= 6'} 496 | dependencies: 497 | kleur: 3.0.3 498 | sisteransi: 1.0.5 499 | dev: false 500 | 501 | /shebang-command/2.0.0: 502 | resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} 503 | engines: {node: '>=8'} 504 | dependencies: 505 | shebang-regex: 3.0.0 506 | dev: true 507 | 508 | /shebang-regex/3.0.0: 509 | resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} 510 | engines: {node: '>=8'} 511 | dev: true 512 | 513 | /sisteransi/1.0.5: 514 | resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} 515 | dev: false 516 | 517 | /source-map-support/0.5.21: 518 | resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==} 519 | dependencies: 520 | buffer-from: 1.1.2 521 | source-map: 0.6.1 522 | dev: true 523 | 524 | /source-map/0.6.1: 525 | resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} 526 | engines: {node: '>=0.10.0'} 527 | dev: true 528 | 529 | /stream-chain/2.2.5: 530 | resolution: {integrity: sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==} 531 | dev: false 532 | 533 | /stream-json/1.7.5: 534 | resolution: {integrity: sha512-NSkoVduGakxZ8a+pTPUlcGEeAGQpWL9rKJhOFCV+J/QtdQUEU5vtBgVg6eJXn8JB8RZvpbJWZGvXkhz70MLWoA==} 535 | dependencies: 536 | stream-chain: 2.2.5 537 | dev: false 538 | 539 | /string-width/4.2.3: 540 | resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} 541 | engines: {node: '>=8'} 542 | dependencies: 543 | emoji-regex: 8.0.0 544 | is-fullwidth-code-point: 3.0.0 545 | strip-ansi: 6.0.1 546 | dev: false 547 | 548 | /strip-ansi/6.0.1: 549 | resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} 550 | engines: {node: '>=8'} 551 | dependencies: 552 | ansi-regex: 5.0.1 553 | 554 | /supports-color/7.2.0: 555 | resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} 556 | engines: {node: '>=8'} 557 | dependencies: 558 | has-flag: 4.0.0 559 | dev: true 560 | 561 | /tsx/3.12.3: 562 | resolution: {integrity: sha512-Wc5BFH1xccYTXaQob+lEcimkcb/Pq+0en2s+ruiX0VEIC80nV7/0s7XRahx8NnsoCnpCVUPz8wrqVSPi760LkA==} 563 | hasBin: true 564 | dependencies: 565 | '@esbuild-kit/cjs-loader': 2.4.2 566 | '@esbuild-kit/core-utils': 3.0.0 567 | '@esbuild-kit/esm-loader': 2.5.5 568 | optionalDependencies: 569 | fsevents: 2.3.2 570 | dev: true 571 | 572 | /which/2.0.2: 573 | resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} 574 | engines: {node: '>= 8'} 575 | hasBin: true 576 | dependencies: 577 | isexe: 2.0.0 578 | dev: true 579 | -------------------------------------------------------------------------------- /src/dump.ts: -------------------------------------------------------------------------------- 1 | import { MultiBar, Presets } from "cli-progress"; 2 | import { createWriteStream, existsSync, readFileSync } from "fs"; 3 | import { mkdir, readdir } from "fs/promises"; 4 | import { join } from "path"; 5 | import { PrismaClient } from "./prisma-clients/source"; 6 | import { incrementalFieldInModel } from "./prisma-clients/source/utils"; 7 | import { fileName } from "./utils/fileName"; 8 | import { progressBarFormat } from "./utils/parseProgressBarFormat"; 9 | 10 | type ModelName = keyof typeof incrementalFieldInModel; 11 | 12 | let isFirstExtract = true; 13 | const take = 100000; 14 | const snapshotsPath = join(__dirname, "snapshots"); 15 | const prisma = new PrismaClient({ 16 | datasources: { db: { url: process.env.SOURCE_DATABASE_URL } }, 17 | }); 18 | 19 | export type DumpProps = { 20 | includeTables?: ModelName[]; 21 | excludeTables?: ModelName[]; 22 | }; 23 | 24 | /** 25 | * Read source database and generate timestamped snapshots 26 | */ 27 | export const dump = async (props?: DumpProps) => { 28 | const progressBar = new MultiBar( 29 | { format: progressBarFormat }, 30 | Presets.legacy 31 | ); 32 | if (!existsSync(snapshotsPath)) await mkdir(snapshotsPath); 33 | 34 | isFirstExtract = true; 35 | 36 | const modelsToExtract = ( 37 | Object.keys(incrementalFieldInModel) as ModelName[] 38 | ).filter( 39 | (ModelName) => 40 | (props?.includeTables 41 | ? props.includeTables.includes(ModelName as ModelName) 42 | : true) && 43 | (props?.excludeTables 44 | ? !props.excludeTables.includes(ModelName as ModelName) 45 | : true) 46 | ); 47 | 48 | for (const modelToExtract of modelsToExtract) { 49 | await extractRecords(modelToExtract, progressBar); 50 | } 51 | progressBar.stop(); 52 | }; 53 | 54 | const extractRecords = async (modelName: ModelName, progressBar) => { 55 | const now = new Date(); 56 | const incrementalField = incrementalFieldInModel[modelName]; 57 | const filter = await parseFilter(modelName, now); 58 | const prismaRecord = prisma[modelName] as any; 59 | const totalRecords = (await prismaRecord.count({ 60 | where: incrementalField ? { [incrementalField]: filter } : undefined, 61 | })) as number; 62 | if (totalRecords === 0) return; 63 | if ( 64 | incrementalField === undefined && 65 | totalRecords === (await getLatestSnapshotSize(modelName)) 66 | ) 67 | return; 68 | if (isFirstExtract) { 69 | isFirstExtract = false; 70 | console.log("\n------------------ Extracting ------------------\n"); 71 | } 72 | const currentSnapshotPath = join(snapshotsPath, modelName); 73 | 74 | if (!existsSync(currentSnapshotPath)) await mkdir(currentSnapshotPath); 75 | 76 | const currentStreamPath = join( 77 | currentSnapshotPath, 78 | fileName.transformToValidFilename(now.toISOString()) 79 | ); 80 | 81 | const stream = createWriteStream(currentStreamPath); 82 | 83 | stream.write("["); 84 | let skip = 0; 85 | let batch = []; 86 | let totalDumped = 0; 87 | const progress = progressBar.create(totalRecords, 0); 88 | progress.update(0, { modelName }); 89 | do { 90 | if (totalDumped > 1) stream.write(","); 91 | batch = await prismaRecord.findMany({ 92 | skip, 93 | take, 94 | where: incrementalField ? { [incrementalField]: filter } : undefined, 95 | orderBy: incrementalField ? { [incrementalField]: "asc" } : undefined, 96 | }); 97 | stream.write(JSON.stringify(batch).slice(1, -1)); 98 | totalDumped += batch.length; 99 | progress.update(totalDumped); 100 | skip += take; 101 | } while (batch.length >= take); 102 | 103 | stream.write("]"); 104 | stream.end(); 105 | }; 106 | 107 | const parseFilter = async (recordName: ModelName, now: Date) => { 108 | const recordFolderPath = join(snapshotsPath, recordName); 109 | if (!existsSync(recordFolderPath)) return { lte: now }; 110 | const latestSnapshot = (await readdir(recordFolderPath)) 111 | .map((snapshot) => new Date(snapshot.split(".json")[0])) 112 | .filter((date) => !isNaN(date.getTime())) 113 | .sort((a, b) => new Date(a).getTime() - new Date(b).getTime()) 114 | .pop(); 115 | const latestSnapshotDate = latestSnapshot 116 | ? new Date(latestSnapshot) 117 | : undefined; 118 | return { lte: now, gt: latestSnapshotDate }; 119 | }; 120 | 121 | const getLatestSnapshotSize = async (recordName: ModelName) => { 122 | const recordFolderPath = join(snapshotsPath, recordName); 123 | if (!existsSync(recordFolderPath)) return 0; 124 | const latestSnapshot = (await readdir(recordFolderPath)) 125 | .map((snapshot) => new Date(snapshot.split(".json")[0])) 126 | .filter((date) => !isNaN(date.getTime())) 127 | .sort((a, b) => new Date(a).getTime() - new Date(b).getTime()) 128 | .pop(); 129 | 130 | if (!latestSnapshot) return 0; 131 | 132 | const size = ( 133 | JSON.parse( 134 | readFileSync( 135 | join(recordFolderPath, `${latestSnapshot.toISOString()}.json`) 136 | ).toString() 137 | ) as unknown[] 138 | ).length; 139 | 140 | return size; 141 | }; 142 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import prompts from "prompts"; 2 | import { dump } from "./dump"; 3 | import { inject } from "./inject"; 4 | import { sync } from "./sync"; 5 | import * as dotenv from "dotenv"; 6 | 7 | dotenv.config(); 8 | 9 | export const main = async () => { 10 | const { action } = await prompts({ 11 | type: "select", 12 | name: "action", 13 | message: "", 14 | choices: [ 15 | { 16 | title: "Sync", 17 | value: "sync", 18 | description: 19 | "Watch for changes in your source database and inject it in your target database", 20 | }, 21 | { 22 | title: "Dump", 23 | value: "dump", 24 | description: 25 | "Extract data from your source database and generate timestamped snapshots", 26 | }, 27 | { 28 | title: "Inject", 29 | value: "inject", 30 | description: 31 | "Inject the previously extracted snapshots chronologically", 32 | }, 33 | ], 34 | }); 35 | 36 | switch (action) { 37 | case "sync": 38 | sync(); 39 | break; 40 | case "dump": 41 | await dump(); 42 | break; 43 | case "inject": 44 | await inject(); 45 | break; 46 | } 47 | }; 48 | 49 | main().then(); 50 | -------------------------------------------------------------------------------- /src/inject.ts: -------------------------------------------------------------------------------- 1 | import { MultiBar, Presets } from "cli-progress"; 2 | import { 3 | createReadStream, 4 | existsSync, 5 | readdirSync, 6 | readFileSync, 7 | writeFileSync, 8 | } from "fs"; 9 | import { join } from "path"; 10 | import { chain } from "stream-chain"; 11 | import { parser } from "stream-json"; 12 | import { streamArray } from "stream-json/streamers/StreamArray"; 13 | import { Prisma, PrismaClient } from "./prisma-clients/target"; 14 | import { 15 | incrementalFieldInModel, 16 | nullableJsonFields, 17 | uniqueFields, 18 | } from "./prisma-clients/target/utils"; 19 | import { fileName } from "./utils/fileName"; 20 | import { progressBarFormat } from "./utils/parseProgressBarFormat"; 21 | 22 | type ModelName = keyof typeof incrementalFieldInModel; 23 | 24 | const prisma = new PrismaClient({ 25 | datasources: { db: { url: process.env.TARGET_DATABASE_URL } }, 26 | }); 27 | 28 | let isFirstInject = true; 29 | const injectionLogFileName = "latestSnapshotInjected.log"; 30 | const snapshotsPath = join(__dirname, "snapshots"); 31 | 32 | const parallelQueries = 100; 33 | const totalRowsPerQuery = 100; 34 | 35 | export type InjectProps = { 36 | includeTables?: ModelName[]; 37 | excludeTables?: ModelName[]; 38 | order?: ModelName[]; 39 | }; 40 | 41 | /** 42 | * Inject the previously imported snapshots chronologically 43 | */ 44 | export const inject = async (props?: InjectProps) => { 45 | const progressBar = new MultiBar( 46 | { format: progressBarFormat }, 47 | Presets.legacy 48 | ); 49 | isFirstInject = true; 50 | const modelsToInject = readdirSync(snapshotsPath) as ModelName[]; 51 | const modelNames = (props?.order ?? []) 52 | .concat( 53 | modelsToInject.filter((modelName) => 54 | props?.order ? !props.order.includes(modelName) : true 55 | ) 56 | ) 57 | .filter(filterModelName(props?.includeTables, props?.excludeTables)) 58 | .filter((modelName) => modelsToInject.includes(modelName)); 59 | for (const modelName of modelNames) { 60 | const logPath = join( 61 | snapshotsPath, 62 | modelName, 63 | "latestSnapshotInjected.log" 64 | ); 65 | const latestSnapshotInjected = existsSync(logPath) 66 | ? readFileSync( 67 | join(snapshotsPath, modelName, "latestSnapshotInjected.log") 68 | ).toString() 69 | : undefined; 70 | const snapshotsToInject = readdirSync(join(snapshotsPath, modelName)) 71 | .map((snapshot) => { 72 | let newName = fileName.transformToValidISOString(snapshot); 73 | return new Date(newName.split(".json")[0]); 74 | }) 75 | .filter((snapshot) => !isNaN(snapshot.getTime())) 76 | .filter((snapshot) => 77 | latestSnapshotInjected 78 | ? snapshot > new Date(latestSnapshotInjected) 79 | : true 80 | ) 81 | .sort((a, b) => new Date(a).getTime() - new Date(b).getTime()); 82 | 83 | if (snapshotsToInject.length > 0 && isFirstInject) { 84 | isFirstInject = false; 85 | console.log("\n------------------ Injecting ------------------\n"); 86 | } 87 | for (const snapshotToInject of snapshotsToInject) { 88 | await injectRecords( 89 | modelName as ModelName, 90 | snapshotToInject, 91 | progressBar, 92 | latestSnapshotInjected === undefined 93 | ); 94 | } 95 | } 96 | progressBar.stop(); 97 | }; 98 | 99 | const injectRecords = async ( 100 | modelName: ModelName, 101 | snapshotDate: Date, 102 | progressBar, 103 | isFirstInjection: boolean 104 | ) => 105 | new Promise(async (resolve, reject) => { 106 | const filePath = join( 107 | snapshotsPath, 108 | modelName, 109 | fileName.transformToValidFilename(snapshotDate.toISOString()) 110 | ); 111 | 112 | const listSize = await computeListSize(filePath); 113 | 114 | const progress = progressBar.create(listSize, 0); 115 | progress.update(0, { modelName }); 116 | 117 | let fillingChunk: any[] = []; 118 | let chunks: any[][] = []; 119 | let totalInjected = 0; 120 | 121 | const pipeline = chain([ 122 | createReadStream(filePath), 123 | parser(), 124 | streamArray(), 125 | async ({ key, value }: { key: number; value: object }) => { 126 | fillingChunk.push(value); 127 | if (fillingChunk.length === totalRowsPerQuery || key === listSize - 1) { 128 | chunks.push(fillingChunk); 129 | fillingChunk = []; 130 | } 131 | if (key === listSize - 1 || chunks.length === parallelQueries) { 132 | await Promise.all( 133 | chunks.map((chunk) => 134 | injectRecordsBatch( 135 | chunk, 136 | modelName, 137 | snapshotDate, 138 | isFirstInjection 139 | ) 140 | ) 141 | ); 142 | totalInjected += chunks.reduce((acc, cur) => acc + cur.length, 0); 143 | progress.update(totalInjected); 144 | chunks = []; 145 | fillingChunk = []; 146 | } 147 | if (key === listSize - 1) { 148 | writeFileSync( 149 | join(snapshotsPath, modelName, injectionLogFileName), 150 | snapshotDate.toISOString() 151 | ); 152 | resolve(); 153 | } 154 | return null; 155 | }, 156 | ]); 157 | 158 | pipeline.on("error", reject); 159 | }); 160 | 161 | const computeListSize = (filePath: string): Promise => 162 | new Promise((resolve, reject) => { 163 | let total = 0; 164 | 165 | const pipeline = createReadStream(filePath) 166 | .pipe(parser()) 167 | .pipe(streamArray()); 168 | 169 | pipeline.on("data", () => { 170 | total += 1; 171 | }); 172 | pipeline.on("error", reject); 173 | pipeline.on("end", () => resolve(total)); 174 | }); 175 | 176 | const injectRecordsBatch = async ( 177 | batch: any[], 178 | modelName: ModelName, 179 | snapshotDate: Date, 180 | isFirstInjection: boolean 181 | ) => { 182 | const prismaRecord = prisma[modelName] as any; 183 | const deleteManyWhereFilter = parseDeleteWhereFilter( 184 | batch, 185 | modelName, 186 | snapshotDate 187 | ); 188 | 189 | return prisma.$transaction([ 190 | ...(isFirstInjection 191 | ? [] 192 | : [ 193 | prismaRecord.deleteMany({ 194 | where: deleteManyWhereFilter, 195 | }), 196 | ]), 197 | prismaRecord.createMany({ 198 | data: batch.map(replaceNullWithDbNull(modelName)), 199 | skipDuplicates: true, 200 | }), 201 | ]); 202 | }; 203 | 204 | const parseDeleteWhereFilter = ( 205 | batch: any[], 206 | modelName: ModelName, 207 | snapshotDate: Date 208 | ) => { 209 | const where: Record = {}; 210 | (uniqueFields[modelName] as ModelName[]).forEach((field) => { 211 | where[field] = { 212 | in: batch.map((record) => record[field]), 213 | }; 214 | }); 215 | return { 216 | ...where, 217 | ...(incrementalFieldInModel[modelName] 218 | ? { [incrementalFieldInModel[modelName]]: { lte: snapshotDate } } 219 | : {}), 220 | }; 221 | }; 222 | 223 | const replaceNullWithDbNull = (modelName: ModelName) => (obj: unknown) => { 224 | if (!obj) return obj; 225 | if (typeof obj !== `object`) return obj; 226 | for (const key in obj) { 227 | if (obj[key] === null && nullableJsonFields[modelName].includes(key)) { 228 | obj[key] = Prisma.DbNull; 229 | } 230 | } 231 | 232 | return obj; 233 | }; 234 | 235 | const filterModelName = 236 | (includeTables?: ModelName[], excludeTables?: ModelName[]) => 237 | (modelName: ModelName) => 238 | (includeTables ? includeTables.includes(modelName as ModelName) : true) && 239 | (excludeTables ? !excludeTables.includes(modelName as ModelName) : true); 240 | -------------------------------------------------------------------------------- /src/schemas/generatePrismaUtilsTypes.ts: -------------------------------------------------------------------------------- 1 | import { 2 | generatorHandler, 3 | GeneratorManifest, 4 | GeneratorOptions, 5 | } from "@prisma/generator-helper"; 6 | import { writeFileSync } from "fs"; 7 | import { join } from "path"; 8 | 9 | generatorHandler({ 10 | onManifest() { 11 | return { 12 | prettyName: "Prisma Utils Types", 13 | } satisfies GeneratorManifest; 14 | }, 15 | onGenerate: async (options: GeneratorOptions) => { 16 | const models = options.dmmf.datamodel.models.map((model) => { 17 | const idFields = model.fields.filter((field) => field.isId); 18 | const uniqueFields = model.fields.filter((field) => field.isUnique); 19 | const groupedUniqueFields = model.uniqueFields?.shift() ?? []; 20 | return { 21 | name: camelize(model.name), 22 | incrementalField: 23 | model.fields.find((field) => field.isUpdatedAt)?.name ?? 24 | model.fields.find( 25 | (field) => 26 | field.isUpdatedAt || 27 | (field.type === "DateTime" && 28 | typeof field.default === "object" && 29 | "name" in field.default && 30 | field.default?.name === "now") 31 | )?.name, 32 | uniqueFields: 33 | idFields.length > 0 34 | ? idFields.map((field) => field.name) 35 | : uniqueFields.length > 0 36 | ? uniqueFields.map((field) => field.name) 37 | : groupedUniqueFields, 38 | nullableJsonFields: model.fields 39 | .filter( 40 | (field) => field.type === "Json" && field.isRequired === false 41 | ) 42 | .map((field) => field.name), 43 | }; 44 | }); 45 | 46 | writeFileSync( 47 | join(options.generator.output?.value, "utils.ts"), 48 | `export const incrementalFieldInModel = ${`{${models 49 | .map( 50 | (model) => 51 | `${model.name}: ${ 52 | model.incrementalField 53 | ? `"${model.incrementalField}"` 54 | : "undefined" 55 | }` 56 | ) 57 | .join(", ")}}`} as const; 58 | 59 | export const uniqueFields = ${`{${models 60 | .map( 61 | (model) => 62 | `${model.name}: ${ 63 | model.uniqueFields.length === 0 64 | ? "[]" 65 | : JSON.stringify(model.uniqueFields) 66 | }` 67 | ) 68 | .join(", ")}}`}; 69 | 70 | export const nullableJsonFields = ${`{${models 71 | .map( 72 | (model) => 73 | `${model.name}: ${ 74 | model.nullableJsonFields.length === 0 75 | ? "[]" 76 | : JSON.stringify(model.nullableJsonFields) 77 | }` 78 | ) 79 | .join(", ")}}`}; 80 | ` 81 | ); 82 | }, 83 | }); 84 | 85 | // Copy pasted from https://stackoverflow.com/questions/2970525/converting-any-string-into-camel-case 🤓 86 | const camelize = (str: string) => { 87 | return str.replace(/(?:^\w|[A-Z]|\b\w|\s+)/g, function (match, index) { 88 | if (+match === 0) return ""; // or if (/\s+/.test(match)) for white spaces 89 | return index === 0 ? match.toLowerCase() : match.toUpperCase(); 90 | }); 91 | }; 92 | -------------------------------------------------------------------------------- /src/schemas/source.prisma: -------------------------------------------------------------------------------- 1 | generator client { 2 | provider = "prisma-client-js" 3 | output = "../prisma-clients/source" 4 | } 5 | 6 | generator utils { 7 | provider = "pnpm tsx src/schemas/generatePrismaUtilsTypes.ts" 8 | output = "../prisma-clients/source" 9 | } 10 | 11 | datasource db { 12 | provider = "postgresql" 13 | url = env("SOURCE_DATABASE_URL") 14 | } 15 | -------------------------------------------------------------------------------- /src/schemas/target.prisma: -------------------------------------------------------------------------------- 1 | generator client { 2 | provider = "prisma-client-js" 3 | output = "../prisma-clients/target" 4 | } 5 | 6 | generator utils { 7 | provider = "pnpm tsx src/schemas/generatePrismaUtilsTypes.ts" 8 | output = "../prisma-clients/target" 9 | } 10 | 11 | datasource db { 12 | provider = "mysql" 13 | url = env("TARGET_DATABASE_URL") 14 | } 15 | -------------------------------------------------------------------------------- /src/sync.ts: -------------------------------------------------------------------------------- 1 | import { dump } from "./dump"; 2 | import { inject, InjectProps } from "./inject"; 3 | 4 | export const sync = (props?: InjectProps & { interval?: string }) => { 5 | const parsedInterval = Number( 6 | props?.interval ?? process.env.SYNC_INTERVAL_MS 7 | ); 8 | console.log( 9 | `Will scan source database every ${Math.round( 10 | parsedInterval / 1000 11 | )} seconds` 12 | ); 13 | let isDumping = false; 14 | setInterval(async () => { 15 | if (isDumping) return; 16 | isDumping = true; 17 | await dump(props); 18 | await inject(props); 19 | isDumping = false; 20 | }, parsedInterval); 21 | }; 22 | -------------------------------------------------------------------------------- /src/utils/fileName.ts: -------------------------------------------------------------------------------- 1 | const transformToValidFilename = (ISOString: string) => { 2 | return ISOString.replace(/:/g, "_") + ".json"; 3 | }; 4 | 5 | const transformToValidISOString = (name: string) => { 6 | return name.replace(/_/g, ":"); 7 | }; 8 | 9 | export const fileName = { 10 | transformToValidFilename, 11 | transformToValidISOString, 12 | }; 13 | -------------------------------------------------------------------------------- /src/utils/isEmpty.ts: -------------------------------------------------------------------------------- 1 | export const isEmpty = (value: string | undefined | null): value is undefined => 2 | value === undefined || value === null || value === ""; 3 | -------------------------------------------------------------------------------- /src/utils/parseProgressBarFormat.ts: -------------------------------------------------------------------------------- 1 | export const progressBarFormat = `[{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | {modelName}`; 2 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "esModuleInterop": true 4 | } 5 | } 6 | --------------------------------------------------------------------------------