├── .env.example
├── .github
└── images
│ └── readme-illustration.png
├── .gitignore
├── README.md
├── package.json
├── pnpm-lock.yaml
├── src
├── dump.ts
├── index.ts
├── inject.ts
├── schemas
│ ├── generatePrismaUtilsTypes.ts
│ ├── source.prisma
│ └── target.prisma
├── sync.ts
└── utils
│ ├── fileName.ts
│ ├── isEmpty.ts
│ └── parseProgressBarFormat.ts
└── tsconfig.json
/.env.example:
--------------------------------------------------------------------------------
1 | SOURCE_DATABASE_URL=
2 | TARGET_DATABASE_URL=
3 | SYNC_INTERVAL_MS=30000
4 |
--------------------------------------------------------------------------------
/.github/images/readme-illustration.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/baptisteArno/prisma-database-sync/060aa28f7aa25f5d13c29ef17956db2f3953d9d8/.github/images/readme-illustration.png
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .env
2 | node_modules
3 | prisma-clients
4 | snapshots
5 | migrations
6 | .DS_Store
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
Prisma Databases Sync
4 |
5 |
6 | I implemented this CLI because I had to migrate from a PostgreSQL database to a MySQL database for my startup [Typebot](https://typebot.io/). Prisma takes care of the data transformation and this allows you to migrate without downtime.
7 |
8 | > I just tested the scripts with my setup. You may have to tweak things for yours. Feel free to create an issue so that I can help 👌
9 |
10 | ## Features
11 |
12 | - Sync data from and to any database (PostgreSQL, MySQL, MongoDB etc...) as long as it is supported by Prisma
13 | - Incremental sync (based on `@updatedAt` or `@createdAt`). To avoid extracting all the data every time.
14 |
15 | You just have to provide 2 prisma schemas: source and target.
16 |
17 | The first data sync will take some time because it will download all the data from the beginning. Then, subsequent syncs will be tiny and will happen in an instant.
18 |
19 | ## Get started
20 |
21 | 1. Clone the repository
22 | 2. Copy `.env.example` to `.env` and fill it with your database URLs (It is recommended to disable the prisma pool timeout using the `?pool_timeout=0` query param).
23 | 3. Edit `src/schemas/source.prisma` and `src/schemas/target.prisma` with your schemas. Make sure it contains:
24 |
25 | ```
26 | generator utils {
27 | provider = "pnpm tsx src/generatePrismaUtilsTypes.ts"
28 | output = "prisma-clients/target"
29 | }
30 | ```
31 |
32 | 4. Install dependencies and generate prisma clients: `pnpm install`
33 | 5. Run the CLI: `pnpm start`, it will popup the menu
34 | ```
35 | ? › - Use arrow-keys. Return to submit.
36 | ❯ Sync - Watch for changes in your source database and inject it in your target database
37 | Dump
38 | Inject
39 | ```
40 |
41 | ## How to use
42 |
43 | This library has 3 functions `dump`, `inject`, `sync`:
44 |
45 | - `dump` reads your source database and generate timestamped snapshots
46 | - `inject` injects the previously imported snapshots chronologically
47 | - `sync` executes dump and restore with a set interval
48 |
49 | This is perfect if you are planning on migrating to another database in production. The first dump will be quite big as it will pull all the data from the beginning. Subsequent dumps will be tiny.
50 |
51 | Let's say you need to migrate to a new database and can't afford to have application downtime:
52 |
53 | 1. Launch a sync job to sync the target database with the source database.
54 | 2. Deploy your application version that consumes the target database
55 | 3. Once it's deployed, make sure the sync job doesn't detect new data from source database for at least 30 minutes.
56 | 4. 🎉 Congrats, you migrated to a new database with no downtime.
57 |
58 | ## Limitations
59 |
60 | It doesn't detect if a row has been deleted on the source database.
61 |
62 | Foreign keys can make it difficult to sync data. Two options are:
63 |
64 | - Defer constraints (recommended)
65 | - Disable foreign key triggers, which can silently break referential integrity (not recommended)
66 |
67 | If your target schema uses `prisma` relationMode (or if it's a MongoDB database), you need to set the relationMode to `foreignKeys`
68 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "prisma-database-sync",
3 | "version": "1.0.0",
4 | "description": "",
5 | "main": "index.js",
6 | "scripts": {
7 | "generate": "prisma generate --schema src/schemas/source.prisma && prisma generate --schema src/schemas/target.prisma",
8 | "reset:source": "prisma migrate reset --schema src/schemas/source.prisma",
9 | "reset:target": "prisma migrate reset --schema src/schemas/target.prisma",
10 | "start": "tsx src/index.ts",
11 | "postinstall": "pnpm generate"
12 | },
13 | "keywords": [],
14 | "author": "Baptiste Arnaud",
15 | "license": "ISC",
16 | "dependencies": {
17 | "@prisma/client": "^4.10.0",
18 | "cli-progress": "^3.11.2",
19 | "prompts": "2.4.2",
20 | "stream-chain": "^2.2.5",
21 | "stream-json": "^1.7.5"
22 | },
23 | "devDependencies": {
24 | "@prisma/generator-helper": "^4.10.0",
25 | "@types/cli-progress": "^3.11.0",
26 | "@types/node": "^18.13.0",
27 | "@types/prompts": "2.4.2",
28 | "@types/stream-json": "^1.7.3",
29 | "dotenv": "^16.0.3",
30 | "prisma": "^4.10.0",
31 | "tsx": "^3.12.3"
32 | },
33 | "packageManager": "pnpm@7.27.0"
34 | }
35 |
--------------------------------------------------------------------------------
/pnpm-lock.yaml:
--------------------------------------------------------------------------------
1 | lockfileVersion: 5.4
2 |
3 | specifiers:
4 | '@prisma/client': ^4.10.0
5 | '@prisma/generator-helper': ^4.10.0
6 | '@types/cli-progress': ^3.11.0
7 | '@types/node': ^18.13.0
8 | '@types/prompts': 2.4.2
9 | '@types/stream-json': ^1.7.3
10 | cli-progress: ^3.11.2
11 | dotenv: ^16.0.3
12 | prisma: ^4.10.0
13 | prompts: 2.4.2
14 | stream-chain: ^2.2.5
15 | stream-json: ^1.7.5
16 | tsx: ^3.12.3
17 |
18 | dependencies:
19 | '@prisma/client': 4.10.0_prisma@4.10.0
20 | cli-progress: 3.11.2
21 | prompts: 2.4.2
22 | stream-chain: 2.2.5
23 | stream-json: 1.7.5
24 |
25 | devDependencies:
26 | '@prisma/generator-helper': 4.10.0
27 | '@types/cli-progress': 3.11.0
28 | '@types/node': 18.13.0
29 | '@types/prompts': 2.4.2
30 | '@types/stream-json': 1.7.3
31 | dotenv: 16.0.3
32 | prisma: 4.10.0
33 | tsx: 3.12.3
34 |
35 | packages:
36 |
37 | /@esbuild-kit/cjs-loader/2.4.2:
38 | resolution: {integrity: sha512-BDXFbYOJzT/NBEtp71cvsrGPwGAMGRB/349rwKuoxNSiKjPraNNnlK6MIIabViCjqZugu6j+xeMDlEkWdHHJSg==}
39 | dependencies:
40 | '@esbuild-kit/core-utils': 3.0.0
41 | get-tsconfig: 4.4.0
42 | dev: true
43 |
44 | /@esbuild-kit/core-utils/3.0.0:
45 | resolution: {integrity: sha512-TXmwH9EFS3DC2sI2YJWJBgHGhlteK0Xyu1VabwetMULfm3oYhbrsWV5yaSr2NTWZIgDGVLHbRf0inxbjXqAcmQ==}
46 | dependencies:
47 | esbuild: 0.15.18
48 | source-map-support: 0.5.21
49 | dev: true
50 |
51 | /@esbuild-kit/esm-loader/2.5.5:
52 | resolution: {integrity: sha512-Qwfvj/qoPbClxCRNuac1Du01r9gvNOT+pMYtJDapfB1eoGN1YlJ1BixLyL9WVENRx5RXgNLdfYdx/CuswlGhMw==}
53 | dependencies:
54 | '@esbuild-kit/core-utils': 3.0.0
55 | get-tsconfig: 4.4.0
56 | dev: true
57 |
58 | /@esbuild/android-arm/0.15.18:
59 | resolution: {integrity: sha512-5GT+kcs2WVGjVs7+boataCkO5Fg0y4kCjzkB5bAip7H4jfnOS3dA6KPiww9W1OEKTKeAcUVhdZGvgI65OXmUnw==}
60 | engines: {node: '>=12'}
61 | cpu: [arm]
62 | os: [android]
63 | requiresBuild: true
64 | dev: true
65 | optional: true
66 |
67 | /@esbuild/linux-loong64/0.15.18:
68 | resolution: {integrity: sha512-L4jVKS82XVhw2nvzLg/19ClLWg0y27ulRwuP7lcyL6AbUWB5aPglXY3M21mauDQMDfRLs8cQmeT03r/+X3cZYQ==}
69 | engines: {node: '>=12'}
70 | cpu: [loong64]
71 | os: [linux]
72 | requiresBuild: true
73 | dev: true
74 | optional: true
75 |
76 | /@prisma/client/4.10.0_prisma@4.10.0:
77 | resolution: {integrity: sha512-sBmYb1S6SMKFIESaLMfKqWSalv3pH73cMCsFt9HslJvYjIIcKQCA6PDL2O4SZGWvc4JBef9cg5Gd7d9x3AtKjw==}
78 | engines: {node: '>=14.17'}
79 | requiresBuild: true
80 | peerDependencies:
81 | prisma: '*'
82 | peerDependenciesMeta:
83 | prisma:
84 | optional: true
85 | dependencies:
86 | '@prisma/engines-version': 4.10.0-84.ca7fcef713137fa11029d519a9780db130cca91d
87 | prisma: 4.10.0
88 | dev: false
89 |
90 | /@prisma/debug/4.10.0:
91 | resolution: {integrity: sha512-rxVOZKsEyjlQCwN/pkkJO7wEdARt1yRyukSjLa+BF2QTvy2+VgtBmrfys4WDQSnj3jVWeHMpi5GeAoJjKkSKyA==}
92 | dependencies:
93 | '@types/debug': 4.1.7
94 | debug: 4.3.4
95 | strip-ansi: 6.0.1
96 | transitivePeerDependencies:
97 | - supports-color
98 | dev: true
99 |
100 | /@prisma/engines-version/4.10.0-84.ca7fcef713137fa11029d519a9780db130cca91d:
101 | resolution: {integrity: sha512-UVpmVlvSaGfY4ue+hh8CTkIesbuXCFUfrr8zk//+u85WwkKfWMtt6nLB2tNSzR1YO8eAA8+HqNf8LM7mnXIq5w==}
102 | dev: false
103 |
104 | /@prisma/engines/4.10.0:
105 | resolution: {integrity: sha512-ZPPo7q+nQZdTlPFedS7mFXPE3oZ2kWtTh3GO4sku0XQ8ikLqEyinuTPJbQCw/8qel2xglIEQicsK6yI4Jgh20A==}
106 | requiresBuild: true
107 |
108 | /@prisma/generator-helper/4.10.0:
109 | resolution: {integrity: sha512-NkQOfZpHUjVjqJ7NN2FymHSLkGd/E0fz5c3RkyESKvQqBy2sFBxt+aFxGsUbUy3FfwvkckC04HdQOXpisAko0A==}
110 | dependencies:
111 | '@prisma/debug': 4.10.0
112 | '@types/cross-spawn': 6.0.2
113 | chalk: 4.1.2
114 | cross-spawn: 7.0.3
115 | transitivePeerDependencies:
116 | - supports-color
117 | dev: true
118 |
119 | /@types/cli-progress/3.11.0:
120 | resolution: {integrity: sha512-XhXhBv1R/q2ahF3BM7qT5HLzJNlIL0wbcGyZVjqOTqAybAnsLisd7gy1UCyIqpL+5Iv6XhlSyzjLCnI2sIdbCg==}
121 | dependencies:
122 | '@types/node': 18.13.0
123 | dev: true
124 |
125 | /@types/cross-spawn/6.0.2:
126 | resolution: {integrity: sha512-KuwNhp3eza+Rhu8IFI5HUXRP0LIhqH5cAjubUvGXXthh4YYBuP2ntwEX+Cz8GJoZUHlKo247wPWOfA9LYEq4cw==}
127 | dependencies:
128 | '@types/node': 18.13.0
129 | dev: true
130 |
131 | /@types/debug/4.1.7:
132 | resolution: {integrity: sha512-9AonUzyTjXXhEOa0DnqpzZi6VHlqKMswga9EXjpXnnqxwLtdvPPtlO8evrI5D9S6asFRCQ6v+wpiUKbw+vKqyg==}
133 | dependencies:
134 | '@types/ms': 0.7.31
135 | dev: true
136 |
137 | /@types/ms/0.7.31:
138 | resolution: {integrity: sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==}
139 | dev: true
140 |
141 | /@types/node/18.13.0:
142 | resolution: {integrity: sha512-gC3TazRzGoOnoKAhUx+Q0t8S9Tzs74z7m0ipwGpSqQrleP14hKxP4/JUeEQcD3W1/aIpnWl8pHowI7WokuZpXg==}
143 | dev: true
144 |
145 | /@types/prompts/2.4.2:
146 | resolution: {integrity: sha512-TwNx7qsjvRIUv/BCx583tqF5IINEVjCNqg9ofKHRlSoUHE62WBHrem4B1HGXcIrG511v29d1kJ9a/t2Esz7MIg==}
147 | dependencies:
148 | '@types/node': 18.13.0
149 | kleur: 3.0.3
150 | dev: true
151 |
152 | /@types/stream-chain/2.0.1:
153 | resolution: {integrity: sha512-D+Id9XpcBpampptkegH7WMsEk6fUdf9LlCIX7UhLydILsqDin4L0QT7ryJR0oycwC7OqohIzdfcMHVZ34ezNGg==}
154 | dependencies:
155 | '@types/node': 18.13.0
156 | dev: true
157 |
158 | /@types/stream-json/1.7.3:
159 | resolution: {integrity: sha512-Jqsyq5VPOTWorvEmzWhEWH5tJnHA+bB8vt/Zzb11vSDj8esfSHDMj2rbVjP0mfJQzl3YBJSXBBq08iiyaBK3KA==}
160 | dependencies:
161 | '@types/node': 18.13.0
162 | '@types/stream-chain': 2.0.1
163 | dev: true
164 |
165 | /ansi-regex/5.0.1:
166 | resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==}
167 | engines: {node: '>=8'}
168 |
169 | /ansi-styles/4.3.0:
170 | resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==}
171 | engines: {node: '>=8'}
172 | dependencies:
173 | color-convert: 2.0.1
174 | dev: true
175 |
176 | /buffer-from/1.1.2:
177 | resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==}
178 | dev: true
179 |
180 | /chalk/4.1.2:
181 | resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==}
182 | engines: {node: '>=10'}
183 | dependencies:
184 | ansi-styles: 4.3.0
185 | supports-color: 7.2.0
186 | dev: true
187 |
188 | /cli-progress/3.11.2:
189 | resolution: {integrity: sha512-lCPoS6ncgX4+rJu5bS3F/iCz17kZ9MPZ6dpuTtI0KXKABkhyXIdYB3Inby1OpaGti3YlI3EeEkM9AuWpelJrVA==}
190 | engines: {node: '>=4'}
191 | dependencies:
192 | string-width: 4.2.3
193 | dev: false
194 |
195 | /color-convert/2.0.1:
196 | resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==}
197 | engines: {node: '>=7.0.0'}
198 | dependencies:
199 | color-name: 1.1.4
200 | dev: true
201 |
202 | /color-name/1.1.4:
203 | resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
204 | dev: true
205 |
206 | /cross-spawn/7.0.3:
207 | resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==}
208 | engines: {node: '>= 8'}
209 | dependencies:
210 | path-key: 3.1.1
211 | shebang-command: 2.0.0
212 | which: 2.0.2
213 | dev: true
214 |
215 | /debug/4.3.4:
216 | resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==}
217 | engines: {node: '>=6.0'}
218 | peerDependencies:
219 | supports-color: '*'
220 | peerDependenciesMeta:
221 | supports-color:
222 | optional: true
223 | dependencies:
224 | ms: 2.1.2
225 | dev: true
226 |
227 | /dotenv/16.0.3:
228 | resolution: {integrity: sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==}
229 | engines: {node: '>=12'}
230 | dev: true
231 |
232 | /emoji-regex/8.0.0:
233 | resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==}
234 | dev: false
235 |
236 | /esbuild-android-64/0.15.18:
237 | resolution: {integrity: sha512-wnpt3OXRhcjfIDSZu9bnzT4/TNTDsOUvip0foZOUBG7QbSt//w3QV4FInVJxNhKc/ErhUxc5z4QjHtMi7/TbgA==}
238 | engines: {node: '>=12'}
239 | cpu: [x64]
240 | os: [android]
241 | requiresBuild: true
242 | dev: true
243 | optional: true
244 |
245 | /esbuild-android-arm64/0.15.18:
246 | resolution: {integrity: sha512-G4xu89B8FCzav9XU8EjsXacCKSG2FT7wW9J6hOc18soEHJdtWu03L3TQDGf0geNxfLTtxENKBzMSq9LlbjS8OQ==}
247 | engines: {node: '>=12'}
248 | cpu: [arm64]
249 | os: [android]
250 | requiresBuild: true
251 | dev: true
252 | optional: true
253 |
254 | /esbuild-darwin-64/0.15.18:
255 | resolution: {integrity: sha512-2WAvs95uPnVJPuYKP0Eqx+Dl/jaYseZEUUT1sjg97TJa4oBtbAKnPnl3b5M9l51/nbx7+QAEtuummJZW0sBEmg==}
256 | engines: {node: '>=12'}
257 | cpu: [x64]
258 | os: [darwin]
259 | requiresBuild: true
260 | dev: true
261 | optional: true
262 |
263 | /esbuild-darwin-arm64/0.15.18:
264 | resolution: {integrity: sha512-tKPSxcTJ5OmNb1btVikATJ8NftlyNlc8BVNtyT/UAr62JFOhwHlnoPrhYWz09akBLHI9nElFVfWSTSRsrZiDUA==}
265 | engines: {node: '>=12'}
266 | cpu: [arm64]
267 | os: [darwin]
268 | requiresBuild: true
269 | dev: true
270 | optional: true
271 |
272 | /esbuild-freebsd-64/0.15.18:
273 | resolution: {integrity: sha512-TT3uBUxkteAjR1QbsmvSsjpKjOX6UkCstr8nMr+q7zi3NuZ1oIpa8U41Y8I8dJH2fJgdC3Dj3CXO5biLQpfdZA==}
274 | engines: {node: '>=12'}
275 | cpu: [x64]
276 | os: [freebsd]
277 | requiresBuild: true
278 | dev: true
279 | optional: true
280 |
281 | /esbuild-freebsd-arm64/0.15.18:
282 | resolution: {integrity: sha512-R/oVr+X3Tkh+S0+tL41wRMbdWtpWB8hEAMsOXDumSSa6qJR89U0S/PpLXrGF7Wk/JykfpWNokERUpCeHDl47wA==}
283 | engines: {node: '>=12'}
284 | cpu: [arm64]
285 | os: [freebsd]
286 | requiresBuild: true
287 | dev: true
288 | optional: true
289 |
290 | /esbuild-linux-32/0.15.18:
291 | resolution: {integrity: sha512-lphF3HiCSYtaa9p1DtXndiQEeQDKPl9eN/XNoBf2amEghugNuqXNZA/ZovthNE2aa4EN43WroO0B85xVSjYkbg==}
292 | engines: {node: '>=12'}
293 | cpu: [ia32]
294 | os: [linux]
295 | requiresBuild: true
296 | dev: true
297 | optional: true
298 |
299 | /esbuild-linux-64/0.15.18:
300 | resolution: {integrity: sha512-hNSeP97IviD7oxLKFuii5sDPJ+QHeiFTFLoLm7NZQligur8poNOWGIgpQ7Qf8Balb69hptMZzyOBIPtY09GZYw==}
301 | engines: {node: '>=12'}
302 | cpu: [x64]
303 | os: [linux]
304 | requiresBuild: true
305 | dev: true
306 | optional: true
307 |
308 | /esbuild-linux-arm/0.15.18:
309 | resolution: {integrity: sha512-UH779gstRblS4aoS2qpMl3wjg7U0j+ygu3GjIeTonCcN79ZvpPee12Qun3vcdxX+37O5LFxz39XeW2I9bybMVA==}
310 | engines: {node: '>=12'}
311 | cpu: [arm]
312 | os: [linux]
313 | requiresBuild: true
314 | dev: true
315 | optional: true
316 |
317 | /esbuild-linux-arm64/0.15.18:
318 | resolution: {integrity: sha512-54qr8kg/6ilcxd+0V3h9rjT4qmjc0CccMVWrjOEM/pEcUzt8X62HfBSeZfT2ECpM7104mk4yfQXkosY8Quptug==}
319 | engines: {node: '>=12'}
320 | cpu: [arm64]
321 | os: [linux]
322 | requiresBuild: true
323 | dev: true
324 | optional: true
325 |
326 | /esbuild-linux-mips64le/0.15.18:
327 | resolution: {integrity: sha512-Mk6Ppwzzz3YbMl/ZZL2P0q1tnYqh/trYZ1VfNP47C31yT0K8t9s7Z077QrDA/guU60tGNp2GOwCQnp+DYv7bxQ==}
328 | engines: {node: '>=12'}
329 | cpu: [mips64el]
330 | os: [linux]
331 | requiresBuild: true
332 | dev: true
333 | optional: true
334 |
335 | /esbuild-linux-ppc64le/0.15.18:
336 | resolution: {integrity: sha512-b0XkN4pL9WUulPTa/VKHx2wLCgvIAbgwABGnKMY19WhKZPT+8BxhZdqz6EgkqCLld7X5qiCY2F/bfpUUlnFZ9w==}
337 | engines: {node: '>=12'}
338 | cpu: [ppc64]
339 | os: [linux]
340 | requiresBuild: true
341 | dev: true
342 | optional: true
343 |
344 | /esbuild-linux-riscv64/0.15.18:
345 | resolution: {integrity: sha512-ba2COaoF5wL6VLZWn04k+ACZjZ6NYniMSQStodFKH/Pu6RxzQqzsmjR1t9QC89VYJxBeyVPTaHuBMCejl3O/xg==}
346 | engines: {node: '>=12'}
347 | cpu: [riscv64]
348 | os: [linux]
349 | requiresBuild: true
350 | dev: true
351 | optional: true
352 |
353 | /esbuild-linux-s390x/0.15.18:
354 | resolution: {integrity: sha512-VbpGuXEl5FCs1wDVp93O8UIzl3ZrglgnSQ+Hu79g7hZu6te6/YHgVJxCM2SqfIila0J3k0csfnf8VD2W7u2kzQ==}
355 | engines: {node: '>=12'}
356 | cpu: [s390x]
357 | os: [linux]
358 | requiresBuild: true
359 | dev: true
360 | optional: true
361 |
362 | /esbuild-netbsd-64/0.15.18:
363 | resolution: {integrity: sha512-98ukeCdvdX7wr1vUYQzKo4kQ0N2p27H7I11maINv73fVEXt2kyh4K4m9f35U1K43Xc2QGXlzAw0K9yoU7JUjOg==}
364 | engines: {node: '>=12'}
365 | cpu: [x64]
366 | os: [netbsd]
367 | requiresBuild: true
368 | dev: true
369 | optional: true
370 |
371 | /esbuild-openbsd-64/0.15.18:
372 | resolution: {integrity: sha512-yK5NCcH31Uae076AyQAXeJzt/vxIo9+omZRKj1pauhk3ITuADzuOx5N2fdHrAKPxN+zH3w96uFKlY7yIn490xQ==}
373 | engines: {node: '>=12'}
374 | cpu: [x64]
375 | os: [openbsd]
376 | requiresBuild: true
377 | dev: true
378 | optional: true
379 |
380 | /esbuild-sunos-64/0.15.18:
381 | resolution: {integrity: sha512-On22LLFlBeLNj/YF3FT+cXcyKPEI263nflYlAhz5crxtp3yRG1Ugfr7ITyxmCmjm4vbN/dGrb/B7w7U8yJR9yw==}
382 | engines: {node: '>=12'}
383 | cpu: [x64]
384 | os: [sunos]
385 | requiresBuild: true
386 | dev: true
387 | optional: true
388 |
389 | /esbuild-windows-32/0.15.18:
390 | resolution: {integrity: sha512-o+eyLu2MjVny/nt+E0uPnBxYuJHBvho8vWsC2lV61A7wwTWC3jkN2w36jtA+yv1UgYkHRihPuQsL23hsCYGcOQ==}
391 | engines: {node: '>=12'}
392 | cpu: [ia32]
393 | os: [win32]
394 | requiresBuild: true
395 | dev: true
396 | optional: true
397 |
398 | /esbuild-windows-64/0.15.18:
399 | resolution: {integrity: sha512-qinug1iTTaIIrCorAUjR0fcBk24fjzEedFYhhispP8Oc7SFvs+XeW3YpAKiKp8dRpizl4YYAhxMjlftAMJiaUw==}
400 | engines: {node: '>=12'}
401 | cpu: [x64]
402 | os: [win32]
403 | requiresBuild: true
404 | dev: true
405 | optional: true
406 |
407 | /esbuild-windows-arm64/0.15.18:
408 | resolution: {integrity: sha512-q9bsYzegpZcLziq0zgUi5KqGVtfhjxGbnksaBFYmWLxeV/S1fK4OLdq2DFYnXcLMjlZw2L0jLsk1eGoB522WXQ==}
409 | engines: {node: '>=12'}
410 | cpu: [arm64]
411 | os: [win32]
412 | requiresBuild: true
413 | dev: true
414 | optional: true
415 |
416 | /esbuild/0.15.18:
417 | resolution: {integrity: sha512-x/R72SmW3sSFRm5zrrIjAhCeQSAWoni3CmHEqfQrZIQTM3lVCdehdwuIqaOtfC2slvpdlLa62GYoN8SxT23m6Q==}
418 | engines: {node: '>=12'}
419 | hasBin: true
420 | requiresBuild: true
421 | optionalDependencies:
422 | '@esbuild/android-arm': 0.15.18
423 | '@esbuild/linux-loong64': 0.15.18
424 | esbuild-android-64: 0.15.18
425 | esbuild-android-arm64: 0.15.18
426 | esbuild-darwin-64: 0.15.18
427 | esbuild-darwin-arm64: 0.15.18
428 | esbuild-freebsd-64: 0.15.18
429 | esbuild-freebsd-arm64: 0.15.18
430 | esbuild-linux-32: 0.15.18
431 | esbuild-linux-64: 0.15.18
432 | esbuild-linux-arm: 0.15.18
433 | esbuild-linux-arm64: 0.15.18
434 | esbuild-linux-mips64le: 0.15.18
435 | esbuild-linux-ppc64le: 0.15.18
436 | esbuild-linux-riscv64: 0.15.18
437 | esbuild-linux-s390x: 0.15.18
438 | esbuild-netbsd-64: 0.15.18
439 | esbuild-openbsd-64: 0.15.18
440 | esbuild-sunos-64: 0.15.18
441 | esbuild-windows-32: 0.15.18
442 | esbuild-windows-64: 0.15.18
443 | esbuild-windows-arm64: 0.15.18
444 | dev: true
445 |
446 | /fsevents/2.3.2:
447 | resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==}
448 | engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
449 | os: [darwin]
450 | requiresBuild: true
451 | dev: true
452 | optional: true
453 |
454 | /get-tsconfig/4.4.0:
455 | resolution: {integrity: sha512-0Gdjo/9+FzsYhXCEFueo2aY1z1tpXrxWZzP7k8ul9qt1U5o8rYJwTJYmaeHdrVosYIVYkOy2iwCJ9FdpocJhPQ==}
456 | dev: true
457 |
458 | /has-flag/4.0.0:
459 | resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==}
460 | engines: {node: '>=8'}
461 | dev: true
462 |
463 | /is-fullwidth-code-point/3.0.0:
464 | resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==}
465 | engines: {node: '>=8'}
466 | dev: false
467 |
468 | /isexe/2.0.0:
469 | resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
470 | dev: true
471 |
472 | /kleur/3.0.3:
473 | resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==}
474 | engines: {node: '>=6'}
475 |
476 | /ms/2.1.2:
477 | resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==}
478 | dev: true
479 |
480 | /path-key/3.1.1:
481 | resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==}
482 | engines: {node: '>=8'}
483 | dev: true
484 |
485 | /prisma/4.10.0:
486 | resolution: {integrity: sha512-xUHcF3Glc8QGgW8x0rfPITvyyTo04fskUdG7pI4kQbvDX/rhzDP4046x/FvazYqYHXMLR5/KTIi2p2Gth5vKOQ==}
487 | engines: {node: '>=14.17'}
488 | hasBin: true
489 | requiresBuild: true
490 | dependencies:
491 | '@prisma/engines': 4.10.0
492 |
493 | /prompts/2.4.2:
494 | resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==}
495 | engines: {node: '>= 6'}
496 | dependencies:
497 | kleur: 3.0.3
498 | sisteransi: 1.0.5
499 | dev: false
500 |
501 | /shebang-command/2.0.0:
502 | resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==}
503 | engines: {node: '>=8'}
504 | dependencies:
505 | shebang-regex: 3.0.0
506 | dev: true
507 |
508 | /shebang-regex/3.0.0:
509 | resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==}
510 | engines: {node: '>=8'}
511 | dev: true
512 |
513 | /sisteransi/1.0.5:
514 | resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==}
515 | dev: false
516 |
517 | /source-map-support/0.5.21:
518 | resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==}
519 | dependencies:
520 | buffer-from: 1.1.2
521 | source-map: 0.6.1
522 | dev: true
523 |
524 | /source-map/0.6.1:
525 | resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==}
526 | engines: {node: '>=0.10.0'}
527 | dev: true
528 |
529 | /stream-chain/2.2.5:
530 | resolution: {integrity: sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==}
531 | dev: false
532 |
533 | /stream-json/1.7.5:
534 | resolution: {integrity: sha512-NSkoVduGakxZ8a+pTPUlcGEeAGQpWL9rKJhOFCV+J/QtdQUEU5vtBgVg6eJXn8JB8RZvpbJWZGvXkhz70MLWoA==}
535 | dependencies:
536 | stream-chain: 2.2.5
537 | dev: false
538 |
539 | /string-width/4.2.3:
540 | resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==}
541 | engines: {node: '>=8'}
542 | dependencies:
543 | emoji-regex: 8.0.0
544 | is-fullwidth-code-point: 3.0.0
545 | strip-ansi: 6.0.1
546 | dev: false
547 |
548 | /strip-ansi/6.0.1:
549 | resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==}
550 | engines: {node: '>=8'}
551 | dependencies:
552 | ansi-regex: 5.0.1
553 |
554 | /supports-color/7.2.0:
555 | resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==}
556 | engines: {node: '>=8'}
557 | dependencies:
558 | has-flag: 4.0.0
559 | dev: true
560 |
561 | /tsx/3.12.3:
562 | resolution: {integrity: sha512-Wc5BFH1xccYTXaQob+lEcimkcb/Pq+0en2s+ruiX0VEIC80nV7/0s7XRahx8NnsoCnpCVUPz8wrqVSPi760LkA==}
563 | hasBin: true
564 | dependencies:
565 | '@esbuild-kit/cjs-loader': 2.4.2
566 | '@esbuild-kit/core-utils': 3.0.0
567 | '@esbuild-kit/esm-loader': 2.5.5
568 | optionalDependencies:
569 | fsevents: 2.3.2
570 | dev: true
571 |
572 | /which/2.0.2:
573 | resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==}
574 | engines: {node: '>= 8'}
575 | hasBin: true
576 | dependencies:
577 | isexe: 2.0.0
578 | dev: true
579 |
--------------------------------------------------------------------------------
/src/dump.ts:
--------------------------------------------------------------------------------
1 | import { MultiBar, Presets } from "cli-progress";
2 | import { createWriteStream, existsSync, readFileSync } from "fs";
3 | import { mkdir, readdir } from "fs/promises";
4 | import { join } from "path";
5 | import { PrismaClient } from "./prisma-clients/source";
6 | import { incrementalFieldInModel } from "./prisma-clients/source/utils";
7 | import { fileName } from "./utils/fileName";
8 | import { progressBarFormat } from "./utils/parseProgressBarFormat";
9 |
10 | type ModelName = keyof typeof incrementalFieldInModel;
11 |
12 | let isFirstExtract = true;
13 | const take = 100000;
14 | const snapshotsPath = join(__dirname, "snapshots");
15 | const prisma = new PrismaClient({
16 | datasources: { db: { url: process.env.SOURCE_DATABASE_URL } },
17 | });
18 |
19 | export type DumpProps = {
20 | includeTables?: ModelName[];
21 | excludeTables?: ModelName[];
22 | };
23 |
24 | /**
25 | * Read source database and generate timestamped snapshots
26 | */
27 | export const dump = async (props?: DumpProps) => {
28 | const progressBar = new MultiBar(
29 | { format: progressBarFormat },
30 | Presets.legacy
31 | );
32 | if (!existsSync(snapshotsPath)) await mkdir(snapshotsPath);
33 |
34 | isFirstExtract = true;
35 |
36 | const modelsToExtract = (
37 | Object.keys(incrementalFieldInModel) as ModelName[]
38 | ).filter(
39 | (ModelName) =>
40 | (props?.includeTables
41 | ? props.includeTables.includes(ModelName as ModelName)
42 | : true) &&
43 | (props?.excludeTables
44 | ? !props.excludeTables.includes(ModelName as ModelName)
45 | : true)
46 | );
47 |
48 | for (const modelToExtract of modelsToExtract) {
49 | await extractRecords(modelToExtract, progressBar);
50 | }
51 | progressBar.stop();
52 | };
53 |
54 | const extractRecords = async (modelName: ModelName, progressBar) => {
55 | const now = new Date();
56 | const incrementalField = incrementalFieldInModel[modelName];
57 | const filter = await parseFilter(modelName, now);
58 | const prismaRecord = prisma[modelName] as any;
59 | const totalRecords = (await prismaRecord.count({
60 | where: incrementalField ? { [incrementalField]: filter } : undefined,
61 | })) as number;
62 | if (totalRecords === 0) return;
63 | if (
64 | incrementalField === undefined &&
65 | totalRecords === (await getLatestSnapshotSize(modelName))
66 | )
67 | return;
68 | if (isFirstExtract) {
69 | isFirstExtract = false;
70 | console.log("\n------------------ Extracting ------------------\n");
71 | }
72 | const currentSnapshotPath = join(snapshotsPath, modelName);
73 |
74 | if (!existsSync(currentSnapshotPath)) await mkdir(currentSnapshotPath);
75 |
76 | const currentStreamPath = join(
77 | currentSnapshotPath,
78 | fileName.transformToValidFilename(now.toISOString())
79 | );
80 |
81 | const stream = createWriteStream(currentStreamPath);
82 |
83 | stream.write("[");
84 | let skip = 0;
85 | let batch = [];
86 | let totalDumped = 0;
87 | const progress = progressBar.create(totalRecords, 0);
88 | progress.update(0, { modelName });
89 | do {
90 | if (totalDumped > 1) stream.write(",");
91 | batch = await prismaRecord.findMany({
92 | skip,
93 | take,
94 | where: incrementalField ? { [incrementalField]: filter } : undefined,
95 | orderBy: incrementalField ? { [incrementalField]: "asc" } : undefined,
96 | });
97 | stream.write(JSON.stringify(batch).slice(1, -1));
98 | totalDumped += batch.length;
99 | progress.update(totalDumped);
100 | skip += take;
101 | } while (batch.length >= take);
102 |
103 | stream.write("]");
104 | stream.end();
105 | };
106 |
107 | const parseFilter = async (recordName: ModelName, now: Date) => {
108 | const recordFolderPath = join(snapshotsPath, recordName);
109 | if (!existsSync(recordFolderPath)) return { lte: now };
110 | const latestSnapshot = (await readdir(recordFolderPath))
111 | .map((snapshot) => new Date(snapshot.split(".json")[0]))
112 | .filter((date) => !isNaN(date.getTime()))
113 | .sort((a, b) => new Date(a).getTime() - new Date(b).getTime())
114 | .pop();
115 | const latestSnapshotDate = latestSnapshot
116 | ? new Date(latestSnapshot)
117 | : undefined;
118 | return { lte: now, gt: latestSnapshotDate };
119 | };
120 |
121 | const getLatestSnapshotSize = async (recordName: ModelName) => {
122 | const recordFolderPath = join(snapshotsPath, recordName);
123 | if (!existsSync(recordFolderPath)) return 0;
124 | const latestSnapshot = (await readdir(recordFolderPath))
125 | .map((snapshot) => new Date(snapshot.split(".json")[0]))
126 | .filter((date) => !isNaN(date.getTime()))
127 | .sort((a, b) => new Date(a).getTime() - new Date(b).getTime())
128 | .pop();
129 |
130 | if (!latestSnapshot) return 0;
131 |
132 | const size = (
133 | JSON.parse(
134 | readFileSync(
135 | join(recordFolderPath, `${latestSnapshot.toISOString()}.json`)
136 | ).toString()
137 | ) as unknown[]
138 | ).length;
139 |
140 | return size;
141 | };
142 |
--------------------------------------------------------------------------------
/src/index.ts:
--------------------------------------------------------------------------------
1 | import prompts from "prompts";
2 | import { dump } from "./dump";
3 | import { inject } from "./inject";
4 | import { sync } from "./sync";
5 | import * as dotenv from "dotenv";
6 |
7 | dotenv.config();
8 |
9 | export const main = async () => {
10 | const { action } = await prompts({
11 | type: "select",
12 | name: "action",
13 | message: "",
14 | choices: [
15 | {
16 | title: "Sync",
17 | value: "sync",
18 | description:
19 | "Watch for changes in your source database and inject it in your target database",
20 | },
21 | {
22 | title: "Dump",
23 | value: "dump",
24 | description:
25 | "Extract data from your source database and generate timestamped snapshots",
26 | },
27 | {
28 | title: "Inject",
29 | value: "inject",
30 | description:
31 | "Inject the previously extracted snapshots chronologically",
32 | },
33 | ],
34 | });
35 |
36 | switch (action) {
37 | case "sync":
38 | sync();
39 | break;
40 | case "dump":
41 | await dump();
42 | break;
43 | case "inject":
44 | await inject();
45 | break;
46 | }
47 | };
48 |
49 | main().then();
50 |
--------------------------------------------------------------------------------
/src/inject.ts:
--------------------------------------------------------------------------------
1 | import { MultiBar, Presets } from "cli-progress";
2 | import {
3 | createReadStream,
4 | existsSync,
5 | readdirSync,
6 | readFileSync,
7 | writeFileSync,
8 | } from "fs";
9 | import { join } from "path";
10 | import { chain } from "stream-chain";
11 | import { parser } from "stream-json";
12 | import { streamArray } from "stream-json/streamers/StreamArray";
13 | import { Prisma, PrismaClient } from "./prisma-clients/target";
14 | import {
15 | incrementalFieldInModel,
16 | nullableJsonFields,
17 | uniqueFields,
18 | } from "./prisma-clients/target/utils";
19 | import { fileName } from "./utils/fileName";
20 | import { progressBarFormat } from "./utils/parseProgressBarFormat";
21 |
22 | type ModelName = keyof typeof incrementalFieldInModel;
23 |
24 | const prisma = new PrismaClient({
25 | datasources: { db: { url: process.env.TARGET_DATABASE_URL } },
26 | });
27 |
28 | let isFirstInject = true;
29 | const injectionLogFileName = "latestSnapshotInjected.log";
30 | const snapshotsPath = join(__dirname, "snapshots");
31 |
32 | const parallelQueries = 100;
33 | const totalRowsPerQuery = 100;
34 |
35 | export type InjectProps = {
36 | includeTables?: ModelName[];
37 | excludeTables?: ModelName[];
38 | order?: ModelName[];
39 | };
40 |
41 | /**
42 | * Inject the previously imported snapshots chronologically
43 | */
44 | export const inject = async (props?: InjectProps) => {
45 | const progressBar = new MultiBar(
46 | { format: progressBarFormat },
47 | Presets.legacy
48 | );
49 | isFirstInject = true;
50 | const modelsToInject = readdirSync(snapshotsPath) as ModelName[];
51 | const modelNames = (props?.order ?? [])
52 | .concat(
53 | modelsToInject.filter((modelName) =>
54 | props?.order ? !props.order.includes(modelName) : true
55 | )
56 | )
57 | .filter(filterModelName(props?.includeTables, props?.excludeTables))
58 | .filter((modelName) => modelsToInject.includes(modelName));
59 | for (const modelName of modelNames) {
60 | const logPath = join(
61 | snapshotsPath,
62 | modelName,
63 | "latestSnapshotInjected.log"
64 | );
65 | const latestSnapshotInjected = existsSync(logPath)
66 | ? readFileSync(
67 | join(snapshotsPath, modelName, "latestSnapshotInjected.log")
68 | ).toString()
69 | : undefined;
70 | const snapshotsToInject = readdirSync(join(snapshotsPath, modelName))
71 | .map((snapshot) => {
72 | let newName = fileName.transformToValidISOString(snapshot);
73 | return new Date(newName.split(".json")[0]);
74 | })
75 | .filter((snapshot) => !isNaN(snapshot.getTime()))
76 | .filter((snapshot) =>
77 | latestSnapshotInjected
78 | ? snapshot > new Date(latestSnapshotInjected)
79 | : true
80 | )
81 | .sort((a, b) => new Date(a).getTime() - new Date(b).getTime());
82 |
83 | if (snapshotsToInject.length > 0 && isFirstInject) {
84 | isFirstInject = false;
85 | console.log("\n------------------ Injecting ------------------\n");
86 | }
87 | for (const snapshotToInject of snapshotsToInject) {
88 | await injectRecords(
89 | modelName as ModelName,
90 | snapshotToInject,
91 | progressBar,
92 | latestSnapshotInjected === undefined
93 | );
94 | }
95 | }
96 | progressBar.stop();
97 | };
98 |
99 | const injectRecords = async (
100 | modelName: ModelName,
101 | snapshotDate: Date,
102 | progressBar,
103 | isFirstInjection: boolean
104 | ) =>
105 | new Promise(async (resolve, reject) => {
106 | const filePath = join(
107 | snapshotsPath,
108 | modelName,
109 | fileName.transformToValidFilename(snapshotDate.toISOString())
110 | );
111 |
112 | const listSize = await computeListSize(filePath);
113 |
114 | const progress = progressBar.create(listSize, 0);
115 | progress.update(0, { modelName });
116 |
117 | let fillingChunk: any[] = [];
118 | let chunks: any[][] = [];
119 | let totalInjected = 0;
120 |
121 | const pipeline = chain([
122 | createReadStream(filePath),
123 | parser(),
124 | streamArray(),
125 | async ({ key, value }: { key: number; value: object }) => {
126 | fillingChunk.push(value);
127 | if (fillingChunk.length === totalRowsPerQuery || key === listSize - 1) {
128 | chunks.push(fillingChunk);
129 | fillingChunk = [];
130 | }
131 | if (key === listSize - 1 || chunks.length === parallelQueries) {
132 | await Promise.all(
133 | chunks.map((chunk) =>
134 | injectRecordsBatch(
135 | chunk,
136 | modelName,
137 | snapshotDate,
138 | isFirstInjection
139 | )
140 | )
141 | );
142 | totalInjected += chunks.reduce((acc, cur) => acc + cur.length, 0);
143 | progress.update(totalInjected);
144 | chunks = [];
145 | fillingChunk = [];
146 | }
147 | if (key === listSize - 1) {
148 | writeFileSync(
149 | join(snapshotsPath, modelName, injectionLogFileName),
150 | snapshotDate.toISOString()
151 | );
152 | resolve();
153 | }
154 | return null;
155 | },
156 | ]);
157 |
158 | pipeline.on("error", reject);
159 | });
160 |
161 | const computeListSize = (filePath: string): Promise =>
162 | new Promise((resolve, reject) => {
163 | let total = 0;
164 |
165 | const pipeline = createReadStream(filePath)
166 | .pipe(parser())
167 | .pipe(streamArray());
168 |
169 | pipeline.on("data", () => {
170 | total += 1;
171 | });
172 | pipeline.on("error", reject);
173 | pipeline.on("end", () => resolve(total));
174 | });
175 |
176 | const injectRecordsBatch = async (
177 | batch: any[],
178 | modelName: ModelName,
179 | snapshotDate: Date,
180 | isFirstInjection: boolean
181 | ) => {
182 | const prismaRecord = prisma[modelName] as any;
183 | const deleteManyWhereFilter = parseDeleteWhereFilter(
184 | batch,
185 | modelName,
186 | snapshotDate
187 | );
188 |
189 | return prisma.$transaction([
190 | ...(isFirstInjection
191 | ? []
192 | : [
193 | prismaRecord.deleteMany({
194 | where: deleteManyWhereFilter,
195 | }),
196 | ]),
197 | prismaRecord.createMany({
198 | data: batch.map(replaceNullWithDbNull(modelName)),
199 | skipDuplicates: true,
200 | }),
201 | ]);
202 | };
203 |
204 | const parseDeleteWhereFilter = (
205 | batch: any[],
206 | modelName: ModelName,
207 | snapshotDate: Date
208 | ) => {
209 | const where: Record = {};
210 | (uniqueFields[modelName] as ModelName[]).forEach((field) => {
211 | where[field] = {
212 | in: batch.map((record) => record[field]),
213 | };
214 | });
215 | return {
216 | ...where,
217 | ...(incrementalFieldInModel[modelName]
218 | ? { [incrementalFieldInModel[modelName]]: { lte: snapshotDate } }
219 | : {}),
220 | };
221 | };
222 |
223 | const replaceNullWithDbNull = (modelName: ModelName) => (obj: unknown) => {
224 | if (!obj) return obj;
225 | if (typeof obj !== `object`) return obj;
226 | for (const key in obj) {
227 | if (obj[key] === null && nullableJsonFields[modelName].includes(key)) {
228 | obj[key] = Prisma.DbNull;
229 | }
230 | }
231 |
232 | return obj;
233 | };
234 |
235 | const filterModelName =
236 | (includeTables?: ModelName[], excludeTables?: ModelName[]) =>
237 | (modelName: ModelName) =>
238 | (includeTables ? includeTables.includes(modelName as ModelName) : true) &&
239 | (excludeTables ? !excludeTables.includes(modelName as ModelName) : true);
240 |
--------------------------------------------------------------------------------
/src/schemas/generatePrismaUtilsTypes.ts:
--------------------------------------------------------------------------------
1 | import {
2 | generatorHandler,
3 | GeneratorManifest,
4 | GeneratorOptions,
5 | } from "@prisma/generator-helper";
6 | import { writeFileSync } from "fs";
7 | import { join } from "path";
8 |
9 | generatorHandler({
10 | onManifest() {
11 | return {
12 | prettyName: "Prisma Utils Types",
13 | } satisfies GeneratorManifest;
14 | },
15 | onGenerate: async (options: GeneratorOptions) => {
16 | const models = options.dmmf.datamodel.models.map((model) => {
17 | const idFields = model.fields.filter((field) => field.isId);
18 | const uniqueFields = model.fields.filter((field) => field.isUnique);
19 | const groupedUniqueFields = model.uniqueFields?.shift();
20 | return {
21 | name: camelize(model.name),
22 | incrementalField:
23 | model.fields.find((field) => field.isUpdatedAt)?.name ??
24 | model.fields.find(
25 | (field) =>
26 | field.isUpdatedAt ||
27 | (field.type === "DateTime" &&
28 | typeof field.default === "object" &&
29 | "name" in field.default &&
30 | field.default?.name === "now")
31 | )?.name,
32 | uniqueFields:
33 | idFields.length > 0
34 | ? idFields.map((field) => field.name)
35 | : uniqueFields.length > 0
36 | ? uniqueFields.map((field) => field.name)
37 | : groupedUniqueFields,
38 | nullableJsonFields: model.fields
39 | .filter(
40 | (field) => field.type === "Json" && field.isRequired === false
41 | )
42 | .map((field) => field.name),
43 | };
44 | });
45 |
46 | writeFileSync(
47 | join(options.generator.output?.value, "utils.ts"),
48 | `export const incrementalFieldInModel = ${`{${models
49 | .map(
50 | (model) =>
51 | `${model.name}: ${
52 | model.incrementalField
53 | ? `"${model.incrementalField}"`
54 | : "undefined"
55 | }`
56 | )
57 | .join(", ")}}`} as const;
58 |
59 | export const uniqueFields = ${`{${models
60 | .map(
61 | (model) =>
62 | `${model.name}: ${
63 | model.uniqueFields.length === 0
64 | ? "[]"
65 | : JSON.stringify(model.uniqueFields)
66 | }`
67 | )
68 | .join(", ")}}`};
69 |
70 | export const nullableJsonFields = ${`{${models
71 | .map(
72 | (model) =>
73 | `${model.name}: ${
74 | model.nullableJsonFields.length === 0
75 | ? "[]"
76 | : JSON.stringify(model.nullableJsonFields)
77 | }`
78 | )
79 | .join(", ")}}`};
80 | `
81 | );
82 | },
83 | });
84 |
85 | // Copy pasted from https://stackoverflow.com/questions/2970525/converting-any-string-into-camel-case 🤓
86 | const camelize = (str: string) => {
87 | return str.replace(/(?:^\w|[A-Z]|\b\w|\s+)/g, function (match, index) {
88 | if (+match === 0) return ""; // or if (/\s+/.test(match)) for white spaces
89 | return index === 0 ? match.toLowerCase() : match.toUpperCase();
90 | });
91 | };
92 |
--------------------------------------------------------------------------------
/src/schemas/source.prisma:
--------------------------------------------------------------------------------
1 | generator client {
2 | provider = "prisma-client-js"
3 | output = "../prisma-clients/source"
4 | }
5 |
6 | generator utils {
7 | provider = "pnpm tsx src/schemas/generatePrismaUtilsTypes.ts"
8 | output = "../prisma-clients/source"
9 | }
10 |
11 | datasource db {
12 | provider = "postgresql"
13 | url = env("SOURCE_DATABASE_URL")
14 | }
15 |
--------------------------------------------------------------------------------
/src/schemas/target.prisma:
--------------------------------------------------------------------------------
1 | generator client {
2 | provider = "prisma-client-js"
3 | output = "../prisma-clients/target"
4 | }
5 |
6 | generator utils {
7 | provider = "pnpm tsx src/schemas/generatePrismaUtilsTypes.ts"
8 | output = "../prisma-clients/target"
9 | }
10 |
11 | datasource db {
12 | provider = "mysql"
13 | url = env("TARGET_DATABASE_URL")
14 | }
15 |
--------------------------------------------------------------------------------
/src/sync.ts:
--------------------------------------------------------------------------------
1 | import { dump } from "./dump";
2 | import { inject, InjectProps } from "./inject";
3 |
4 | export const sync = (props?: InjectProps & { interval?: string }) => {
5 | const parsedInterval = Number(
6 | props?.interval ?? process.env.SYNC_INTERVAL_MS
7 | );
8 | console.log(
9 | `Will scan source database every ${Math.round(
10 | parsedInterval / 1000
11 | )} seconds`
12 | );
13 | let isDumping = false;
14 | setInterval(async () => {
15 | if (isDumping) return;
16 | isDumping = true;
17 | await dump(props);
18 | await inject(props);
19 | isDumping = false;
20 | }, parsedInterval);
21 | };
22 |
--------------------------------------------------------------------------------
/src/utils/fileName.ts:
--------------------------------------------------------------------------------
1 | const transformToValidFilename = (ISOString: string) => {
2 | return ISOString.replace(/:/g, "_") + ".json";
3 | };
4 |
5 | const transformToValidISOString = (name: string) => {
6 | return name.replace(/_/g, ":");
7 | };
8 |
9 | export const fileName = {
10 | transformToValidFilename,
11 | transformToValidISOString,
12 | };
13 |
--------------------------------------------------------------------------------
/src/utils/isEmpty.ts:
--------------------------------------------------------------------------------
1 | export const isEmpty = (value: string | undefined | null): value is undefined =>
2 | value === undefined || value === null || value === "";
3 |
--------------------------------------------------------------------------------
/src/utils/parseProgressBarFormat.ts:
--------------------------------------------------------------------------------
1 | export const progressBarFormat = `[{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | {modelName}`;
2 |
--------------------------------------------------------------------------------
/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "esModuleInterop": true
4 | }
5 | }
6 |
--------------------------------------------------------------------------------