├── .gitignore ├── .editorconfig ├── CHANGELOG.md ├── package.json ├── LICENSE.md ├── bin ├── mongodb-snapshot-read └── mongodb-snapshot-write ├── test └── test.js ├── index.js └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | /package-lock.json 3 | /test/test.snapshot 4 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | end_of_line = lf 8 | indent_size = 2 9 | indent_style = space 10 | insert_final_newline = true 11 | trim_trailing_whitespace = true 12 | 13 | [*.md] 14 | trim_trailing_whitespace = false 15 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 1.1.0 (2025-01-22) 4 | 5 | * Added `--exclude=name1,name2...` option for excluding entire collections in either command, with a similar feature in the API. 6 | * Added `--filter-name1='{ mongodb criteria here }'` option for filtering the documents in a specific collection in the `mongodb-snapshot-write` command, with a similar feature in the API. Note that the collection name is given after `--filter-` as part of the parameter name. 7 | 8 | ## 1.0.0 (2025-01-22) 9 | 10 | * Initial release. 11 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@apostrophecms/mongodb-snapshot", 3 | "version": "1.1.0", 4 | "description": "Save and restore mongodb snapshots without mongodump and mongorestore", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "mocha" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/apostrophecms/mongodb-snapshot.git" 12 | }, 13 | "author": "Apostrophe Technologies", 14 | "license": "MIT", 15 | "bugs": { 16 | "url": "https://github.com/apostrophecms/mongodb-snapshot/issues" 17 | }, 18 | "homepage": "https://github.com/apostrophecms/mongodb-snapshot#readme", 19 | "dependencies": { 20 | "boring": "^1.1.1", 21 | "bson": "^6.10.1", 22 | "mongodb": "^6.12.0" 23 | }, 24 | "devDependencies": { 25 | "mocha": "^11.0.1" 26 | }, 27 | "bin": { 28 | "mongodb-snapshot-write": "./bin/mongodb-snapshot-write", 29 | "mongodb-snapshot-read": "./bin/mongodb-snapshot-read" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2025 Apostrophe Technologies, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /bin/mongodb-snapshot-read: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const { MongoClient } = require('mongodb'); 4 | const { erase, read } = require('../index.js'); 5 | const argv = require('boring')(); 6 | const { existsSync } = require('fs'); 7 | 8 | (async () => { 9 | if (!(argv.from && argv.to)) { 10 | fail( 11 | 'Usage: mongodb-snapshot-read --from=filename.snapshot --to=mongodb://localhost:27017/dbname [--erase]\n\n' + 12 | 'This command will read the snapshot into the database. If --erase is given,\n' + 13 | 'ALL previous contents of the database are removed first.' 14 | ); 15 | } 16 | try { 17 | const client = new MongoClient(argv.to); 18 | await client.connect(); 19 | const db = await client.db(); 20 | const exclude = argv.exclude && argv.exclude.split(','); 21 | if (!existsSync(argv.from)) { 22 | fail(`${from} does not exist`); 23 | } 24 | // If you want to replace the current contents and avoid unique key errors and/or duplicate 25 | // documents, call erase first 26 | if (argv.erase) { 27 | await erase(db); 28 | } 29 | await read(db, argv.from, { exclude }); 30 | await client.close(); 31 | } catch (e) { 32 | fail(e); 33 | } 34 | })(); 35 | 36 | function fail(e) { 37 | console.error(e); 38 | process.exit(1); 39 | } 40 | -------------------------------------------------------------------------------- /bin/mongodb-snapshot-write: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const { MongoClient } = require('mongodb'); 4 | const { write } = require('../index.js'); 5 | const argv = require('boring')(); 6 | 7 | (async () => { 8 | if (!(argv.from && argv.to)) { 9 | fail( 10 | 'Usage: mongodb-snapshot-write --from=mongodb://localhost:27017/dbname --to=some-filename.snapshot\n' + 11 | ' [--exclude=collection1,collection2] [--filter-COLLECTION-NAME-HERE=\'{"prop":"value"}\']\n\n' + 12 | 'This command will write a database snapshot to a file. The filter parameter(s) may contain any\n' + 13 | 'valid MongoDB query as valid JSON.' 14 | ); 15 | } 16 | try { 17 | const client = new MongoClient(argv.from); 18 | await client.connect(); 19 | const db = await client.db(); 20 | const exclude = argv.exclude && argv.exclude.split(','); 21 | const filters = Object.fromEntries(Object.entries(argv) 22 | .filter(([ name ]) => 23 | name.startsWith('filter-') 24 | ).map(([ name, value ]) => 25 | [ name.replace('filter-', ''), JSON.parse(value) ] 26 | ) 27 | ); 28 | await write(db, argv.to, { exclude, filters }); 29 | await client.close(); 30 | } catch (e) { 31 | fail(e); 32 | } 33 | })(); 34 | 35 | function fail(e) { 36 | console.error(e); 37 | process.exit(1); 38 | } 39 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const uri = 'mongodb://localhost:27017/mongodb-snapshot-test-only'; 4 | 5 | const { MongoClient } = require('mongodb'); 6 | 7 | const assert = require('assert'); 8 | 9 | const { read, write, erase } = require('../index.js'); 10 | 11 | 12 | let client, db; 13 | 14 | describe('test mongodb-snapshot', function() { 15 | before(async function() { 16 | await connect(); 17 | await erase(db); 18 | }); 19 | after(async function() { 20 | await close(); 21 | }); 22 | it('can write a snapshot', async function() { 23 | const docs = db.collection('docs'); 24 | await docs.insertOne({ 25 | tull: '35 cents please' 26 | }); 27 | await docs.insertOne({ 28 | tull: 'jethro' 29 | }); 30 | await docs.insertOne({ 31 | tull: 'unwanted' 32 | }); 33 | // Test whether indexes are included in shapshots 34 | await docs.createIndex({ 35 | tull: 1 36 | }, { 37 | unique: true 38 | }); 39 | const cookies = db.collection('cookies'); 40 | await cookies.insertOne({ 41 | name: 'chocolate chip' 42 | }); 43 | const colors = db.collection('colors'); 44 | await colors.insertOne({ 45 | name: 'blue' 46 | }); 47 | await write(db, `${__dirname}/test.snapshot`, { 48 | exclude: [ 'cookies' ], 49 | filters: { 50 | docs: { 51 | tull: { 52 | $ne: 'unwanted' 53 | } 54 | } 55 | } 56 | }); 57 | }); 58 | it('can read a snapshot', async function() { 59 | const docs = db.collection('docs'); 60 | await docs.deleteMany({}); 61 | await docs.dropIndexes(); 62 | const cookies = db.collection('cookies'); 63 | await cookies.deleteMany({}); 64 | await cookies.dropIndexes(); 65 | const colors = db.collection('colors'); 66 | await colors.deleteMany({}); 67 | await colors.dropIndexes(); 68 | await read(db, `${__dirname}/test.snapshot`, { exclude: [ 'colors' ]}); 69 | const result = await docs.find({}).sort({ tull: 1 }).toArray(); 70 | assert.strictEqual(result.length, 2); 71 | assert.strictEqual(result[0].tull, '35 cents please'); 72 | assert.strictEqual(result[1].tull, 'jethro'); 73 | assert.rejects(async function() { 74 | // Should be blocked by the index if it was restored properly 75 | return docs.insertOne({ 76 | tull: 'jethro' 77 | }); 78 | }); 79 | // Verify exclude option of "write" works 80 | const foundCookies = await cookies.find({}).toArray(); 81 | assert.strictEqual(foundCookies.length, 0); 82 | // Verify exclude option of "read" works 83 | const foundColors = await cookies.find({}).toArray(); 84 | assert.strictEqual(foundColors.length, 0); 85 | }); 86 | }); 87 | 88 | async function connect() { 89 | client = new MongoClient(uri); 90 | await client.connect(); 91 | db = await client.db(); 92 | return db; 93 | } 94 | 95 | async function close() { 96 | await client.close(); 97 | } 98 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const readline = require('readline'); 4 | const { EJSON } = require('bson'); 5 | 6 | const { createReadStream, createWriteStream } = require('fs'); 7 | const assert = require('assert'); 8 | 9 | module.exports = { 10 | async read(db, filename, { exclude = null } = {}) { 11 | const input = createReadStream(filename); 12 | const rl = readline.createInterface({ 13 | input, 14 | crlfDelay: Infinity 15 | }); 16 | 17 | let collectionName; 18 | let collection; 19 | let version; 20 | let skipCollection; 21 | 22 | for await (const line of rl) { 23 | const data = EJSON.parse(line); 24 | if (data.metaType === 'version') { 25 | assert.strictEqual(data.value, 1); 26 | version = data.value; 27 | } else if (data.metaType === 'collection') { 28 | assert(version); 29 | collectionName = data.value; 30 | collection = db.collection(collectionName); 31 | skipCollection = exclude && exclude.includes(collectionName); 32 | } else if (data.metaType === 'index') { 33 | assert(collection); 34 | if (skipCollection) { 35 | continue; 36 | } 37 | const { 38 | key, 39 | v, 40 | ...rest 41 | } = data.value; 42 | await collection.createIndex(key, rest); 43 | } else if (data.metaType === 'doc') { 44 | assert(collection); 45 | if (skipCollection) { 46 | continue; 47 | } 48 | await collection.insertOne(data.value); 49 | } 50 | } 51 | }, 52 | async write(db, filename, { exclude = null, filters = {} } = {}) { 53 | const output = createWriteStream(filename); 54 | const collectionsInfo = await db.listCollections().toArray(); 55 | const collectionNames = collectionsInfo.map(({ name }) => name); 56 | await write('version', 1); 57 | for (const name of collectionNames) { 58 | if (exclude && exclude.includes(name)) { 59 | continue; 60 | } 61 | const collection = db.collection(name); 62 | await write('collection', name); 63 | const indexes = await collection.listIndexes().toArray(); 64 | const filter = Object.hasOwn(filters, name) ? filters[name] : {}; 65 | for (const index of indexes) { 66 | await write('index', index); 67 | } 68 | // Get all the _id properties in one go. In theory, we could run out of RAM 69 | // here, but that is very unlikely at a database size that would be encountered 70 | // in Apostrophe. 71 | const idsInfo = await collection.find({}, { _id: 1 }).toArray(); 72 | const _ids = idsInfo.map(({ _id }) => _id); 73 | for (const _id of _ids) { 74 | const doc = await collection.findOne({ 75 | _id, 76 | $and: [ 77 | filter 78 | ] 79 | }); 80 | if (doc) { 81 | await write('doc', doc); 82 | } else { 83 | // This is not an error, documents do go away between operations sometimes 84 | } 85 | } 86 | } 87 | async function write(metaType, value) { 88 | const ready = output.write(EJSON.stringify({ 89 | metaType, 90 | value 91 | }) + '\n'); 92 | // Handle backpressure so we don't buffer the entire database into RAM 93 | if (!ready) { 94 | await new Promise(resolve => { 95 | output.once('drain', () => resolve()); 96 | }); 97 | } 98 | } 99 | output.end(); 100 | await new Promise((resolve, reject) => { 101 | output.on('finish', () => resolve()); 102 | output.on('error', e => reject(e)); 103 | }); 104 | }, 105 | async erase(db) { 106 | const collectionsInfo = await db.listCollections().toArray(); 107 | const collectionNames = collectionsInfo.map(({ name }) => name); 108 | for (const name of collectionNames) { 109 | const collection = db.collection(name); 110 | await collection.drop(); 111 | } 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # @apostrophecms/mongodb-snapshot 2 | 3 | A simple nodejs utility library to create and restore snapshots of mongodb databases without the need for `mongodump` and `mongorestore` to be installed. This reduces unnecessary dependencies on packages users may not be eager to install, especially developers relying solely on MongoDB Atlas for MongoDB hosting. 4 | 5 | Because the format of mongodb archive files is a bit more complicated and apparently undocumented, and the BSON format has no readily available streaming support in JavaScript, this module **does not** read and write mongodump files. Instead it reads and writes a simple format based on EJSON (Extended JSON). 6 | 7 | As a bonus, command line `mongodb-snapshot-write` and `mongodb-snapshot-read` utilities are provided. Of course these do not perform as quickly as `mongodump` and `mongorestore`. 8 | 9 | ## Installation 10 | ```bash 11 | # If you want to use the utilities 12 | npm install -g @apostrophecms/mongodb-snapshot 13 | 14 | # If you want to use the library in your nodejs code 15 | npm install @apostrophecms/mongodb-snapshot 16 | ``` 17 | 18 | ## Command line usage 19 | 20 | ```bash 21 | mongodb-snapshot-write --from=mongodb://localhost:27017/mydbname --to=myfile.snapshot 22 | # add --erase ONLY if you want the previous contents removed first! 23 | mongodb-snapshot-read --from=myfile.snapshot --to=mongodb://localhost:27017/mydbname --erase 24 | ``` 25 | 26 | ### Including and excluding collections 27 | 28 | With both commands, if you want to exclude certain certain collections, just add: 29 | 30 | ``` 31 | --exclude=name1,name2 32 | ``` 33 | 34 | ### Filtering individual documents 35 | 36 | With the `mongodb-snapshot-write` command **only**, you can specify a MongoDB query to 37 | filter the documents for any collection. For example: 38 | 39 | ```bash 40 | mongodb-snapshot-write --from=mongodb://localhost:27017/mydbname --to=myfile.snapshot --filter-COLLECTION-NAME-HERE='{"type":"interesting"}' 41 | ``` 42 | 43 | The query must be valid JSON. Watch out for shell escaping rules (note the single quotes in the example). Note that 44 | MongoDB has a `$regex` operator which can be useful here. 45 | 46 | You may specify filters for as many collections as you wish. Note that the parameter name begins with `--filter-` followed by the collection name. 47 | 48 | ## Node.js API usage 49 | 50 | ```javascript 51 | // Note: ESM import syntax also works. 52 | 53 | // writing 54 | 55 | const { MongoClient } = require('mongodb'); 56 | const { write } = require('@apostrophecms/mongodb-snapshot'); 57 | 58 | async function saveASnapshot() { 59 | const client = new MongoClient(yourMongodbUriGoesHere); 60 | await client.connect(); 61 | const db = await client.db(); 62 | // Writes the contents of db to myfilename.snapshot, including indexes 63 | // Two collections are excluded (optional third argument) 64 | await write(db, 'myfilename.snapshot', { 65 | // No password hashes please 66 | exclude: [ 'aposUsersSafe' ], 67 | filters: { 68 | aposDocs: { 69 | type: { 70 | // No users please 71 | $ne: '@apostrophecms/user' 72 | } 73 | } 74 | }}); 75 | } 76 | ``` 77 | 78 | ```javascript 79 | const { MongoClient } = require('mongodb'); 80 | const { erase, read } = require('@apostrophecms/mongodb-snapshot'); 81 | 82 | async function saveASnapshot() { 83 | const client = new MongoClient(yourMongodbUriGoesHere); 84 | await client.connect(); 85 | const db = await client.db(); 86 | // If you want to replace the current contents and avoid unique key errors and/or duplicate 87 | // documents, call erase first 88 | await erase(db); 89 | // Two collections are excluded (optional third argument) 90 | await read(db, 'myfilename.snapshot', { exclude: [ 'coll3', 'coll4' ]}); 91 | } 92 | ``` 93 | 94 | ## Limitations 95 | 96 | Correctness comes before performance, for now. Sensible amounts of parallelism might help. 97 | 98 | ## Credits 99 | 100 | This module was [originally created for use with ApostropheCMS](https://apostrophecms.com), an open-source Node.js CMS with robust support for in-context, on-page editing, multitenant, multisite projects and lots of other great features worth checking out. 101 | --------------------------------------------------------------------------------