├── index.js ├── .gitignore ├── images └── concept.png ├── lib ├── aliases.js ├── util.js ├── defs.js ├── types.js ├── mysql.js └── tailer.js ├── test ├── wait.js ├── momyfile.exclusions.json ├── momyfile.inclusions.json ├── momyfile.camel.json ├── momyfile.snake.json ├── momyfile.prefix.json ├── mysql-connector.js ├── momyfile.json └── specs │ ├── prefix.js │ ├── filter.js │ ├── importing.js │ ├── types.js │ ├── field-case.js │ └── core.js ├── dev ├── build ├── build-for-release ├── mysql ├── mongo ├── Dockerfile ├── README.md ├── entrypoint └── up ├── circle.yml ├── Dockerfile ├── bin └── momy.js ├── package.json ├── LICENSE └── README.md /index.js: -------------------------------------------------------------------------------- 1 | export * from './lib/tailer.js' 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | coverage 3 | *.log 4 | -------------------------------------------------------------------------------- /images/concept.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cognitom/momy/HEAD/images/concept.png -------------------------------------------------------------------------------- /lib/aliases.js: -------------------------------------------------------------------------------- 1 | export default { 2 | boolean: 'TINYINT', 3 | number: 'DOUBLE', 4 | string: 'VARCHAR' 5 | } 6 | -------------------------------------------------------------------------------- /test/wait.js: -------------------------------------------------------------------------------- 1 | export default function wait (msec) { 2 | return new Promise(function (resolve) { 3 | setTimeout(() => resolve(), msec) 4 | }) 5 | } 6 | -------------------------------------------------------------------------------- /dev/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd "$(dirname "$0")/.." 4 | dirpath="$( pwd -P )" # study where I am 5 | project=${dirpath##*/} # set the name of dir 6 | 7 | docker build -t $project-app -f ./dev/Dockerfile . 8 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | machine: 2 | timezone: Asia/Tokyo 3 | node: 4 | version: 4 5 | 6 | database: 7 | override: 8 | - mysql -u ubuntu -e "create database momy" 9 | 10 | test: 11 | post: 12 | - bash <(curl -s https://codecov.io/bash) 13 | -------------------------------------------------------------------------------- /dev/build-for-release: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd "$(dirname "$0")/.." 4 | dirpath="$( pwd -P )" # study where I am 5 | package_version=$(sed 's/.*"version": "\(.*\)".*/\1/;t;d' ./package.json) 6 | 7 | docker build \ 8 | -t cognitom/momy:latest \ 9 | -t cognitom/momy:v$package_version \ 10 | . 11 | -------------------------------------------------------------------------------- /test/momyfile.exclusions.json: -------------------------------------------------------------------------------- 1 | { 2 | "src": "mongodb://momy-mongod:27017/momy", 3 | "dist": "mysql://root@momy-mysqld:3306/momy", 4 | "exclusions": "\uFFFD", 5 | "collections": { 6 | "colWithExclusions": { 7 | "_id": "string", 8 | "field1": "boolean", 9 | "field2": "number", 10 | "field3": "string" 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /test/momyfile.inclusions.json: -------------------------------------------------------------------------------- 1 | { 2 | "src": "mongodb://momy-mongod:27017/momy", 3 | "dist": "mysql://root@momy-mysqld:3306/momy", 4 | "inclusions": "\u0000-\u007F", 5 | "collections": { 6 | "colWithInclusions": { 7 | "_id": "string", 8 | "field1": "boolean", 9 | "field2": "number", 10 | "field3": "string" 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /dev/mysql: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd "$(dirname "$0")/.." # move to the parent dir 4 | dirpath="$( pwd -P )" # study the dir path 5 | project=${dirpath##*/} # set the name of dir 6 | 7 | echo $@ 8 | 9 | docker run \ 10 | --name $project-mysql-client \ 11 | --interactive --tty --rm \ 12 | --network $project \ 13 | mysql:5.6 \ 14 | mysql --host=$project-mysqld $@ 15 | -------------------------------------------------------------------------------- /dev/mongo: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mongo_version="5.0.26" 4 | 5 | cd "$(dirname "$0")/.." # move to the parent dir 6 | dirpath="$( pwd -P )" # study the dir path 7 | project=${dirpath##*/} # set the name of dir 8 | 9 | docker run \ 10 | --name $project-mongo-client \ 11 | --interactive --tty --rm \ 12 | --network $project \ 13 | mongo:$mongo_version \ 14 | mongo --host=$project-mongod $@ 15 | -------------------------------------------------------------------------------- /lib/util.js: -------------------------------------------------------------------------------- 1 | function info (message) { 2 | process.stdout.write(message + '\n') 3 | } 4 | function error (message) { 5 | process.stderr.write(message + '\n') 6 | } 7 | export const logger = { info, error } 8 | 9 | export function getDbNameFromUri (uri) { 10 | const matches = uri.match(/(?<=\/)\w+(?=\?|$)/) 11 | if (!matches) { 12 | throw new Error('No database name is specified in uri string.') 13 | } 14 | return matches.shift() 15 | } 16 | -------------------------------------------------------------------------------- /test/momyfile.camel.json: -------------------------------------------------------------------------------- 1 | { 2 | "src": "mongodb://momy-mongod:27017/momy", 3 | "dist": "mysql://root@momy-mysqld:3306/momy", 4 | "fieldCase": "camel", 5 | "collections": { 6 | "colCamelCases": { 7 | "_id": "string", 8 | "field1": "string", 9 | "field2.sub1": "string", 10 | "field2.sub2": "string", 11 | "field3.sub1.sub2": "string", 12 | "field4_sub1_sub2": "string", 13 | "field5Sub1Sub2": "string" 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /test/momyfile.snake.json: -------------------------------------------------------------------------------- 1 | { 2 | "src": "mongodb://momy-mongod:27017/momy", 3 | "dist": "mysql://root@momy-mysqld:3306/momy", 4 | "fieldCase": "snake", 5 | "collections": { 6 | "colSnakeCases": { 7 | "_id": "string", 8 | "field1": "string", 9 | "field2.sub1": "string", 10 | "field2.sub2": "string", 11 | "field3.sub1.sub2": "string", 12 | "field4_sub1_sub2": "string", 13 | "field5Sub1Sub2": "string" 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /test/momyfile.prefix.json: -------------------------------------------------------------------------------- 1 | { 2 | "src": "mongodb://momy-mongod:27017/momy", 3 | "dist": "mysql://root@momy-mysqld:3306/momy", 4 | "prefix": "p_", 5 | "collections": { 6 | "colWithPrefix1": { 7 | "_id": "string", 8 | "field1": "boolean", 9 | "field2": "number", 10 | "field3": "string" 11 | }, 12 | "colWithPrefix2": { 13 | "_id": "string", 14 | "field1": "boolean", 15 | "field2": "number", 16 | "field3": "string" 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:18 AS build-env 2 | 3 | # install dependencies via npm 4 | WORKDIR /app 5 | COPY package*.json ./ 6 | RUN npm install --only=production 7 | 8 | # multi-stage build for smaller package 9 | FROM node:18-alpine 10 | COPY --from=build-env /app /app 11 | 12 | # copy momy itself 13 | COPY bin /app/bin 14 | COPY lib /app/lib 15 | 16 | # set working directory which would be bound to host's $PWD 17 | WORKDIR /workdir 18 | 19 | # wrap tini for signal handling 20 | ENTRYPOINT ["/usr/local/bin/node", "/app/bin/momy.js"] 21 | -------------------------------------------------------------------------------- /bin/momy.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import Tailer from '../lib/tailer.js' 3 | import fs from 'fs' 4 | 5 | const DEFAULT_CONFIG_PATH = 'momyfile.json' 6 | const refresh = process.argv.some(c => c === '--import') 7 | const finder = (p, c, i, a) => c === '--config' && a[i + 1] ? a[i + 1] : p 8 | const file = process.argv.reduce(finder, DEFAULT_CONFIG_PATH) 9 | const config = JSON.parse(fs.readFileSync(process.cwd() + '/' + file)) 10 | const tailer = new Tailer(config) 11 | 12 | if (refresh) tailer.importAndStart() 13 | else tailer.start() 14 | -------------------------------------------------------------------------------- /test/mysql-connector.js: -------------------------------------------------------------------------------- 1 | import mysql from 'mysql' 2 | 3 | export class MysqlConnector { 4 | constructor (url) { 5 | this.con = mysql.createConnection(url) 6 | this.con.connect(function (err) { 7 | if (err) console.log(`SQL CONNECT ERROR: ${err}`) 8 | }) 9 | } 10 | query (sql) { 11 | return new Promise((resolve, reject) => { 12 | this.con.query(sql, (err, results) => { 13 | if (err) return reject(err) 14 | resolve(results) 15 | }) 16 | }) 17 | } 18 | close () { 19 | this.con.end() 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /dev/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:18 2 | 3 | USER root 4 | 5 | # Basic tools 6 | RUN apt-get update && \ 7 | cd /tmp && git clone https://github.com/ncopa/su-exec.git && cd su-exec && make && mv su-exec /usr/local/bin && \ 8 | cd /usr/bin && curl https://getmic.ro | bash 9 | 10 | # Working directory 11 | WORKDIR /app 12 | 13 | # Node tools 14 | ENV NPM_CONFIG_LOGLEVEL=warn 15 | RUN npm install --global \ 16 | npm@10.5.1 \ 17 | npm-check-updates@^16.14.18 \ 18 | mocha@^10.4.0 \ 19 | standard@^17.1.0 20 | 21 | ENTRYPOINT ["bash", "/app/dev/entrypoint"] 22 | -------------------------------------------------------------------------------- /dev/README.md: -------------------------------------------------------------------------------- 1 | # Development 2 | 3 | Enter the Docker container for development: 4 | 5 | ```bash 6 | $ bash dev/up 7 | ``` 8 | 9 | To install dependencies: 10 | 11 | ```bash 12 | $ npm install 13 | ``` 14 | 15 | To run the tests: 16 | 17 | ```bash 18 | $ npm test 19 | ``` 20 | 21 | To test the code manually, start `momy`: 22 | 23 | ```bash 24 | $ npm run try 25 | ``` 26 | 27 | Then, open a new terminal on the host (not inside the container), and run mongo client: 28 | 29 | ```bash 30 | $ bash dev/mongo 31 | ``` 32 | 33 | Do something, and run mysql client to check syncing: 34 | 35 | ```bash 36 | $ bash dev/mysql 37 | ``` 38 | -------------------------------------------------------------------------------- /test/momyfile.json: -------------------------------------------------------------------------------- 1 | { 2 | "src": "mongodb://momy-mongod:27017/momy", 3 | "dist": "mysql://root@momy-mysqld:3306/momy", 4 | "collections": { 5 | "colBasicTypes": { 6 | "_id": "string", 7 | "field1": "boolean", 8 | "field2": "number", 9 | "field3": "string" 10 | }, 11 | "colNumberTypes": { 12 | "_id": "string", 13 | "field1": "BIGINT", 14 | "field2": "DOUBLE", 15 | "field3": "TINYINT" 16 | }, 17 | "colDateTypes": { 18 | "_id": "string", 19 | "field1": "DATE", 20 | "field2": "DATETIME", 21 | "field3": "TIME" 22 | }, 23 | "colStringTypes": { 24 | "_id": "string", 25 | "field1": "VARCHAR", 26 | "field2": "TEXT" 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "momy", 3 | "version": "0.8.2", 4 | "description": "MongoDB to MySQL replication", 5 | "type": "module", 6 | "exports": "./index.js", 7 | "bin": { 8 | "momy": "bin/momy.js" 9 | }, 10 | "files": [ 11 | "bin", 12 | "lib", 13 | "index.js" 14 | ], 15 | "scripts": { 16 | "test": "npm run standard && npm run mocha", 17 | "mocha": "env TZ='Asia/Tokyo' mocha -t 20000 test/specs/*.js", 18 | "mocha-inspect": "env TZ='Asia/Tokyo' mocha -t 20000 --inspect-brk=0.0.0 test/specs/*.js", 19 | "standard": "standard bin/*.js lib/*.js test/**/*.js", 20 | "unit-test": "env TZ='Asia/Tokyo' mocha -t 20000 test/specs/types.js", 21 | "try": "node ./bin/momy.js --config test/momyfile.json" 22 | }, 23 | "dependencies": { 24 | "change-case": "^5.4.4", 25 | "moment": "^2.30.1", 26 | "mongodb": "^6.5.0", 27 | "mysql": "^2.18.1", 28 | "sqlstring": "^2.3.3" 29 | }, 30 | "repository": { 31 | "type": "git", 32 | "url": "git://github.com/cognitom/momy.git" 33 | }, 34 | "author": "Tsutomu Kawamura", 35 | "license": "MIT" 36 | } 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 Bruce Dou 4 | Copyright (c) 2015 Tsutomu Kawamura 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining 7 | a copy of this software and associated documentation files (the 8 | "Software"), to deal in the Software without restriction, including 9 | without limitation the rights to use, copy, modify, merge, publish, 10 | distribute, sublicense, and/or sell copies of the Software, and to 11 | permit persons to whom the Software is furnished to do so, subject to 12 | the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be 15 | included in all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /dev/entrypoint: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ ! -f /.dockerenv ]; then 4 | echo '🚨 Do not run it outside a Docker container.' 1>&2 5 | exit 1 6 | fi 7 | 8 | # Create a user which has the common uid:gid with the host, 9 | # or check if the user exists 10 | 11 | host_uid=$(ls -n $0 | awk '{print $3}') # study who my owner is 12 | host_gid=$(ls -n $0 | awk '{print $4}') # study what I belong to 13 | 14 | user=$(cat /etc/passwd | grep ":x:$host_uid:" | cut -d: -f1) 15 | group=$(cat /etc/group | grep ":x:$host_gid:" | cut -d: -f1) 16 | 17 | if [ -z "$group" ]; then 18 | group=app 19 | echo "Creating a group..." 20 | groupadd --gid $host_gid $group 21 | fi 22 | if [ -z "$user" ]; then 23 | user=app 24 | echo "Creating an user..." 25 | useradd --uid $host_uid --gid $host_gid --shell /bin/bash $user 26 | 27 | # Add some default files to the home 28 | mkdir -p /home/$user 29 | cp -r /etc/skel/. /home/$user 30 | chown $user:$group /home/$user /home/$user/.bash* /home/$user/.profile 31 | else 32 | echo "The user $user already exists in this container (id:$host_uid)" 33 | fi 34 | 35 | # Execute the command left, or enter `bash` 36 | if [ $# -gt 0 ]; then 37 | exec su-exec $user $@ 38 | else 39 | exec su-exec $user bash 40 | fi 41 | -------------------------------------------------------------------------------- /dev/up: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mongo_version="5.0.26" 4 | mysql_version="5.6" 5 | app_dir="/app" 6 | home_dir="/home/app" 7 | 8 | cd "$(dirname "$0")/.." # move to the parent dir 9 | dirpath="$( pwd -P )" # study the dir path 10 | project=${dirpath##*/} # set the name of dir 11 | 12 | # 13 | # Starting up... 14 | # 15 | docker network create $project > /dev/null 16 | 17 | docker run \ 18 | --name $project-mongod \ 19 | --detach --rm \ 20 | --network $project \ 21 | --publish 27017:27017 \ 22 | mongo:$mongo_version --replSet "rs0" > /dev/null 23 | 24 | docker run \ 25 | --name $project-mysqld \ 26 | --detach --rm \ 27 | --network $project \ 28 | --publish 3306:3306 \ 29 | --env MYSQL_ALLOW_EMPTY_PASSWORD=yes \ 30 | mysql:$mysql_version > /dev/null 31 | 32 | # Build dev image 33 | if [ -z "$(docker image ls -q $project-app)" ]; then 34 | bash dev/build 35 | fi 36 | 37 | # Initialize databases 38 | echo -n "Waiting mysql." 39 | while ! docker exec $project-mysqld mysqladmin ping --silent > /dev/null; do 40 | echo -n . 41 | sleep 1 42 | done 43 | echo " OK." 44 | docker exec $project-mongod mongo --eval 'rs.initiate()' > /dev/null 45 | docker exec $project-mysqld mysql -e "CREATE DATABASE momy;" > /dev/null 46 | 47 | # Run dev container 48 | docker run \ 49 | --name $project-app \ 50 | --interactive --tty --rm \ 51 | --network $project \ 52 | --publish 9229:9229 \ 53 | --mount type=bind,source=$dirpath,target=$app_dir \ 54 | --mount type=bind,source="$HOME/.ssh",target="$home_dir/.ssh" \ 55 | --mount type=bind,source="$HOME/.gitconfig",target="$home_dir/.gitconfig" \ 56 | $project-app $@ 57 | 58 | # 59 | # Shutting down... 60 | # 61 | docker stop $project-mysqld > /dev/null 62 | docker stop $project-mongod > /dev/null 63 | docker network rm $project > /dev/null 64 | -------------------------------------------------------------------------------- /test/specs/prefix.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import assert from 'assert' 4 | import fs from 'fs' 5 | import { MongoClient } from 'mongodb' 6 | import wait from '../wait.js' 7 | 8 | import Tailer from '../../lib/tailer.js' 9 | import { MysqlConnector } from '../mysql-connector.js' 10 | import { getDbNameFromUri } from '../../lib/util.js' 11 | 12 | const momyfilePrefix = JSON.parse(fs.readFileSync('./test/momyfile.prefix.json')) 13 | const waitingTime = 500 14 | 15 | describe('Momy: prefix', () => { 16 | it('adds prefix to collections', async function () { 17 | const config = momyfilePrefix 18 | const dbname = getDbNameFromUri(config.src) 19 | const client = new MongoClient(config.src) 20 | const mo = client.db(dbname) 21 | const prefix = 'p_' 22 | const colName1 = 'colWithPrefix1' 23 | const colName2 = 'colWithPrefix2' 24 | 25 | // clear existing records 26 | await mo.collection(colName1).deleteMany({}) 27 | await mo.collection(colName2).deleteMany({}) 28 | 29 | const doc = { 30 | field1: true, // boolean 31 | field2: 123, // number 32 | field3: 'Tom' // string 33 | } 34 | const r1 = await mo.collection(colName1).insertOne(doc) 35 | const r2 = await mo.collection(colName2).insertOne(doc) 36 | client.close() 37 | 38 | const tailer = new Tailer(config, false) 39 | tailer.importAndStart(false) 40 | await wait(waitingTime * 4) // wait for syncing 41 | tailer.stop() 42 | 43 | const my = new MysqlConnector(config.dist) 44 | const q1 = await my.query(`SELECT * FROM ${prefix}${colName1} WHERE _id = "${r1.insertedId}"`) 45 | const q2 = await my.query(`SELECT * FROM ${prefix}${colName2} WHERE _id = "${r2.insertedId}"`) 46 | my.close() 47 | 48 | assert.equal(q1[0].field3, 'Tom') 49 | assert.equal(q2[0].field3, 'Tom') 50 | await wait(waitingTime * 2) 51 | }) 52 | }) 53 | -------------------------------------------------------------------------------- /test/specs/filter.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import assert from 'assert' 4 | import fs from 'fs' 5 | import { MongoClient } from 'mongodb' 6 | import wait from '../wait.js' 7 | 8 | import Tailer from '../../lib/tailer.js' 9 | import { MysqlConnector } from '../mysql-connector.js' 10 | import { getDbNameFromUri } from '../../lib/util.js' 11 | 12 | const momyfileExclusions = JSON.parse(fs.readFileSync('./test/momyfile.exclusions.json')) 13 | const momyfileInclusions = JSON.parse(fs.readFileSync('./test/momyfile.inclusions.json')) 14 | 15 | const waitingTime = 500 16 | 17 | describe('Momy: filter', () => { 18 | it('excludes custom chars', async function () { 19 | const config = momyfileExclusions 20 | const dbname = getDbNameFromUri(config.src) 21 | const client = new MongoClient(config.src) 22 | const mo = client.db(dbname) 23 | const colName1 = 'colWithExclusions' 24 | 25 | // clear existing records 26 | await mo.collection(colName1).deleteMany({}) 27 | 28 | const doc = { 29 | field1: true, // boolean 30 | field2: 123, // number 31 | field3: 'T\uFFFDo\uFFFDm' // string 32 | } 33 | const r1 = await mo.collection(colName1).insertOne(doc) 34 | client.close() 35 | 36 | const tailer = new Tailer(config, false) 37 | tailer.importAndStart(false) 38 | await wait(waitingTime * 4) // wait for syncing 39 | tailer.stop() 40 | 41 | const my = new MysqlConnector(config.dist) 42 | const q1 = await my.query(`SELECT * FROM ${colName1} WHERE _id = "${r1.insertedId}"`) 43 | my.close() 44 | 45 | assert.equal(q1[0].field3, 'Tom') 46 | await wait(waitingTime * 2) 47 | }) 48 | 49 | it('includes custom chars', async function () { 50 | const config = momyfileInclusions 51 | const dbname = getDbNameFromUri(config.src) 52 | const client = new MongoClient(config.src) 53 | const mo = client.db(dbname) 54 | const colName1 = 'colWithInclusions' 55 | 56 | // clear existing records 57 | await mo.collection(colName1).deleteMany({}) 58 | 59 | const doc = { 60 | field1: true, // boolean 61 | field2: 123, // number 62 | field3: '河村Tom奨' // string 63 | } 64 | const r1 = await mo.collection(colName1).insertOne(doc) 65 | client.close() 66 | 67 | const tailer = new Tailer(config, false) 68 | tailer.importAndStart(false) 69 | await wait(waitingTime * 4) // wait for syncing 70 | tailer.stop() 71 | 72 | const my = new MysqlConnector(config.dist) 73 | const q1 = await my.query(`SELECT * FROM ${colName1} WHERE _id = "${r1.insertedId}"`) 74 | my.close() 75 | 76 | assert.equal(q1[0].field3, 'Tom') 77 | await wait(waitingTime * 2) 78 | }) 79 | }) 80 | -------------------------------------------------------------------------------- /lib/defs.js: -------------------------------------------------------------------------------- 1 | import * as changeCase from 'change-case' 2 | import NATIVE_TYPES from './types.js' 3 | import TYPE_ALIASES from './aliases.js' 4 | 5 | /** 6 | * Create definition list from config object 7 | * @param {object} collections - config object 8 | * @param {string} dbName - name of database 9 | * @param {string} opts - options 10 | * @returns {undefined} void 11 | */ 12 | export function createDefs (collections, dbName, opts) { 13 | const acceptedTypes = Object.keys(TYPE_ALIASES).concat(Object.keys(NATIVE_TYPES)) 14 | return Object.keys(collections).map(name => { 15 | // Primary key must be `_id` or `id` 16 | const idName = collections[name]._id ? '_id' : 'id' 17 | return { 18 | name, 19 | ns: `${dbName}.${name}`, 20 | distName: opts.prefix + name, 21 | idName, 22 | idDistName: convertCase(idName, opts.fieldCase), 23 | // `_id` or `id` must be string or number 24 | idType: collections[name][idName], 25 | fields: Object.keys(collections[name]) 26 | // Skip unknown types 27 | .filter(fieldName => acceptedTypes.some(accepted => 28 | accepted === collections[name][fieldName])) 29 | // Build definition 30 | .map(fieldName => { 31 | const fieldType = collections[name][fieldName] 32 | const nativeType = TYPE_ALIASES[fieldType] || fieldType 33 | const convert = NATIVE_TYPES[nativeType].convert 34 | return { 35 | name: fieldName, 36 | distName: convertCase(fieldName, opts.fieldCase), 37 | type: NATIVE_TYPES[nativeType].type, 38 | convert: val => { 39 | const isText = nativeType === 'VARCHAR' || nativeType === 'TEXT' 40 | if (isText) val = filterChars(val, opts.exclusions, opts.inclusions) 41 | return convert(val) 42 | }, 43 | primary: /^_?id$/.test(fieldName) // set primary for 'id' or '_id' 44 | } 45 | }) 46 | } 47 | }) 48 | } 49 | 50 | /** 51 | * Change case of string 52 | * @param {string} str - field name 53 | * @returns {string} converted string 54 | */ 55 | function convertCase (str, fieldCase) { 56 | return fieldCase === 'camel' 57 | ? changeCase.camelCase(str) 58 | : fieldCase === 'snake' 59 | ? changeCase.snakeCase(str) 60 | : str 61 | } 62 | 63 | /** 64 | * Exclude or include some chars 65 | * @param {string} str - string 66 | * @param {string} exclusions - char set to exclude 67 | * @param {string} inclusions - char set to include 68 | * @returns {string} converted string 69 | */ 70 | function filterChars (str, exclusions, inclusions) { 71 | str = str || '' 72 | if (typeof str !== 'string') str = str.toString() 73 | if (exclusions) str = str.replace(new RegExp(`[${exclusions}]`, 'g'), '') 74 | if (inclusions) str = str.replace(new RegExp(`[^${inclusions}]`, 'g'), '') 75 | return str 76 | } 77 | -------------------------------------------------------------------------------- /test/specs/importing.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import assert from 'assert' 4 | import fs from 'fs' 5 | import { MongoClient } from 'mongodb' 6 | import wait from '../wait.js' 7 | 8 | import Tailer from '../../lib/tailer.js' 9 | import { MysqlConnector } from '../mysql-connector.js' 10 | import { getDbNameFromUri } from '../../lib/util.js' 11 | 12 | const momyfile = JSON.parse(fs.readFileSync('./test/momyfile.json')) 13 | const waitingTime = 500 14 | 15 | describe('Momy: importing', () => { 16 | it('imports all docs already exist', async function () { 17 | const dbname = getDbNameFromUri(momyfile.src) 18 | const client = new MongoClient(momyfile.src) 19 | const mo = client.db(dbname) 20 | const colName = 'colBasicTypes' 21 | 22 | // clear existing records 23 | await mo.collection(colName).deleteMany({}) 24 | 25 | const docs = Array.from(Array(10)).map((_, i) => ({ 26 | field1: true, 27 | field2: i, 28 | field3: `Tom-${i}` 29 | })) 30 | for (const doc of docs) { 31 | const r = await mo.collection(colName).insertOne(doc) 32 | doc._id = r.insertedId 33 | } 34 | client.close() 35 | 36 | const tailer = new Tailer(momyfile, false) 37 | tailer.importAndStart(false) 38 | await wait(waitingTime * 2) // wait for syncing 39 | tailer.stop() 40 | await wait(waitingTime * 2) 41 | 42 | const my = new MysqlConnector(momyfile.dist) 43 | for (const doc of docs) { 44 | const r = await my.query(`SELECT * FROM ${colName} WHERE _id = "${doc._id}"`) 45 | assert.equal(r[0].field2, doc.field2) 46 | } 47 | my.close() 48 | }) 49 | 50 | it('imports all docs and restart syncing', async function () { 51 | const dbname = getDbNameFromUri(momyfile.src) 52 | const client = new MongoClient(momyfile.src) 53 | const mo = client.db(dbname) 54 | const colName = 'colBasicTypes' 55 | 56 | // clear existing records 57 | await mo.collection(colName).deleteMany({}) 58 | 59 | const r0 = await mo.collection(colName).insertOne({ 60 | field1: true, 61 | field2: 1, 62 | field3: 'Tom' 63 | }) 64 | const tailer = new Tailer(momyfile, false) 65 | tailer.importAndStart(false) 66 | await wait(waitingTime * 2) // wait for syncing 67 | tailer.stop() 68 | await wait(waitingTime) // wait for stopping 69 | 70 | tailer.start(false) 71 | await wait(waitingTime * 2) // wait for starting 72 | const r1 = await mo.collection(colName).insertOne({ 73 | field1: true, 74 | field2: 2, 75 | field3: 'John' 76 | }) 77 | client.close() 78 | await wait(waitingTime * 2) // wait for syncing 79 | tailer.stop() 80 | await wait(waitingTime) // wait for stopping 81 | 82 | const my = new MysqlConnector(momyfile.dist) 83 | const q0 = await my.query(`SELECT * FROM ${colName} WHERE _id = "${r0.insertedId}"`) 84 | const q1 = await my.query(`SELECT * FROM ${colName} WHERE _id = "${r1.insertedId}"`) 85 | my.close() 86 | 87 | assert.equal(q0[0].field3, 'Tom') 88 | assert.equal(q1[0].field3, 'John') 89 | }) 90 | }) 91 | -------------------------------------------------------------------------------- /lib/types.js: -------------------------------------------------------------------------------- 1 | import moment from 'moment' 2 | import sqlstring from 'sqlstring' 3 | 4 | const controlRegex = /[\x00-\x1F\x7F]/g // eslint-disable-line no-control-regex 5 | export default { 6 | BIGINT: { 7 | type: 'BIGINT', 8 | convert: val => parseInt(val || 0) 9 | }, 10 | DATE: { 11 | type: 'DATE', 12 | convert: val => { 13 | if (typeof val === 'string') val = getValueOfDate(val) 14 | if (typeof val === 'number' || val instanceof Date) { 15 | val = moment(val).format('YYYY-MM-DD') 16 | return `"${val}"` 17 | } 18 | return 'NULL' 19 | } 20 | }, 21 | DATETIME: { 22 | type: 'DATETIME', 23 | convert: val => { 24 | if (typeof val === 'string') val = getValueOfDate(val) 25 | if (typeof val === 'number' || val instanceof Date) { 26 | val = moment(val).format('YYYY-MM-DD HH:mm:ss') 27 | return `"${val}"` 28 | } 29 | return 'NULL' 30 | } 31 | }, 32 | DOUBLE: { 33 | type: 'DOUBLE(20, 10)', 34 | convert: val => parseFloat(val || 0) 35 | }, 36 | TIME: { 37 | type: 'TIME', 38 | convert: val => { 39 | if (typeof val === 'string') val = normalizeTime(val) 40 | if (typeof val === 'number' || val instanceof Date) val = moment(val).format('HH:mm:ss') 41 | if (typeof val !== 'string') return 'NULL' 42 | return `"${val}"` 43 | } 44 | }, 45 | TINYINT: { 46 | type: 'TINYINT', 47 | convert: val => !!val 48 | }, 49 | VARCHAR: { 50 | type: 'VARCHAR(255)', 51 | convert: val => { 52 | val = (val || '').toString() 53 | val = val.substring(0, 255) 54 | val = sqlstring.escape(val) // escape \0 \b \t \n \r \x1a 55 | val = val.replace(controlRegex, '') 56 | return val 57 | } 58 | }, 59 | TEXT: { 60 | type: 'TEXT', 61 | convert: val => { 62 | val = (val || '').toString() 63 | val = sqlstring.escape(val) // escape \0 \b \t \n \r \x1a 64 | val = val.replace(controlRegex, '') 65 | return val 66 | } 67 | } 68 | } 69 | 70 | /** 71 | * Get a number expression from a date string 72 | * @param {string} str - date value 73 | * @returns {number|null} Timestamp in msec or null if not a valid value 74 | */ 75 | function getValueOfDate (str) { 76 | const reIso8601 = /^\d{4}-\d{2}-\d{2}([ T]\d{2}(:\d{2}(:\d{2}(\.\d{3})?)?)?([+-]\d{2}(:?\d{2})?)?)?$/ 77 | const reIso8601Short = /^\d{4}\d{2}\d{2}(T\d{2}(\d{2}(\d{2}(\.\d{3})?)?)?)?$/ 78 | const reTimestamp = /^\d+$/ 79 | if (reIso8601.test(str) || reIso8601Short.test(str)) return moment(str).valueOf() 80 | if (reTimestamp.test(str)) return parseInt(str) 81 | return null 82 | } 83 | 84 | /** 85 | * Get a normalized time string 86 | * @param {string} str - time value 87 | * @returns {string|null} time or null 88 | */ 89 | function normalizeTime (str) { 90 | const re = /^(\d{1,2}):(\d{2})(:(\d{2}))?$/ 91 | if (!re.test(str)) return null 92 | const found = str.match(re) 93 | const hh = found[1].length === 1 ? '0' + found[1] : found[1] 94 | const mm = found[2] 95 | const ss = found[4] || '00' 96 | return `${hh}:${mm}:${ss}` 97 | } 98 | -------------------------------------------------------------------------------- /test/specs/types.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 'use strict' 3 | 4 | import assert from 'assert' 5 | import NATIVE_TYPES from '../../lib/types.js' 6 | 7 | describe('Momy Types', () => { 8 | it('BIGINT', () => { 9 | const convert = NATIVE_TYPES.BIGINT.convert 10 | assert.equal(convert(1), 1) 11 | assert.equal(convert(1.2), 1) 12 | assert.equal(convert(123456789), 123456789) 13 | assert.equal(convert(undefined), 0) 14 | }) 15 | 16 | it('DATE', () => { 17 | const convert = NATIVE_TYPES.DATE.convert 18 | assert.equal(convert(undefined), 'NULL') 19 | assert.equal(convert('abcde'), 'NULL') 20 | assert.equal(convert('2016-11-10'), '"2016-11-10"') 21 | assert.equal(convert(1478775921696), '"2016-11-10"') 22 | assert.equal(convert('1478775921696'), '"2016-11-10"') 23 | }) 24 | 25 | it('DATETIME', () => { 26 | const convert = NATIVE_TYPES.DATETIME.convert 27 | assert.equal(convert(undefined), 'NULL') 28 | assert.equal(convert('abcde'), 'NULL') 29 | assert.equal(convert('2016-11-10'), '"2016-11-10 00:00:00"') 30 | assert.equal(convert(1478775921696), '"2016-11-10 20:05:21"') 31 | assert.equal(convert('1478775921696'), '"2016-11-10 20:05:21"') 32 | }) 33 | 34 | it('DOUBLE', () => { 35 | const convert = NATIVE_TYPES.DOUBLE.convert 36 | assert.equal(convert(1), 1) 37 | assert.equal(convert(1.2), 1.2) 38 | assert.equal(convert(1.23456789), 1.23456789) 39 | assert.equal(convert(undefined), 0) 40 | }) 41 | 42 | it('TIME', () => { 43 | const convert = NATIVE_TYPES.TIME.convert 44 | assert.equal(convert(undefined), 'NULL') 45 | assert.equal(convert('abcde'), 'NULL') 46 | assert.equal(convert('1:23'), '"01:23:00"') 47 | assert.equal(convert('12:34'), '"12:34:00"') 48 | assert.equal(convert('12:34:56'), '"12:34:56"') 49 | assert.equal(convert(1478775921696), '"20:05:21"') 50 | }) 51 | 52 | it('TINYINT', () => { 53 | const convert = NATIVE_TYPES.TINYINT.convert 54 | assert.equal(convert(true), 1) 55 | assert.equal(convert(false), 0) 56 | assert.equal(convert(1), 1) 57 | assert.equal(convert(0), 0) 58 | assert.equal(convert(undefined), 0) 59 | }) 60 | 61 | it('VARCHAR', () => { 62 | const convert = NATIVE_TYPES.VARCHAR.convert 63 | const allAscii = Array(95).map((_, i) => String.fromCharCode(32 + i)).join('') 64 | const a1000 = Array(1000).map(() => 'a').join('') 65 | const a255 = Array(255).map(() => 'a').join('') 66 | assert.equal(convert(allAscii), `'${allAscii}'`) 67 | assert.equal(convert(a1000), `'${a255}'`) // truncate 68 | assert.equal(convert(undefined), "''") 69 | assert.equal(convert('\x07'), "''") // skip a control char 70 | }) 71 | 72 | it('TEXT', () => { 73 | const convert = NATIVE_TYPES.TEXT.convert 74 | const allAscii = Array(95).map((_, i) => String.fromCharCode(32 + i)).join('') 75 | const a1000 = Array(1000).map(() => 'a').join('') 76 | assert.equal(convert(allAscii), `'${allAscii}'`) 77 | assert.equal(convert(a1000), `'${a1000}'`) // truncate 78 | assert.equal(convert(undefined), "''") 79 | assert.equal(convert('\x07'), "''") // skip a control char 80 | }) 81 | }) 82 | -------------------------------------------------------------------------------- /test/specs/field-case.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import assert from 'assert' 4 | import fs from 'fs' 5 | import { MongoClient } from 'mongodb' 6 | import wait from '../wait.js' 7 | 8 | import Tailer from '../../lib/tailer.js' 9 | import { MysqlConnector } from '../mysql-connector.js' 10 | import { getDbNameFromUri } from '../../lib/util.js' 11 | 12 | const momyfileCamel = JSON.parse(fs.readFileSync('./test/momyfile.camel.json')) 13 | const momyfileSnake = JSON.parse(fs.readFileSync('./test/momyfile.snake.json')) 14 | const waitingTime = 1000 15 | 16 | describe('Momy: fieldCase', () => { 17 | it('camel', async function () { 18 | const config = momyfileCamel 19 | const dbname = getDbNameFromUri(config.src) 20 | const client = new MongoClient(config.src) 21 | const mo = client.db(dbname) 22 | const colName = 'colCamelCases' 23 | 24 | // clear existing records 25 | await mo.collection(colName).deleteMany({}) 26 | 27 | const doc = { 28 | field1: 'abc', 29 | field2: { sub1: 'def', sub2: 'ghi' }, 30 | field3: { sub1: { sub2: 'jkl' } }, 31 | field4_sub1_sub2: 'mno', 32 | field5Sub1Sub2: 'pqr' 33 | } 34 | const r0 = await mo.collection(colName).insertOne(doc) 35 | client.close() 36 | 37 | const tailer = new Tailer(config, false) 38 | tailer.importAndStart(false) 39 | await wait(waitingTime * 2) // wait for syncing 40 | tailer.stop() 41 | 42 | const my = new MysqlConnector(momyfileCamel.dist) 43 | const r1 = await my.query(`SELECT * FROM ${colName} WHERE id = "${r0.insertedId}"`) 44 | my.close() 45 | 46 | assert.equal(r1[0].field1, 'abc') 47 | assert.equal(r1[0].field2Sub1, 'def') 48 | assert.equal(r1[0].field2Sub2, 'ghi') 49 | assert.equal(r1[0].field3Sub1Sub2, 'jkl') 50 | assert.equal(r1[0].field4Sub1Sub2, 'mno') 51 | assert.equal(r1[0].field5Sub1Sub2, 'pqr') 52 | await wait(waitingTime * 2) 53 | }) 54 | 55 | it('snake', async function () { 56 | const config = momyfileSnake 57 | const dbname = getDbNameFromUri(config.src) 58 | const client = new MongoClient(config.src) 59 | const mo = client.db(dbname) 60 | const colName = 'colSnakeCases' 61 | 62 | // clear existing records 63 | await mo.collection(colName).deleteMany({}) 64 | 65 | const doc = { 66 | field1: 'abc', 67 | field2: { sub1: 'def', sub2: 'ghi' }, 68 | field3: { sub1: { sub2: 'jkl' } }, 69 | field4_sub1_sub2: 'mno', 70 | field5Sub1Sub2: 'pqr' 71 | } 72 | const r0 = await mo.collection(colName).insertOne(doc) 73 | client.close() 74 | 75 | const tailer = new Tailer(config, false) 76 | tailer.importAndStart(false) 77 | await wait(waitingTime * 2) // wait for syncing 78 | tailer.stop() 79 | 80 | const my = new MysqlConnector(config.dist) 81 | const r1 = await my.query(`SELECT * FROM ${colName} WHERE id = "${r0.insertedId}"`) 82 | my.close() 83 | 84 | assert.equal(r1[0].field1, 'abc') 85 | assert.equal(r1[0].field2_sub1, 'def') 86 | assert.equal(r1[0].field2_sub2, 'ghi') 87 | assert.equal(r1[0].field3_sub1_sub2, 'jkl') 88 | assert.equal(r1[0].field4_sub1_sub2, 'mno') 89 | assert.equal(r1[0].field5_sub1_sub2, 'pqr') 90 | await wait(waitingTime * 2) 91 | }) 92 | }) 93 | -------------------------------------------------------------------------------- /lib/mysql.js: -------------------------------------------------------------------------------- 1 | import mysql from 'mysql' 2 | import { logger } from './util.js' 3 | 4 | /** 5 | * MySQL helper 6 | * @class 7 | */ 8 | export default class MySQL { 9 | /** 10 | * Constructor 11 | * @param {string} url - database url 12 | * @param {object} defs - syncing fields definitions 13 | */ 14 | constructor (url, defs) { 15 | this.url = url || 'mysql://localhost/test?user=root' 16 | this.defs = defs 17 | this.dbName = this.url.split(/\/|\?/)[3] 18 | this.con = null 19 | } 20 | 21 | /** 22 | * Insert the record 23 | * @param {object} def - definition of fields 24 | * @param {object} item - the data of the record to insert 25 | * @param {boolean} replaceFlag - set true to replace the record 26 | * @param {function} callback - callback 27 | */ 28 | insert (def, item, replaceFlag, callback) { 29 | if (typeof replaceFlag === 'function') { 30 | callback = replaceFlag 31 | replaceFlag = false 32 | } 33 | const command = replaceFlag ? 'REPLACE' : 'INSERT' 34 | const fs = def.fields.map(field => '`' + field.distName + '`') 35 | const vs = def.fields.map(field => field.convert(getFieldVal(field.name, item))) 36 | const sql = `${command} INTO \`${def.distName}\`` + 37 | ` (${fs.join(', ')}) VALUES (${vs.join(', ')});` 38 | const promise = this.query(sql) 39 | .catch(err => { 40 | logger.error(sql) 41 | throw err 42 | }) 43 | 44 | if (callback) promise.then(() => callback()) 45 | } 46 | 47 | /** 48 | * Update the record 49 | * @param {object} def - definition of fields 50 | * @param {string} id - the id of the record to update 51 | * @param {object} item - the columns to update 52 | * @param {object} unsetItems - the columns to drop 53 | * @param {function} callback - callback 54 | */ 55 | update (def, id, item, unsetItems, callback) { 56 | const fields = def.fields.filter(field => 57 | (!!item && typeof getFieldVal(field.name, item) !== 'undefined') || 58 | (!!unsetItems && typeof getFieldVal(field.name, unsetItems) !== 'undefined')) 59 | const sets = fields.map(field => { 60 | const val = field.convert(getFieldVal(field.name, item)) 61 | return `\`${field.distName}\` = ${val}` 62 | }) 63 | if (!sets.length) return 64 | 65 | const setsStr = sets.join(', ') 66 | const id2 = def.idType === 'number' ? id : `'${id}'` 67 | const sql = `UPDATE \`${def.distName}\` SET ${setsStr} WHERE ${def.idDistName} = ${id2};` 68 | const promise = this.query(sql) 69 | .catch(err => { 70 | logger.error(sql) 71 | throw err 72 | }) 73 | 74 | if (callback) promise.then(() => callback()) 75 | } 76 | 77 | /** 78 | * Remove the record 79 | * @param {object} def - definition of fields 80 | * @param {string} id - the id of the record to remove 81 | * @param {function} callback - callback 82 | */ 83 | remove (def, id, callback) { 84 | const id2 = def.idType === 'number' ? id : `'${id}'` 85 | const sql = `DELETE FROM \`${def.distName}\` WHERE ${def.idDistName} = ${id2};` 86 | const promise = this.query(sql) 87 | .catch(err => { 88 | logger.error(sql) 89 | throw err 90 | }) 91 | 92 | if (callback) promise.then(() => callback()) 93 | } 94 | 95 | /** 96 | * Create tables 97 | * @returns {Promise} with no value 98 | */ 99 | createTable () { 100 | // TODO: Create mongo_to_mysql table only if not exists 101 | const sql0 = 'DROP TABLE IF EXISTS mongo_to_mysql; ' + 102 | 'CREATE TABLE mongo_to_mysql (service varchar(20), timestamp BIGINT);' 103 | const sql1 = 'INSERT INTO mongo_to_mysql ' + 104 | `(service, timestamp) VALUES ("${this.dbName}", 0);` 105 | const sql2 = this.defs.map(def => { 106 | const fields = def.fields.map(field => 107 | `\`${field.distName}\` ${field.type}${field.primary ? ' PRIMARY KEY' : ''}`) 108 | return `DROP TABLE IF EXISTS \`${def.distName}\`; ` + 109 | `CREATE TABLE \`${def.distName}\` (${fields.join(', ')});` 110 | }).join('') 111 | 112 | return this.query(sql0) 113 | .then(() => this.query(sql1)) 114 | .then(() => this.query(sql2)) 115 | } 116 | 117 | /** 118 | * Read timestamp 119 | * @returns {Promise} with timestamp 120 | */ 121 | readTimestamp () { 122 | const q = `SELECT timestamp FROM mongo_to_mysql WHERE service = '${this.dbName}'` 123 | return this.query(q) 124 | .then(results => (results[0] && results[0].timestamp) || 0) 125 | .catch(err => { 126 | logger.error(q) 127 | throw err 128 | }) 129 | } 130 | 131 | /** 132 | * Update timestamp 133 | * @param {number} ts - a new timestamp 134 | */ 135 | updateTimestamp (ts) { 136 | const q = `UPDATE mongo_to_mysql SET timestamp = ${ts} WHERE service = '${this.dbName}';` 137 | this.getConnection() 138 | .query(q) 139 | } 140 | 141 | /** 142 | * Connect to MySQL 143 | * @returns {connection} MySQL connection 144 | */ 145 | getConnection () { 146 | if (this.con && 147 | this.con._socket && 148 | this.con._socket.readable && 149 | this.con._socket.writable) return this.con 150 | 151 | const params = 'multipleStatements=true' 152 | const url = this.url + (/\?/.test(this.url) ? '&' : '?') + params 153 | const con = mysql.createConnection(url) 154 | 155 | logger.info('Connect to MySQL...') 156 | con.connect(function (err) { 157 | if (err) logger.error(`SQL CONNECT ERROR: ${err}`) 158 | }) 159 | con.on('close', () => logger.info('SQL CONNECTION CLOSED.')) 160 | con.on('error', err => logger.error(`SQL CONNECTION ERROR: ${err}`)) 161 | 162 | return (this.con = con) 163 | } 164 | 165 | /** 166 | * Close the connection if exists 167 | */ 168 | clearConnection () { 169 | if (this.con) { 170 | this.con.end() 171 | this.con = null 172 | } 173 | } 174 | 175 | /** 176 | * Query method with promise 177 | * @param {string} sql - SQL string 178 | * @returns {Promise} with results 179 | */ 180 | query (sql) { 181 | return new Promise((resolve, reject) => { 182 | this.getConnection() 183 | .query(sql, (err, results) => { 184 | if (err) reject(err) 185 | else resolve(results) 186 | }) 187 | }) 188 | } 189 | } 190 | 191 | function getFieldVal (name, record) { 192 | return name.split('.').reduce((p, c) => p && p[c], record) 193 | } 194 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status][circle-image]][circle-url] 2 | [![NPM Status][npm-image]][npm-url] 3 | [![Codecov Status][codecov-image]][codecov-url] 4 | 5 | # Momy 6 | 7 | [Momy](https://goo.gl/maps/s9hXxKyoACv) is a simple cli tool for replicating MongoDB to MySQL in realtime. 8 | 9 | - Enable SQL query on data in NoSQL database 10 | - Enable to be accessed by Excel / Access 11 | 12 | ![Momy](images/concept.png) 13 | 14 | ## Installation 15 | 16 | Install via npm: 17 | 18 | ```bash 19 | $ npm install -g momy 20 | ``` 21 | 22 | Or use docker: 23 | 24 | ```bash 25 | $ docker run -it --rm -v $(pwd):/workdir cognitom/momy 26 | ``` 27 | 28 | You might want to create an alias, for example 29 | 30 | ```bash 31 | $ echo 'alias momy="docker run -it --rm -v $(pwd):/workdir cognitom/momy"' >> ~/.bashrc 32 | ``` 33 | 34 | See more detail about Docker configurations below. 35 | 36 | ## Preparation 37 | 38 | ### MongoDB 39 | 40 | Momy uses [Replica Set](http://docs.mongodb.org/manual/replication/) feature in MongoDB. But you don't have to replicate between MongoDB actually. Just follow the steps below. 41 | 42 | Start a new mongo instance with no data: 43 | 44 | ```bash 45 | $ mongod --replSet "rs0" --oplogSize 100 46 | ``` 47 | 48 | Open another terminal, and go to MongoDB Shell: 49 | 50 | ```bash 51 | $ mongo 52 | .... 53 | > rs.initiate() 54 | ``` 55 | 56 | `rs.initiate()` command prepare the collections that is needed for replication. 57 | 58 | ### MySQL 59 | 60 | Launch MySQL instance, and create the new database to use. The tables will be created or updated when syncing. You'll see `mongo_to_mysql`, too. This is needed to store the information for syncing. (don't remove it) 61 | 62 | ### Configuration 63 | 64 | Create a new `momyfile.json` file like this: 65 | 66 | ```json 67 | { 68 | "src": "mongodb://localhost:27017/dbname", 69 | "dist": "mysql://root:password@localhost:3306/dbname", 70 | "prefix": "t_", 71 | "case": "camel", 72 | "collections": { 73 | "collection1": { 74 | "_id": "number", 75 | "createdAt": "DATETIME", 76 | "field1": "number", 77 | "field2": "string", 78 | "field3": "boolean", 79 | "field4.subfield": "string" 80 | }, 81 | "collection2": { 82 | "_id": "string", 83 | "createdAt": "DATETIME", 84 | "field1": "number", 85 | "field2": "string", 86 | "field3": "boolean", 87 | "field4": "TEXT" 88 | } 89 | } 90 | } 91 | ``` 92 | 93 | - `src`: the URL of the MongoDB server 94 | - `dist`: the URL of the MySQL server 95 | - `prefix`: optional prefix for table name. The name of the table would be `t_collection1` in the example above. 96 | - `fieldCase`: optional. `snake` or `camel`. See the section below. 97 | - `exclusions`: optional. Chars or a range of chars to exclude: `"\uFFFD"` 98 | - `inclusions`: optional. Chars or a range of chars to include: `"\u0000-\u007F"` 99 | - `collections`: set the collections and fields to sync 100 | 101 | `_id` field is required for each collection and should be `string` or `number`. 102 | 103 | ### Field names and types 104 | 105 | ``` 106 | "": "" 107 | ``` 108 | or, field_name could be dot-concatenated: 109 | ``` 110 | ".": "" 111 | ``` 112 | 113 | For example, if you have `{ a: { b: { c: 'hey!' } } }` then `"a.b.c": "string"` 114 | 115 | Currently these native types are supported: 116 | 117 | - `BIGINT` 118 | - `TINYINT` 119 | - `VARCHAR` 120 | - `DATE` 121 | - `DATETIME` 122 | - `TIME` 123 | - `TEXT` 124 | 125 | There're also some aliases: 126 | 127 | - `number` => `BIGINT` 128 | - `boolean` => `TINYINT` 129 | - `string` => `VARCHAR` 130 | 131 | ### Field name normalization: fieldCase 132 | 133 | Some system like Microsoft Access don't allow *dot-concatenated field names*, so `address.street` will cause an error. For such a case, use `fieldCase`: 134 | 135 | - `snake`: `address.street` --> `address_street` 136 | - `camel`: `address.street` --> `addressStreet` 137 | 138 | **Note**: if you set `fieldCase` value, the name of `_id` field will change into `id` without `_`, too. 139 | 140 | ## Usage 141 | 142 | At the first run, we need to import all the data from MongoDB: 143 | 144 | ```bash 145 | $ momy --config momyfile.json --import 146 | ``` 147 | 148 | Then start the daemon to streaming data: 149 | 150 | ```bash 151 | $ momy --config momyfile.json 152 | ``` 153 | 154 | or 155 | 156 | ```bash 157 | $ forever momy --config momyfile.json 158 | ``` 159 | 160 | ## Usage with Docker 161 | 162 | First thing first, create a network for your containers: 163 | 164 | ```bash 165 | $ docker network create my-net 166 | ``` 167 | 168 | Then, launch database servers: 169 | 170 | ```bash 171 | $ docker run \ 172 | --name my-mongod \ 173 | --detach --rm \ 174 | --network my-net \ 175 | --mount type=volume,source=my-mongo-store,target=/data/db \ 176 | mongo --replSet "rs0" 177 | $ docker run \ 178 | --name my-mysqld \ 179 | --detach --rm \ 180 | --network my-net \ 181 | --mount type=volume,source=my-mysql-store,target=/var/lib/mysql \ 182 | --env MYSQL_ALLOW_EMPTY_PASSWORD=yes \ 183 | mysql 184 | ``` 185 | 186 | If this is the first time to run the containers above, you need to initialize them: 187 | 188 | ```bash 189 | $ docker exec my-mongod mongo --eval 'rs.initiate()' 190 | $ docker exec my-mysqld mysql -e 'CREATE DATABASE momy;' 191 | ``` 192 | 193 | Create `momyfile.json` like this: 194 | 195 | ```json 196 | { 197 | "src": "mongodb://my-mongod:27017/momy", 198 | "dist": "mysql://root@my-mysqld:3306/momy", 199 | "collections": {...} 200 | } 201 | ``` 202 | 203 | **Note**: you must change username, password, port, ...etc. to fit your environment. 204 | 205 | OK, let's run `momy` with `--import` option: 206 | 207 | ```bash 208 | $ docker run \ 209 | --interactive --tty --rm \ 210 | --network my-net \ 211 | --mount type=bind,source=$(pwd),target=/workdir \ 212 | cognitom/momy --import 213 | ``` 214 | 215 | Everything goes well? Then, stop the container (Ctrl + C). Now you can run it as a daemon: 216 | 217 | ```bash 218 | $ docker run \ 219 | --detach --rm \ 220 | --restart unless-stopped \ 221 | --init \ 222 | --network my-net \ 223 | --mount type=bind,source=$(pwd),target=/workdir \ 224 | cognitom/momy 225 | ``` 226 | 227 | ## For contributors 228 | 229 | See [dev](dev) directory. 230 | 231 | ## License 232 | 233 | MIT 234 | 235 | This library was originally made by @doubaokun as [MongoDB-to-MySQL](https://github.com/doubaokun/MongoDB-to-MySQL) and rewritten by @cognitom. 236 | 237 | [circle-image]:https://img.shields.io/circleci/project/github/cognitom/momy.svg?style=flat-square 238 | [circle-url]:https://circleci.com/gh/cognitom/momy 239 | [npm-image]:https://img.shields.io/npm/v/momy.svg?style=flat-square 240 | [npm-url]:https://www.npmjs.com/package/momy 241 | [codecov-image]:https://img.shields.io/codecov/c/github/cognitom/momy.svg?style=flat-square 242 | [codecov-url]:https://codecov.io/gh/cognitom/momy 243 | -------------------------------------------------------------------------------- /lib/tailer.js: -------------------------------------------------------------------------------- 1 | import { MongoClient, Timestamp } from 'mongodb' 2 | import { logger, getDbNameFromUri } from './util.js' 3 | import MySQL from './mysql.js' 4 | import { createDefs } from './defs.js' 5 | 6 | /** 7 | * Tailer 8 | * @class 9 | */ 10 | export default class Tailer { 11 | /** 12 | * Constructor 13 | * @param {object} config - configulation options 14 | * @param {boolean} cliMode - set false for testing 15 | */ 16 | constructor (config, cliMode) { 17 | const opts = { 18 | prefix: config.prefix || '', 19 | fieldCase: config.fieldCase || '', 20 | exclusions: config.exclusions || '', 21 | inclusions: config.inclusions || '' 22 | } 23 | this.cliMode = cliMode === undefined ? true : !!cliMode 24 | this.url = config.src || 'mongodb://localhost:27017/test' 25 | this.dbName = getDbNameFromUri(this.url) 26 | this.defs = createDefs(config.collections, this.dbName, opts) 27 | this.lastTs = 0 28 | this.mysql = new MySQL(config.dist, this.defs) 29 | this.client = null 30 | } 31 | 32 | /** 33 | * Start tailing 34 | * @param {boolean} forever - set false for testing 35 | */ 36 | start (forever) { 37 | this.client = new MongoClient(this.url) 38 | forever = forever === undefined ? true : !!forever 39 | this.mysql.readTimestamp() 40 | .then(ts => this.updateTimestamp(ts, true)) 41 | .then(() => forever ? this.tailForever() : this.tail()) 42 | .catch(err => this.stop(err)) 43 | } 44 | 45 | /** 46 | * Import all and start tailing 47 | * @param {boolean} forever - set false for testing 48 | */ 49 | importAndStart (forever) { 50 | this.client = new MongoClient(this.url) 51 | forever = forever === undefined ? true : !!forever 52 | this.mysql.createTable() 53 | .then(() => this.importAll()) 54 | .then(() => this.updateTimestamp()) 55 | .then(() => forever ? this.tailForever() : this.tail()) 56 | .catch(err => this.stop(err)) 57 | } 58 | 59 | stop (err) { 60 | this.mysql.clearConnection() 61 | this.client.close() 62 | 63 | if (!this.cliMode) return 64 | 65 | if (err) logger.error(err) 66 | logger.info('Bye') 67 | process.exit() 68 | } 69 | 70 | /** 71 | * Import all 72 | * @returns {Promise} with no value 73 | */ 74 | importAll () { 75 | logger.info('Begin to import...') 76 | let promise = Promise.resolve() 77 | this.defs.forEach(def => { 78 | promise = promise.then(() => this.importCollection(def)) 79 | }) 80 | promise.then(() => { 81 | logger.info('Done.') 82 | }) 83 | return promise 84 | } 85 | 86 | /** 87 | * Import collection 88 | * @param {object} def - definition of fields 89 | * @returns {Promise} with no value 90 | */ 91 | importCollection (def) { 92 | logger.info(`Import records in ${def.ns}`) 93 | return new Promise(resolve => { 94 | const db = this.client.db(this.dbName) 95 | const stream = db.collection(def.name).find().stream() 96 | stream 97 | .on('data', item => { 98 | stream.pause() 99 | this.mysql.insert(def, item, () => stream.resume()) 100 | }) 101 | .on('end', () => { 102 | resolve() 103 | }) 104 | }) 105 | } 106 | 107 | /** 108 | * Check the latest log in Mongo, then catch the timestamp up in MySQL 109 | * @param {number} ts - unless null then skip updating in MySQL 110 | * @param {boolean} skipUpdateMySQL - skip update in MySQL 111 | * @returns {Promise} with no value 112 | */ 113 | updateTimestamp (ts, skipUpdateMySQL) { 114 | if (ts) { 115 | this.lastTs = ts 116 | if (!skipUpdateMySQL) this.mysql.updateTimestamp(ts) 117 | return Promise.resolve() 118 | } 119 | return new Promise(resolve => { 120 | const db = this.client.db('local') 121 | db.collection('oplog.rs').find().sort({ $natural: -1 }).limit(1) 122 | .next() 123 | .then(item => { 124 | ts = item.ts.toNumber() 125 | this.lastTs = ts 126 | if (!skipUpdateMySQL) this.mysql.updateTimestamp(ts) 127 | resolve() 128 | }) 129 | }) 130 | } 131 | 132 | /** 133 | * Tail forever 134 | * @returns {Promise} with no value 135 | */ 136 | tailForever () { 137 | return new Promise((resolve, reject) => { 138 | let counter = 0 139 | let promise = Promise.resolve() 140 | const chainPromise = () => { 141 | promise = promise 142 | .then(() => { 143 | const message = counter++ 144 | ? 'Reconnect to MongoDB...' 145 | : 'Connect to MongoDB...' 146 | logger.info(message) 147 | return this.tail() 148 | }) 149 | .catch(err => reject(err)) 150 | .then(chainPromise) 151 | } 152 | chainPromise() 153 | }) 154 | } 155 | 156 | /** 157 | * Tail the log of Mongo by tailable cursors 158 | * @returns {Promise} with no value 159 | */ 160 | tail () { 161 | const ts = this.lastTs 162 | const nss = this.defs.map(def => def.ns) 163 | const filters = { 164 | ns: { $in: nss }, 165 | ts: { $gt: Timestamp.fromNumber(ts) } 166 | } 167 | const curOpts = { 168 | tailable: true, 169 | awaitdata: true, 170 | numberOfRetries: 60 * 60 * 24, // Number.MAX_VALUE, 171 | tailableRetryInterval: 1000 172 | } 173 | 174 | logger.info(`Begin to watch... (from ${ts})`) 175 | return new Promise((resolve, reject) => { 176 | const db = this.client.db('local') 177 | const stream = db.collection('oplog.rs').find(filters, curOpts).stream() 178 | stream 179 | .on('data', log => { 180 | if (log.op === 'n' || log.ts.toNumber() === ts) return 181 | this.process(log) 182 | }) 183 | .on('close', () => { 184 | logger.info('Stream closed....') 185 | resolve() 186 | }) 187 | .on('error', err => { 188 | reject(err) 189 | }) 190 | }) 191 | } 192 | 193 | /** 194 | * Process the log and sync to MySQL 195 | * @param {object} log - the log retrieved from oplog.rs 196 | * @returns {undefined} 197 | */ 198 | process (log) { 199 | const def = this.defs.filter(def => log.ns === def.ns)[0] 200 | if (!def) return 201 | 202 | this.updateTimestamp(log.ts.toNumber()) 203 | switch (log.op) { 204 | case 'i': 205 | logger.info(`Insert a new record into ${def.ns}`) 206 | return this.mysql.insert(def, log.o) 207 | case 'u': 208 | switch (log.o.$v) { 209 | case 2: // MongoDB 5.0 or newer 210 | if (log.o.diff && (log.o.diff.i || log.o.diff.u || log.o.diff.d)) { 211 | logger.info(`Update a record in ${def.ns} (${def.idName}=${log.o2[def.idName]})`) 212 | const toSet = Object.assign({}, log.o.diff.i, log.o.diff.u) 213 | const toUnset = log.o.diff.d 214 | return this.mysql.update(def, log.o2[def.idName], toSet, toUnset) 215 | } else { 216 | const replaceFlag = true 217 | logger.info(`Replace a record in ${def.ns} (${def.idName}=${log.o[def.idName]})`) 218 | return this.mysql.insert(def, log.o, replaceFlag) 219 | } 220 | default: // MongoDB 4.x or older 221 | if (log.o.$set || log.o.$unset) { 222 | logger.info(`Update a record in ${def.ns} (${def.idName}=${log.o2[def.idName]})`) 223 | return this.mysql.update(def, log.o2[def.idName], log.o.$set, log.o.$unset) 224 | } else { 225 | const replaceFlag = true 226 | logger.info(`Replace a record in ${def.ns} (${def.idName}=${log.o[def.idName]})`) 227 | return this.mysql.insert(def, log.o, replaceFlag) 228 | } 229 | } 230 | case 'd': 231 | logger.info(`Delete a record in ${def.ns} (${def.idName}=${log.o[def.idName]})`) 232 | return this.mysql.remove(def, log.o[def.idName]) 233 | } 234 | } 235 | } 236 | -------------------------------------------------------------------------------- /test/specs/core.js: -------------------------------------------------------------------------------- 1 | /* eslint-env mocha */ 2 | 3 | import assert from 'assert' 4 | import fs from 'fs' 5 | import moment from 'moment' 6 | import { MongoClient } from 'mongodb' 7 | import wait from '../wait.js' 8 | 9 | import Tailer from '../../lib/tailer.js' 10 | import { MysqlConnector } from '../mysql-connector.js' 11 | import { getDbNameFromUri } from '../../lib/util.js' 12 | 13 | const momyfile = JSON.parse(fs.readFileSync('./test/momyfile.json')) 14 | const waitingTime = 500 15 | 16 | describe('Momy: core', () => { 17 | let mo 18 | let my 19 | let client 20 | let tailer 21 | 22 | before(async function () { 23 | const dbname = getDbNameFromUri(momyfile.src) 24 | client = new MongoClient(momyfile.src) 25 | mo = client.db(dbname) 26 | // clear existing records 27 | await mo.collection('colBasicTypes').deleteMany({}) 28 | await mo.collection('colNumberTypes').deleteMany({}) 29 | await mo.collection('colDateTypes').deleteMany({}) 30 | await mo.collection('colStringTypes').deleteMany({}) 31 | 32 | my = new MysqlConnector(momyfile.dist) 33 | tailer = new Tailer(momyfile, false) 34 | tailer.importAndStart(false) 35 | await wait(1000) // wait for syncing 36 | }) 37 | 38 | it('syncs a single doc with basic types', async function () { 39 | const colName = 'colBasicTypes' 40 | const doc = { 41 | field1: true, // boolean 42 | field2: 123, // number 43 | field3: 'Tom' // string 44 | } 45 | const r0 = await mo.collection(colName).insertOne(doc) 46 | await wait(waitingTime) // wait for syncing 47 | const r1 = await my.query(`SELECT * FROM ${colName} WHERE _id = "${r0.insertedId}"`) 48 | 49 | assert.equal(r1[0].field1, 1) 50 | assert.equal(r1[0].field2, doc.field2) 51 | assert.equal(r1[0].field3, doc.field3) 52 | }) 53 | 54 | it('syncs a single doc with number types', async function () { 55 | const colName = 'colNumberTypes' 56 | const doc = { 57 | field1: 1234567, // BIGINT 58 | field2: 123.4567, // DOUBLE 59 | field3: true // TINYINT 60 | } 61 | const r0 = await mo.collection(colName).insertOne(doc) 62 | await wait(waitingTime) // wait for syncing 63 | const r1 = await my.query(`SELECT * FROM ${colName} WHERE _id = "${r0.insertedId}"`) 64 | 65 | assert.equal(r1[0].field1, 1234567) 66 | assert.equal(r1[0].field2, 123.4567) 67 | assert.equal(r1[0].field3, 1) 68 | }) 69 | 70 | it('syncs a single doc with date types', async function () { 71 | const colName = 'colDateTypes' 72 | const now = Date.now() 73 | const doc = { 74 | field1: now, // DATE 75 | field2: now, // DATETIME 76 | field3: now // TIME 77 | } 78 | const r0 = await mo.collection(colName).insertOne(doc) 79 | await wait(waitingTime) // wait for syncing 80 | const r1 = await my.query(`SELECT * FROM ${colName} WHERE _id = "${r0.insertedId}"`) 81 | 82 | assert.equal( 83 | moment(r1[0].field1).format('YYYY-MM-DD'), 84 | moment(now).format('YYYY-MM-DD')) 85 | assert.equal( 86 | moment(r1[0].field2).format('YYYY-MM-DD HH:mm:ss'), 87 | moment(now).format('YYYY-MM-DD HH:mm:ss')) 88 | assert.equal(r1[0].field3, moment(now).format('HH:mm:ss')) 89 | }) 90 | 91 | it('syncs a single doc with date types (object edition) #15', async function () { 92 | const colName = 'colDateTypes' 93 | const now = new Date() 94 | const doc = { 95 | field1: now, // DATE 96 | field2: now, // DATETIME 97 | field3: now // TIME 98 | } 99 | const r0 = await mo.collection(colName).insertOne(doc) 100 | await wait(waitingTime) // wait for syncing 101 | const r1 = await my.query(`SELECT * FROM ${colName} WHERE _id = "${r0.insertedId}"`) 102 | 103 | assert.equal( 104 | moment(r1[0].field1).format('YYYY-MM-DD'), 105 | moment(now).format('YYYY-MM-DD')) 106 | assert.equal( 107 | moment(r1[0].field2).format('YYYY-MM-DD HH:mm:ss'), 108 | moment(now).format('YYYY-MM-DD HH:mm:ss')) 109 | assert.equal(r1[0].field3, moment(now).format('HH:mm:ss')) 110 | }) 111 | 112 | it('syncs a single doc with string types', async function () { 113 | const colName = 'colStringTypes' 114 | const allAscii = Array.from(Array(95)).map((_, i) => String.fromCharCode(32 + i)).join('') 115 | const string285 = allAscii + allAscii + allAscii 116 | const doc = { 117 | field1: string285, // VARCHAR 118 | field2: string285 // TEXT 119 | } 120 | const r0 = await mo.collection(colName).insertOne(doc) 121 | await wait(waitingTime) // wait for syncing 122 | const r1 = await my.query(`SELECT * FROM ${colName} WHERE _id = "${r0.insertedId}"`) 123 | 124 | assert.equal(r1[0].field1, string285.substring(0, 255)) 125 | assert.equal(r1[0].field2, string285) 126 | }) 127 | 128 | it('syncs a doc updated', async function () { 129 | const colName = 'colBasicTypes' 130 | const doc = { 131 | field1: true, // boolean 132 | field2: 123, // number 133 | field3: 'Tom' // string 134 | } 135 | const r0 = await mo.collection(colName).insertOne(doc) 136 | await wait(waitingTime) // wait for syncing 137 | const r1 = await my.query(`SELECT * FROM ${colName} WHERE _id = "${r0.insertedId}"`) 138 | assert.equal(r1[0].field3, 'Tom') 139 | 140 | await mo.collection(colName).updateOne({ _id: r0.insertedId }, { $set: { field3: 'John' } }) 141 | await wait(waitingTime) // wait for syncing 142 | const r2 = await my.query(`SELECT * FROM ${colName} WHERE _id = "${r0.insertedId}"`) 143 | assert.equal(r2[0].field3, 'John') 144 | 145 | await mo.collection(colName).updateOne({ _id: r0.insertedId }, { $unset: { field3: true } }) 146 | await wait(waitingTime) // wait for syncing 147 | const r3 = await my.query(`SELECT * FROM ${colName} WHERE _id = "${r0.insertedId}"`) 148 | assert.equal(r3[0].field3, '') 149 | 150 | await mo.collection(colName).replaceOne({ _id: r0.insertedId }, doc) 151 | await wait(waitingTime) // wait for syncing 152 | const r4 = await my.query(`SELECT * FROM ${colName} WHERE _id = "${r0.insertedId}"`) 153 | assert.equal(r4[0].field3, 'Tom') 154 | }) 155 | 156 | it('syncs a doc without a specific field which is added later', async function () { 157 | const colName = 'colBasicTypes' 158 | const doc = { 159 | field1: true, // boolean 160 | field2: 123 // number 161 | } 162 | const r0 = await mo.collection(colName).insertOne(doc) 163 | await wait(waitingTime) // wait for syncing 164 | const r1 = await my.query(`SELECT * FROM ${colName} WHERE _id = "${r0.insertedId}"`) 165 | console.log(r1[0].field3) 166 | 167 | await mo.collection(colName).updateOne({ _id: r0.insertedId }, { $set: { field3: 'John' } }) 168 | await wait(waitingTime) // wait for syncing 169 | const r2 = await my.query(`SELECT * FROM ${colName} WHERE _id = "${r0.insertedId}"`) 170 | assert.equal(r2[0].field3, 'John') 171 | }) 172 | 173 | it('inserts and remove a doc', async function () { 174 | const colName = 'colBasicTypes' 175 | const doc = { 176 | field1: true, // boolean 177 | field2: 123, // number 178 | field3: 'Tom' // string 179 | } 180 | const r0 = await mo.collection(colName).insertOne(doc) 181 | await wait(waitingTime) // wait for syncing 182 | const r1 = await my.query(`SELECT * FROM ${colName} WHERE _id = "${r0.insertedId}"`) 183 | assert.equal(r1[0].field3, 'Tom') 184 | 185 | await mo.collection(colName).deleteOne({ _id: r0.insertedId }) 186 | await wait(waitingTime) // wait for syncing 187 | const r2 = await my.query(`SELECT * FROM ${colName} WHERE _id = "${r0.insertedId}"`) 188 | assert.equal(r2.length, 0) 189 | }) 190 | 191 | it('inserts multiple docs', async function () { 192 | const colName = 'colBasicTypes' 193 | const docs = Array.from(Array(10)).map((_, i) => ({ 194 | field1: true, 195 | field2: i, 196 | field3: `Tom-${i}` 197 | })) 198 | for (const doc of docs) { 199 | const r = await mo.collection(colName).insertOne(doc) 200 | doc._id = r.insertedId 201 | } 202 | await wait(waitingTime) // wait for syncing 203 | for (const doc of docs) { 204 | const r = await my.query(`SELECT * FROM ${colName} WHERE _id = "${doc._id}"`) 205 | assert.equal(r[0].field2, doc.field2) 206 | } 207 | }) 208 | 209 | it('inserts 100 docs', async function () { 210 | const colName = 'colBasicTypes' 211 | const docs = Array.from(Array(100)).map((_, i) => ({ 212 | field1: true, 213 | field2: i, 214 | field3: `Tom-${i}` 215 | })) 216 | for (const doc of docs) { 217 | const r = await mo.collection(colName).insertOne(doc) 218 | doc._id = r.insertedId 219 | } 220 | await wait(waitingTime * 10) // wait for syncing 221 | for (const doc of docs) { 222 | const r = await my.query(`SELECT * FROM ${colName} WHERE _id = "${doc._id}"`) 223 | assert.equal(r[0].field2, doc.field2) 224 | } 225 | }) 226 | 227 | after(async function () { 228 | tailer.stop() 229 | my.close() 230 | client.close() 231 | await wait(waitingTime * 2) 232 | }) 233 | }) 234 | --------------------------------------------------------------------------------