├── .dockerignore ├── .eslintrc.js ├── .gitignore ├── Dockerfile ├── config └── default.json ├── controllers ├── ali.oss.store.js ├── download.js ├── index.js └── upload.js ├── dump-data.sh ├── encrypt-data.sh ├── package.json ├── readme.md ├── run.sh └── server.js /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | DATA -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | node: true, 4 | browser: true, 5 | es6: true 6 | }, 7 | extends: "eslint:recommended", 8 | parser: "babel-eslint", 9 | rules: { 10 | "no-console": "off", 11 | "no-useless-escape": "off", 12 | quotes: ["error", "double"], 13 | semi: ["error", "always"] 14 | } 15 | }; 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .DS_Store 3 | package-lock.json 4 | DATA -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM daocloud.io/library/ubuntu:14.04 2 | 3 | MAINTAINER Fundebug 4 | 5 | # 设置时区 6 | RUN sh -c "echo 'Asia/Shanghai' > /etc/timezone" && \ 7 | dpkg-reconfigure -f noninteractive tzdata 8 | 9 | 10 | RUN echo '\n\ 11 | deb http://mirrors.aliyun.com/ubuntu/ trusty main restricted universe multiverse\n\ 12 | deb http://mirrors.aliyun.com/ubuntu/ trusty-security main restricted universe multiverse\n\ 13 | deb http://mirrors.aliyun.com/ubuntu/ trusty-updates main restricted universe multiverse\n\ 14 | deb http://mirrors.aliyun.com/ubuntu/ trusty-proposed main restricted universe multiverse\n\ 15 | deb http://mirrors.aliyun.com/ubuntu/ trusty-backports main restricted universe multiverse\n\ 16 | deb-src http://mirrors.aliyun.com/ubuntu/ trusty main restricted universe multiverse\n\ 17 | deb-src http://mirrors.aliyun.com/ubuntu/ trusty-security main restricted universe multiverse\n\ 18 | deb-src http://mirrors.aliyun.com/ubuntu/ trusty-updates main restricted universe multiverse\n\ 19 | deb-src http://mirrors.aliyun.com/ubuntu/ trusty-proposed main restricted universe multiverse\n\ 20 | deb-src http://mirrors.aliyun.com/ubuntu/ trusty-backports main restricted universe multiverse\n'\ 21 | > /etc/apt/sources.list 22 | 23 | RUN sudo apt-get update && \ 24 | sudo apt-get install -y wget 25 | 26 | # 安装node v8.9.1 27 | RUN wget https://npm.taobao.org/mirrors/node/v8.9.1/node-v8.9.1-linux-x64.tar.gz && \ 28 | tar -C /usr/local --strip-components 1 -xzf node-v8.9.1-linux-x64.tar.gz && \ 29 | rm node-v8.9.1-linux-x64.tar.gz 30 | 31 | WORKDIR /app 32 | 33 | # 安装npm模块 34 | ADD package.json /app/package.json 35 | RUN npm install --production -d --registry=https://registry.npm.taobao.org 36 | 37 | COPY . /app 38 | 39 | CMD ["node", "/app/server.js"] 40 | -------------------------------------------------------------------------------- /config/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "port": "9160" 3 | } 4 | -------------------------------------------------------------------------------- /controllers/ali.oss.store.js: -------------------------------------------------------------------------------- 1 | const OSS = require("ali-oss"); 2 | module.exports = OSS({ 3 | accessKeyId: "", 4 | accessKeySecret: "", 5 | bucket: "fundebug-shenzhen", 6 | region: "oss-cn-shenzhen", 7 | secure: true 8 | }); 9 | -------------------------------------------------------------------------------- /controllers/download.js: -------------------------------------------------------------------------------- 1 | const Promise = require("bluebird"); 2 | const moment = require("moment"); 3 | const request = require("request"); 4 | Promise.promisifyAll(request); 5 | const fs = require("fs"); 6 | var log4js = require("log4js"); 7 | var logger = log4js.getLogger(); 8 | const store = require("./ali.oss.store.js"); 9 | const fundebug = require("fundebug-nodejs"); 10 | 11 | exports.downloadDataToAliOSS = async function(req, res, next) { 12 | try { 13 | const day = moment().format("YYYYMMDD"); 14 | const dir = moment().format("YYYYMMDDHHmmss"); 15 | const path = `./DATA/${dir}`; 16 | fs.mkdirSync(path); 17 | const files = await listFilesToDownload(day); 18 | await Promise.mapSeries(files, async file => { 19 | logger.info(`start to download file: ${file.name}`); 20 | await downloadFile(file.name, path); 21 | logger.info(`download file success! ${file.name}`); 22 | }); 23 | logger.info("download all file success!"); 24 | res.sendStatus(200); 25 | } catch (error) { 26 | next(error); 27 | } 28 | }; 29 | 30 | // 获取当天上传到阿里OSS的文件列表 31 | async function listFilesToDownload(day) { 32 | const result = await store.list({ prefix: day }); 33 | return result.objects; 34 | } 35 | 36 | // 将阿里云OSS中的文件下载到本地 37 | async function downloadFile(fileName, path) { 38 | try { 39 | const file = fileName.split("/")[1]; 40 | const filepath = `${path}/${file}`; 41 | await store.get(fileName, filepath); 42 | } catch (error) { 43 | const message = `download file fail! ${fileName}`; 44 | logger.error(message); 45 | logger.error(error); 46 | fundebug.notifyError(error, { 47 | metaData: { 48 | error: error, 49 | message: message 50 | } 51 | }); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /controllers/index.js: -------------------------------------------------------------------------------- 1 | const express = require("express"); 2 | const config = require("config"); 3 | const morgan = require("morgan"); 4 | const log4js = require("log4js"); 5 | const logger = log4js.getLogger(); 6 | const Upload = require("./upload.js"); 7 | const Download = require("./download.js"); 8 | 9 | // 使用Fundebug监控报错：https://docs.fundebug.com/notifier/nodejs/ 10 | const fundebug = require("fundebug-nodejs"); 11 | fundebug.apikey = "API-KEY"; 12 | 13 | const port = config.get("port"); 14 | 15 | var app = express(); 16 | var router = new express.Router(); 17 | 18 | app.use( 19 | morgan("short", { 20 | stream: { 21 | write: function(str) { 22 | logger.debug(str); 23 | } 24 | } 25 | }) 26 | ); 27 | 28 | router.get("/upload", Upload.uploadDataToAliOSS); 29 | router.get("/download", Download.downloadDataToAliOSS); 30 | 31 | // API不存在时返回404 32 | router.use("/*", function(req, res, next) { 33 | var err = new Error("API Not Found"); 34 | err.status = 404; 35 | next(err); 36 | }); 37 | 38 | app.use(router); 39 | 40 | // 生产环境和开发环境错误处理 41 | app.use(function(err, req, res, next) { 42 | logger.error(err); 43 | res.status(err.status || 500); 44 | res.json({ 45 | error: err.message, 46 | code: err.code 47 | }); 48 | next(err); 49 | }); 50 | 51 | // Fundebug错误监控插件应该放在其他中间件之后，并放在其他错误处理中间件之后 52 | app.use(fundebug.ExpressErrorHandler); 53 | 54 | app.listen(port, function() { 55 | logger.info( 56 | `mongodb-backup listen on port ${port}, in ${process.env.NODE_ENV} mode` 57 | ); 58 | }); 59 | 60 | module.exports = app; 61 | -------------------------------------------------------------------------------- /controllers/upload.js: -------------------------------------------------------------------------------- 1 | const Promise = require("bluebird"); 2 | const fs = require("fs"); 3 | var log4js = require("log4js"); 4 | var logger = log4js.getLogger(); 5 | const store = require("./ali.oss.store.js"); 6 | const fundebug = require("fundebug-nodejs"); 7 | 8 | exports.uploadDataToAliOSS = async function(req, res, next) { 9 | try { 10 | const backup_path = "/data/mongodb_backup"; 11 | const directory = fs.readdirSync(backup_path)[0]; 12 | const files = getFilesToUpload(backup_path, directory); 13 | logger.info(files); 14 | // 逐一上传每个文件 15 | await Promise.mapSeries(files, async file => { 16 | logger.info(`start to upload ${file}`); 17 | await uploadFile( 18 | `${directory}/${file}`, 19 | `${backup_path}/${directory}/fundebug-production/${file}` 20 | ); 21 | }); 22 | logger.info("upload all files success!\n\n"); 23 | res.sendStatus(200); 24 | } catch (error) { 25 | next(error); 26 | } 27 | }; 28 | 29 | // 上传单个文件 30 | async function uploadFile(fileName, filePath) { 31 | try { 32 | const result = await store.multipartUpload(fileName, filePath, { 33 | parallel: 4, 34 | partSize: 1024 * 1024, 35 | progress: function(p) { 36 | logger.info("Progress: " + p); 37 | } 38 | }); 39 | if (result.res.statusCode === 200) { 40 | logger.info(`upload file success! ${fileName}`); 41 | } else { 42 | const message = `upload file fail! ${fileName}`; 43 | logger.error(message); 44 | logger.error(result); 45 | fundebug.notifyError(new Error(message), { 46 | metaData: { 47 | message: message, 48 | result: result 49 | } 50 | }); 51 | } 52 | } catch (error) { 53 | const message = `upload file fail! ${fileName}`; 54 | logger.error(message); 55 | logger.error(error); 56 | fundebug.notifyError(error, { 57 | metaData: { 58 | message: message, 59 | error: error 60 | } 61 | }); 62 | } 63 | } 64 | 65 | function getFilesToUpload(backup_path, directory) { 66 | let files = fs.readdirSync( 67 | `${backup_path}/${directory}/fundebug-production` 68 | ); 69 | 70 | // 只上传加密的.gpg文档 71 | files = files.filter(file => { 72 | return file.endsWith(".gpg"); 73 | }); 74 | 75 | return files; 76 | } 77 | -------------------------------------------------------------------------------- /dump-data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 删除前一天导出的数据 4 | rm -rf /data/mongodb_backup 5 | 6 | DIR=`date +%Y%m%d%H%M` 7 | OUT=/data/mongodb_backup/$DIR 8 | mkdir -p $DEST 9 | 10 | # 全量导出MongoDB数据(排除部分集合) 11 | mongodump --host "rs0/192.168.59.11:27017,192.168.59.12:27017,192.168.59.13:27017" \ 12 | --db fundebug-production \ 13 | --excludeCollection events \ 14 | --out $OUT 15 | -------------------------------------------------------------------------------- /encrypt-data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR=`find /data/mongodb_backup/ -maxdepth 1 -type d ! -path /data/mongodb_backup/` 4 | source=$DIR/fundebug-production 5 | cd $source 6 | 7 | # 将导出数据加密 8 | for file in * ; do 9 | gpg --batch --yes -v -e -r fundebug --output $source/$file.gpg --always-trust $file 10 | done ; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "fundebug-mongodb-backup", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "app.js", 6 | "scripts": { 7 | "start": "supervisor -w server.js,controllers,config -s -RV server.js", 8 | "test": "echo \"Error: no test specified\" && exit 1" 9 | }, 10 | "author": "", 11 | "license": "ISC", 12 | "dependencies": { 13 | "ali-oss": "^6.0.1", 14 | "bluebird": "^3.5.0", 15 | "config": "^2.0.1", 16 | "express": "^4.16.3", 17 | "fundebug-nodejs": "^0.0.8", 18 | "log4js": "^1.1.1", 19 | "moment": "^2.22.2", 20 | "morgan": "^1.9.1", 21 | "request": "^2.88.0" 22 | }, 23 | "devDependencies": { 24 | "babel-eslint": "^9.0.0" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | ## Fundebug 是这样备份数据的 2 | 3 | ### 从服务器上传备份数据到阿里 OSS 4 | 5 | #### 步骤 6 | 7 | - 使用 mongodump 导出 mongodb 数据 8 | - 使用 gpg 加密导出的数据 9 | - 将加密数据上传到阿里云对象存储服务 10 | 11 | #### 通过 crontab 定时执行 12 | 13 | ``` 14 | # 每天凌晨4点从服务器上传备份数据到阿里OSS 15 | 00 4 * * * /root/mongodb-backup/dump-data.sh && /root/mongodb-backup/encrypt-data.sh && docker restart mongodb-backup && sleep 1m && curl http://127.0.0.1:9160/upload 16 | ``` 17 | 18 | ### 从阿里云 OSS 将备份数据下载到本地 19 | 20 | #### 通过 crontab 定时执行 21 | 22 | ``` 23 | # 每周六12点从阿里云OSS将备份数据下载到本地 24 | 0 12 * * 6 curl http://127.0.0.1:9160/download 25 | ``` 26 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | # 构建mongodb-backup镜像 2 | echo "\nbuild fundebug/mongodb-backup image..." 3 | sudo docker build -t fundebug/mongodb-backup . 4 | 5 | 6 | # 运行mongodb-backup容器 7 | sudo docker rm -f mongodb-backup > /dev/null 8 | echo "\nstart mongodb-backup container..." 9 | sudo docker run -d \ 10 | --net=host \ 11 | --name=mongodb-backup \ 12 | -v `pwd`/DATA:/app/DATA \ 13 | -v /data/mongodb_backup:/data/mongodb_backup \ 14 | fundebug/mongodb-backup > /dev/null 15 | 16 | echo "" 17 | -------------------------------------------------------------------------------- /server.js: -------------------------------------------------------------------------------- 1 | var fundebug = require("fundebug-nodejs"); 2 | fundebug.apikey = 3 | "a73f74761b778c666f16980408bbd6cbdae9f222667e9648ba6181686de7aeba"; 4 | 5 | // 默认ENV为"development" 6 | if (!process.env.NODE_ENV) { 7 | process.env.NODE_ENV = "development"; 8 | } 9 | 10 | require("./controllers"); 11 | --------------------------------------------------------------------------------