├── .babelrc ├── .gitignore ├── README.md ├── app.js ├── config └── development.js ├── db ├── mongodb.js ├── mysql.js └── redis.js ├── docs ├── SQL │ └── shin_backend.sql └── assets │ ├── 1.png │ ├── architecture.odg │ ├── architecture.png │ ├── shin.odg │ └── shin.png ├── index-worker.js ├── index.js ├── init.js ├── middlewares ├── checkAuth.js ├── checkExport.js ├── errorHandle.js └── index.js ├── models ├── AppGlobalConfig.js ├── BackendUserAccount.js ├── BackendUserRole.js ├── WebMonitor.js ├── WebMonitorRecord.js ├── WebMonitorStatis.js ├── WebPerformance.js ├── WebPerformanceProject.js ├── WebPerformanceStatis.js ├── WebShortChain.js └── index.js ├── package.json ├── public └── blank.gif ├── routers ├── common.js ├── index.js ├── template.js ├── tool.js ├── user.js └── webMonitor.js ├── scripts ├── demo.js └── index.js ├── services ├── backendUserAccount.js ├── backendUserRole.js ├── common.js ├── index.js ├── tool.js └── webMonitor.js ├── static └── img │ ├── avatar.png │ └── cover.jpg ├── test ├── index.js ├── mocha.opts ├── routers │ └── user.js ├── services │ └── user.js └── utils │ └── tool.js ├── utils ├── constant.js ├── index.js ├── murmurhash.js ├── queue.js ├── tools.js └── xfetch.js └── worker ├── agenda.js ├── cronJobs ├── demo.js ├── webMonitorRemove.js ├── webMonitorStatis.js ├── webMonitorWarn.js ├── webPerformanceRemove.js ├── webPerformanceStatis-func.js └── webPerformanceStatis.js └── triggerJobs └── demo.js /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | "env" 4 | ], 5 | "plugins": [ 6 | "transform-object-rest-spread" 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Mac OS 排序文件 2 | .DS_Store 3 | 4 | # vscode 配置目录 5 | .vscode 6 | 7 | # dependencies 8 | /node_modules 9 | /package-lock.json 10 | 11 | 12 | # 本地的配置文件 13 | # config/development.js 14 | 15 | # 生成的目录 16 | dist/ 17 | upload/ 18 | static/upload/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # shin-server 2 |   shin 的读音是[ʃɪn]，谐音就是行，寓意可行的后端系统服务，它的特点是： 3 | 4 | * 站在巨人的肩膀上，依托[KOA2](https://github.com/demopark/koa-docs-Zh-CN)、[bunyan](https://github.com/trentm/node-bunyan)、[Sequelize](https://www.sequelize.com.cn/)等优秀的框架和库所搭建的定制化后端系统服务。 5 | * 一套完整的 Node.js 后端服务解决方案。 6 | * 调试便捷，实时打印出各类请求、日志和所有的查询语句。 7 | * 配合独立的配置文件可连接 MongoDB、MySQL 以及 Redis。 8 | * 已开辟脚本和定时任务目录，可将相应文件补充进来。 9 | * 容易扩展，可引入第三方库，例如队列、云服务等。 10 | 11 |   与[shin-admin](https://github.com/pwstrick/shin-admin)配合使用的话，大致架构如下图。 12 | 13 | ![架构](https://github.com/pwstrick/shin-server/blob/main/docs/assets/architecture.png) 14 | 15 | # 准备工作 16 | #### 1）安装 17 |   在将项目下载下来后，来到其根目录，运行安装命令，自动将依赖包下载到本地。 18 | ```bash 19 | $ npm install 20 | ``` 21 | 22 | #### 2）启动 23 |   在启动服务器之前，需要确保本地已经安装并已开启 MongoDB、MySQL 以及 Redis。 24 | * mongo 启动命令：mongod 25 | * redis 启动命令：redis-server 26 | 27 |   在 docs/SQL 中有个数据库文件，可初始化所需的表，并且注意将 config/development.js 中数据库的账号和密码修改成本机的。 28 | 29 |   执行项目启动命令，成功后的终端如下图所示，端口号默认是 6060，可在 config/development.js 中自定义端口号。 30 | ```bash 31 | $ npm start 32 | ``` 33 | ![启动](https://github.com/pwstrick/shin-server/blob/main/docs/assets/1.png) 34 | 35 |   运行 http://localhost:6060/user/init 可初始化超级管理员账号，后台账号和权限都保存在 MongoDB 中，其他一些业务保存在 MySQL 中。 36 | * 账号：admin@shin.com 37 | * 密码：admin 38 | 39 | #### 3）运行流程 40 |   当向这套后端系统服务请求一个接口时，其大致流程如下图所示。 41 | 42 |

43 | 44 |

45 | 46 | # 目录结构 47 | ``` 48 | ├── shin-server 49 | │ ├── config --------------------------------- 全局配置文件 50 | │ ├── db ------------------------------------- 数据库连接 51 | │ ├── docs ----------------------------------- 说明文档 52 | │ ├── middlewares ---------------------------- 自定义的中间件 53 | │ ├── models --------------------------------- 数据表映射 54 | │ ├── routers -------------------------------- api 路由层 55 | │ ├── scripts -------------------------------- 脚本文件 56 | │ ├── services ------------------------------- api 服务层 57 | │ ├── static --------------------------------- 静态资源 58 | │ ├── test ----------------------------------- 单元测试 59 | │ ├── utils ---------------------------------- 公用工具 60 | │ ├── worker --------------------------------- 定时任务 61 | │ ├── app.js --------------------------------- 启动文件 62 | │ ├── index.js ------------------------------- 入口文件 63 | └───└── index-worker.js ------------------------ 任务的入口文件 64 | ``` 65 | 66 | #### 1）app.js 67 |   在启动文件中，初始化了 [bunyan](https://github.com/trentm/node-bunyan) 日志框架，并声明了一个全局的 logger 变量（可调用的方法包括 info、error、warn、debug等） ，可随时写日志，所有的请求信息（引入了[koa-bunyan-logger](https://github.com/koajs/bunyan-logger)）、数据库查询语句、响应数据等，都会写入到服务器的日志中。 68 | 69 |   [JWT](https://jwt.io/) 认证 HTTP 请求，引入了 koa-jwt 中间件，会在 checkAuth.js 中间件（如下代码所示）中调用 ctx.state.user，以此来判断权限。而判断当前是否是登录会以 GET 方式向 ”api/user“ 接口发送一次请求。 70 | 71 |   还引入了 routers() 函数（位于 routers 目录的 index.js 中），将 services 和 middlewares 两个目录下的文件作为参数传入，这两个目录下都包含 index.js 文件，引用方式为 middlewares.checkAuth()、 services.android 等。 72 | ```javascript 73 | import requireIndex from 'es6-requireindex'; 74 | import services from '../services/'; 75 | import middlewares from '../middlewares'; 76 | 77 | export default (router) => { 78 | const dir = requireIndex(__dirname); 79 | Object.keys(dir).forEach((item) => { 80 | dir[item](router, services, middlewares); 81 | }); 82 | }; 83 | ``` 84 | 85 | #### 2）config 86 |   默认只包含 development.js，即开发环境的配置文件，可包含数据库的地址、各类账号密码等。 87 | 88 |   使用[node-config](https://lorenwest.github.io/node-config/)后，就能根据当前环境（NODE_ENV）调用相应名称的配置文件，例如 production.js、test.js 等。 89 | 90 | #### 3）db 91 |   MySQL 数据库 ORM 系统采用的是 [Sequelize](https://www.sequelize.com.cn/)，MongoDB 数据库 ORM系统采用的是 [Mongoose](http://www.mongoosejs.net/docs/guide.html)，redis 库采用的是 [ioredis](https://github.com/luin/ioredis/blob/master/API.md)。 92 | 93 | #### 4）models 94 |   声明各张表的结构，可用驼峰，也可用下划线的命名方式，函数的参数为 mysql 或 mongodb，可通过 mysql.backend 来指定要使用的数据库名称。 95 | ```javascript 96 | export default ({ mysql }) => 97 | mysql.backend.define("AppGlobalConfig", 98 | { 99 | id: { 100 | type: Sequelize.INTEGER, 101 | field: "id", 102 | autoIncrement: true, 103 | primaryKey: true 104 | }, 105 | title: { 106 | type: Sequelize.STRING, 107 | field: "title" 108 | }, 109 | }, 110 | { 111 | tableName: "app_global_config", 112 | timestamps: false 113 | } 114 | ); 115 | ``` 116 |   models 目录中的 index.js 文件会将当前所有的 model 文件映射到一个 models 对象中。 117 | 118 | #### 5）routers 119 |   前端访问的接口，在此目录下声明，此处代码相当于 MVC 中的 Control 层。 120 | 121 |   在下面的示例中，完成了一次 GET 请求，middlewares.checkAuth()用于检查权限，其值就是在 [authority.js](https://github.com/pwstrick/shin-admin#9authorityjs) 声明的 id，ctx.body 会返回响应。 122 | ```javascript 123 | router.get( 124 | "/tool/short/query", 125 | middlewares.checkAuth("backend.tool.shortChain"), 126 | async (ctx) => { 127 | const { curPage = 1, short, url } = ctx.query; 128 | const { rows, count } = await services.tool.getShortChainList({ 129 | curPage, 130 | short, 131 | url 132 | }); 133 | ctx.body = { code: 0, data: rows, count }; 134 | } 135 | ); 136 | ``` 137 | 138 | #### 6）services 139 |   处理数据，包括读写数据表、调用后端服务、读写缓存等。 140 | 141 |   services 目录中的 index.js 文件会初始化各个 service 文件，并将之前的 models 对象作为参数传入。 142 | 143 |   注意，MySQL中查询数据返回值中会包含各种信息，如果只要表的数据需要在查询条件中加 ”raw:true“（如下所示）或将返回值调用 toJSON()。 144 | ```javascript 145 | async getConfigContent(where) { 146 | return this.models.AppGlobalConfig.findOne({ 147 | where, 148 | raw: true 149 | }); 150 | } 151 | ``` 152 | 153 | #### 7）scripts 154 |   如果要跑脚本，首先修改 scripts/index.js 文件中的最后一行 require() 的参数，即修改 ”./demo“。 155 | ```javascript 156 | global.env = process.env.NODE_ENV; 157 | global.logger = { 158 | trace: console.log, 159 | info: console.log, 160 | debug: console.log, 161 | error: console.log, 162 | warn: console.log, 163 | }; 164 | require('./demo'); 165 | ``` 166 |   当前位置如果与 scripts 目录平级，则执行命令： 167 | ```bash 168 | $ NODE_ENV=development node scripts/index.js 169 | ``` 170 |   其中 NODE_ENV 为环境常量，test、pre 和 production。 171 | 172 | #### 8）static 173 |   上传的文件默认会保存在 static/upload 目录中，git 会忽略该文件，不会提交到仓库中。 174 | 175 | 176 | # 开发步骤 177 | 1. 首先是在 models 目录中新建对应的表（如果不是新表，该步骤可省略）。 178 | 2. 然后是在 routes 目录中新建或修改某个路由文件。 179 | 3. 最后是在 services 目录中新建或修改某个服务文件。 180 | 181 |   在项目研发的过程中，发现很多操作都是对数据库做简单地增删改查，有时候就需要在上述三个目录中各自新建一个文件。 182 | 183 |   这么操作费时费力，到后期会发现有很多这样的接口，在维护上也会增加挑战，因此抽象了一套通用接口，保存在 routes/common.js 中。 184 | 185 | * get：读取一条数据（单表查询） 186 | * gets：读取多条数据（单表查询） 187 | * head：读取聚合数据，例如count()、sum()、max() 和 min() 188 | * post：提交数据，用于增加记录 189 | * put：更新数据 190 | 191 |   所有的接口采用 post 的请求方式，数据库表都是单表查询，不支持联表，若要联表则单独创建接口。 192 | 193 | # 定时任务 194 |   本地调试全任务可执行： 195 | ```bash 196 | $ npm run worker 197 | ``` 198 |   本地调试单任务可执行下面的命令，其中 ? 代表任务名称，即文件名，不用加后缀。 199 | ```bash 200 | $ JOB_TYPES=? npm run worker 201 | ``` 202 |   在 worker 目录中还包含两个目录：cronJobs 和 triggerJobs。 203 | 204 |   前者是定时类任务 （指定时间点执行），使用了 [node-schedule](https://github.com/node-schedule/node-schedule) 库。 205 | ```javascript 206 | module.exports = async () => { 207 | //每 30 秒执行一次定时任务 208 | schedule.scheduleJob({ rule: "*/30 * * * * *" }, () => { 209 | test(result); 210 | }); 211 | }; 212 | ``` 213 |   后者是触发类任务，在代码中输入指令触发执行，使用 [agenda](https://github.com/agenda/agenda) 库。 214 | ```javascript 215 | module.exports = (agenda) => { 216 | // 例如满足某种条件触发邮件通知 217 | agenda.define('send email report', (job, done) => { 218 | // 传递进来的数据 219 | const data = job.attrs.data; 220 | console.log(data); 221 | // 触发此任务，需要先引入 agenda.js，然后调用 now() 方法 222 | // import agenda from '../worker/agenda'; 223 | // agenda.now('send email report', { 224 | // username: realName, 225 | // }); 226 | }); 227 | }; 228 | ``` 229 |   注意，写好的任务记得添加进入口文件 index-worker.js。 230 | ```javascript 231 | require('./worker/cronJobs/demo')(); 232 | require('./worker/triggerJobs/demo')(agenda); 233 | ``` 234 | 235 | # 单元测试 236 |   运行下面的命令就会执行单元测试。 237 | ```bash 238 | $ npm test 239 | ``` 240 |   单元测试使用的框架是 [mocha 3.4](https://mochajs.cn/)，采用的断言是 [chai 4.0](https://www.chaijs.com/api/bdd/)，API测试库是 [supertest 3.0](https://github.com/visionmedia/supertest)，测试替代库 [sion.js](https://sinonjs.org) 241 | ```javascript 242 | // routers 测试 243 | describe('GET /user/list', () => { 244 | const url = '/user/list'; 245 | it('获取用户列表成功', (done) => { 246 | api 247 | .get(url) 248 | .set('Authorization', authToken) 249 | .expect(200, done); 250 | }); 251 | }); 252 | 253 | // serveices 测试 254 | import backendUserRole from '../../services/backendUserRole'; 255 | describe('用户角色', () => { 256 | it('获取指定id的角色信息', async () => { 257 | const service = new backendUserRole(models); 258 | const res = await service.getInfoById('584a4dc24c886205bd771afe'); 259 | // expect(2).toBe(2); 260 | // expect(res.rolePermisson).to.be.an('array'); 261 | }); 262 | }); 263 | ``` -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-02 16:13:00 4 | * @LastEditTime: 2023-04-27 18:05:24 5 | * @LastEditors: strick 6 | * @Description: 启动文件 7 | * @FilePath: /strick/shin-server/app.js 8 | */ 9 | import Koa from 'koa'; 10 | import KoaRouter from 'koa-router'; 11 | import KoaCompress from 'koa-compress'; 12 | import KoaBodyParser from 'koa-bodyparser'; 13 | import KoaValidate from 'koa-validate'; 14 | import KoaStatic from 'koa-static'; 15 | import koaBunyanLogger from 'koa-bunyan-logger'; 16 | import jwt from 'koa-jwt'; 17 | import config from 'config'; 18 | import pino from 'pino'; 19 | import koaPinoLogger from 'koa-pino-logger'; 20 | import routers from './routers/'; 21 | import errorHandle from './middlewares/errorHandle'; 22 | import init from './init'; 23 | 24 | const app = new Koa(); 25 | const router = new KoaRouter(); 26 | 27 | /** 28 | * 基于 OpenTracing 标准的分布式追踪系统，有助于分析服务架构中的计时数据 29 | * https://github.com/openzipkin/zipkin-js 30 | */ 31 | // import { Tracer, ConsoleRecorder, ExplicitContext } from 'zipkin'; 32 | // import { koaMiddleware } from 'zipkin-instrumentation-koa'; 33 | // const ctxImpl = new ExplicitContext(); 34 | // const recorder = new ConsoleRecorder(); 35 | // const tracer = new Tracer({recorder, ctxImpl, localServiceName: 'zipkin-koa-shin'}); 36 | // app.use(koaMiddleware({tracer})); 37 | 38 | 39 | /** 40 | * 全链路日志追踪 41 | * https://github.com/puzpuzpuz/cls-rtracer 42 | */ 43 | const rTracer = require('cls-rtracer'); 44 | 45 | global.logger = pino({ 46 | name: 'shin-server', 47 | level: 'trace', 48 | mixin () { 49 | return { 'req-id': rTracer.id() } 50 | }, 51 | hooks: { 52 | // 格式化日志 53 | logMethod (inputArgs, method) { 54 | const printfs = []; 55 | for(let i=0; i req.id, 85 | res: () => undefined, 86 | }, 87 | customAttributeKeys: { 88 | req: 'req-id', 89 | }, 90 | genReqId: () => rTracer.id(), // 声明 req-id 的值 91 | })); 92 | 93 | app.use(KoaCompress()); 94 | app.use(KoaBodyParser({ jsonLimit: '10mb', textLimit: '10mb', enableTypes: ['json', 'form', 'text'] })); 95 | app.use(KoaStatic('static')); 96 | app.use(KoaStatic('upload')); 97 | app.use(jwt({ 98 | secret: config.get('jwtSecret'), //401 Unauthorized 99 | }).unless({ 100 | path: [/user\/login/, /user\/init/, /\/download/, /common\/upload/, /pe\.gif/, /ma\.gif/, /smap\/del/, /callback/], //跳过登录态的请求 101 | })); 102 | app.use(errorHandle()); 103 | app.use(koaBunyanLogger.requestLogger({ 104 | updateLogFields(fields) { 105 | delete fields.req; 106 | delete fields.res; 107 | fields.operator = this.state.user && this.state.user.realName; 108 | }, 109 | })); 110 | 111 | app.use(router.routes()); 112 | app.use(router.allowedMethods()); 113 | 114 | routers(router); 115 | KoaValidate(app); 116 | 117 | app.listen(config.get('port')); 118 | 119 | logger.info(`Server started on ${config.get('port')}`); 120 | 121 | //开始处理任务 122 | init(); -------------------------------------------------------------------------------- /config/development.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-02 16:17:36 4 | * @LastEditTime: 2021-09-06 13:46:25 5 | * @LastEditors: strick 6 | * @Description: 本地配置文件 7 | * @FilePath: /strick/shin-server/config/development.js 8 | */ 9 | module.exports = { 10 | port: 6060, 11 | jwtSecret: 'abcd', 12 | /** 13 | * api服务域名配置 14 | */ 15 | services: { 16 | adminApi: 'http://localhost:8000/api', 17 | }, 18 | /** 19 | * 数据库相关配置 20 | */ 21 | mongodb: { 22 | hosts: [ 23 | '127.0.0.1:27017', 24 | ], 25 | options: { 26 | dbName: 'shin_backend', 27 | }, 28 | }, 29 | redis: { 30 | aws: { 31 | host: '127.0.0.1', 32 | port: 6379, 33 | }, 34 | }, 35 | // JS 映射文件地址 36 | sourceMapPath: '../../smap', 37 | // 任务队列缓存 38 | kueRedis: { 39 | host: '127.0.0.1', 40 | port: 6379, 41 | }, 42 | mysql: { 43 | backend: { 44 | database: 'shin_backend', 45 | username: null, 46 | password: null, 47 | options: { 48 | dialect: 'mysql', 49 | port: 3306, 50 | replication: { 51 | read: [ 52 | { host: '127.0.0.1', username: 'root', password: '123' }, 53 | ], 54 | write: { host: '127.0.0.1', username: 'root', password: '123' }, 55 | }, 56 | benchmark: true, 57 | dialectOptions: { 58 | charset: 'utf8mb4', 59 | collate: 'utf8mb4_unicode_ci', 60 | supportBigNumbers: true, 61 | bigNumberStrings: true, 62 | }, 63 | pool: { 64 | max: 10, 65 | min: 0, 66 | }, 67 | }, 68 | }, 69 | }, 70 | }; 71 | -------------------------------------------------------------------------------- /db/mongodb.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-02-02 16:18:02 5 | * @LastEditTime: 2023-04-24 18:09:48 6 | * @Description: MongoDB配置 7 | * @FilePath: /strick/shin-server/db/mongodb.js 8 | */ 9 | import mongoose from 'mongoose'; 10 | import config from 'config'; 11 | 12 | mongoose.set('debug', (...args) => { 13 | if (args[0] !== 'agendaJobs') { 14 | logger.debug(...args); 15 | } 16 | }); 17 | 18 | const mongooseConfig = config.get('mongodb'); 19 | const url = `mongodb://${mongooseConfig.hosts.join(',')}`; 20 | mongoose.connect(url, { 21 | poolSize: 10, 22 | useNewUrlParser: true, 23 | ...mongooseConfig.options, 24 | promiseLibrary: Promise, 25 | }); 26 | const db = mongoose.connection; 27 | 28 | db.once('open', () => { 29 | logger.info('mongodb connected'); 30 | }); 31 | db.on('error', (err) => { 32 | logger.error(`mongodb err: ${err}`); 33 | }); 34 | 35 | export default mongoose; 36 | -------------------------------------------------------------------------------- /db/mysql.js: -------------------------------------------------------------------------------- 1 | import Sequelize from 'sequelize'; 2 | import config from 'config'; 3 | 4 | const mysql = {}; 5 | const mysqlServers = config.get('mysql'); 6 | const create = (params) => { 7 | const { database, username, password, options } = params; 8 | return new Sequelize(database, username, password, { 9 | ...options, 10 | logging: (msg, benchmark) => logger.debug(msg, `${benchmark}ms`), 11 | }); 12 | }; 13 | Object.keys(mysqlServers).forEach((item) => { 14 | mysql[item] = create(mysqlServers[item]); 15 | mysql[item].authenticate().then(() => { 16 | logger.info(`${item} mysql connected success`); 17 | }).catch((error) => { 18 | logger.error(`${item} mysql 连接错误: ${error.toString()}`); 19 | }); 20 | }); 21 | 22 | global.Sequelize = Sequelize; 23 | export default mysql; 24 | -------------------------------------------------------------------------------- /db/redis.js: -------------------------------------------------------------------------------- 1 | import Redis from 'ioredis'; 2 | import config from 'config'; 3 | 4 | const connections = {}; 5 | const redisServers = config.get('redis'); 6 | 7 | Object.keys(redisServers).forEach((item) => { 8 | const redisConfig = redisServers[item]; 9 | let client; 10 | // 如果配置项是数组则使用 cluster 模式连接 11 | if (Array.isArray(redisConfig)) { 12 | client = new Redis.Cluster(redisConfig); 13 | } else { 14 | client = new Redis(redisConfig); 15 | } 16 | client.on('error', (err) => { 17 | logger.error(`${item} redis error: ${err}`); 18 | }); 19 | client.on('ready', () => { 20 | logger.info(`${item} redis connected`); 21 | }); 22 | connections[item] = client; 23 | }); 24 | 25 | export default connections; 26 | 27 | -------------------------------------------------------------------------------- /docs/SQL/shin_backend.sql: -------------------------------------------------------------------------------- 1 | CREATE DATABASE IF NOT EXISTS `shin_backend` /*!40100 DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci */; 2 | USE `shin_backend`; 3 | 4 | DROP TABLE IF EXISTS `app_global_config`; 5 | CREATE TABLE `app_global_config` ( 6 | `id` int(11) NOT NULL AUTO_INCREMENT, 7 | `title` varchar(60) COLLATE utf8mb4_bin NOT NULL COMMENT '标题', 8 | `content` text COLLATE utf8mb4_bin NOT NULL COMMENT 'JSON格式的内容', 9 | `key` varchar(40) COLLATE utf8mb4_bin NOT NULL COMMENT '唯一标识', 10 | `ctime` timestamp NULL DEFAULT CURRENT_TIMESTAMP, 11 | `mtime` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 12 | `status` tinyint(4) DEFAULT '1' COMMENT '状态', 13 | PRIMARY KEY (`id`), 14 | UNIQUE KEY `key_UNIQUE` (`key`) 15 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='全局通用配置'; 16 | 17 | DROP TABLE IF EXISTS `web_short_chain`; 18 | CREATE TABLE `web_short_chain` ( 19 | `id` int(11) NOT NULL AUTO_INCREMENT, 20 | `short` varchar(10) COLLATE utf8mb4_bin NOT NULL COMMENT '短链地址中的key', 21 | `url` varchar(200) COLLATE utf8mb4_bin NOT NULL COMMENT '原始地址', 22 | `ctime` timestamp NULL DEFAULT CURRENT_TIMESTAMP, 23 | `mtime` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, 24 | `status` tinyint(4) NOT NULL DEFAULT '1' COMMENT '状态', 25 | PRIMARY KEY (`id`), 26 | UNIQUE KEY `short_UNIQUE` (`short`) 27 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='短链存储'; 28 | 29 | DROP TABLE IF EXISTS `web_monitor`; 30 | CREATE TABLE `web_monitor` ( 31 | `id` bigint(20) NOT NULL AUTO_INCREMENT, 32 | `project` varchar(45) COLLATE utf8mb4_bin NOT NULL COMMENT '项目名称', 33 | `project_subdir` varchar(45) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '项目的子目录，shin-h5项目下会有很多活动，放置在各个子目录中', 34 | `digit` int(11) NOT NULL DEFAULT '1' COMMENT '出现次数', 35 | `message` text COLLATE utf8mb4_bin NOT NULL COMMENT '聚合信息', 36 | `ua` varchar(600) COLLATE utf8mb4_bin NOT NULL COMMENT '代理信息', 37 | `key` varchar(45) COLLATE utf8mb4_bin NOT NULL COMMENT '去重用的标记', 38 | `category` varchar(45) COLLATE utf8mb4_bin NOT NULL COMMENT '日志类型', 39 | `source` varchar(45) COLLATE utf8mb4_bin DEFAULT NULL COMMENT 'SourceMap映射文件的地址', 40 | `ctime` timestamp NULL DEFAULT CURRENT_TIMESTAMP, 41 | `identity` varchar(30) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '身份，用于连贯日志上下文', 42 | `day` int(11) DEFAULT NULL COMMENT '格式化的天（冗余字段），用于排序，20210322', 43 | `hour` tinyint(2) DEFAULT NULL COMMENT '格式化的小时（冗余字段），用于分组，11', 44 | `minute` tinyint(2) DEFAULT NULL COMMENT '格式化的分钟（冗余字段），用于分组，20', 45 | `message_status` int(11) DEFAULT NULL COMMENT 'message中的通信状态码', 46 | `message_path` varchar(45) COLLATE utf8mb4_bin DEFAULT NULL COMMENT 'message通信中的 path', 47 | `message_type` varchar(20) COLLATE utf8mb4_bin DEFAULT NULL COMMENT 'message中的类别字段', 48 | `referer` varchar(200) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '来源地址', 49 | PRIMARY KEY (`id`), 50 | KEY `idx_key_category_project_identity` (`key`,`category`,`project`,`identity`), 51 | KEY `idx_category_project_identity` (`category`,`project`,`identity`), 52 | KEY `index_ctime` (`ctime`), 53 | KEY `index_ctime_category` (`ctime`,`category`), 54 | KEY `idx_category_project_ctime` (`category`,`project`,`ctime`), 55 | KEY `idx_messagepath` (`message_path`), 56 | KEY `idx_category_messagetype_ctime` (`category`,`message_type`,`ctime`) 57 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='前端监控日志'; 58 | 59 | DROP TABLE IF EXISTS `web_monitor_statis`; 60 | CREATE TABLE `web_monitor_statis` ( 61 | `id` int(11) NOT NULL AUTO_INCREMENT, 62 | `date` int(11) NOT NULL COMMENT '日期，格式为20210318，一天只存一条记录', 63 | `statis` text COLLATE utf8mb4_bin NOT NULL COMMENT '以JSON格式保存的统计信息', 64 | PRIMARY KEY (`id`), 65 | UNIQUE KEY `date_UNIQUE` (`date`) 66 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='前端监控日志统计信息表'; 67 | 68 | DROP TABLE IF EXISTS `web_performance`; 69 | CREATE TABLE `web_performance` ( 70 | `id` bigint(20) NOT NULL AUTO_INCREMENT, 71 | `load` int(11) NOT NULL DEFAULT '0' COMMENT '页面加载总时间', 72 | `ready` int(11) NOT NULL DEFAULT '0' COMMENT '用户可操作时间', 73 | `paint` int(11) NOT NULL DEFAULT '0' COMMENT '白屏时间', 74 | `screen` int(11) NOT NULL DEFAULT '0' COMMENT '首屏时间', 75 | `measure` varchar(1000) COLLATE utf8mb4_bin NOT NULL COMMENT '其它测量参数，用JSON格式保存', 76 | `ctime` timestamp NULL DEFAULT CURRENT_TIMESTAMP, 77 | `day` int(11) NOT NULL COMMENT '格式化的天（冗余字段），用于排序，20210322', 78 | `hour` tinyint(2) NOT NULL COMMENT '格式化的小时（冗余字段），用于分组，11', 79 | `minute` tinyint(2) DEFAULT NULL COMMENT '格式化的分钟（冗余字段），用于分组，20', 80 | `identity` varchar(30) COLLATE utf8mb4_bin NOT NULL COMMENT '身份', 81 | `project` varchar(20) COLLATE utf8mb4_bin NOT NULL COMMENT '项目关键字，关联 web_performance_project 表中的key', 82 | `ua` varchar(600) COLLATE utf8mb4_bin NOT NULL COMMENT '代理信息', 83 | `referer` varchar(200) COLLATE utf8mb4_bin DEFAULT NULL COMMENT '来源地址', 84 | `timing` text COLLATE utf8mb4_bin COMMENT '浏览器读取到的性能参数，用于排查', 85 | `resource` text COLLATE utf8mb4_bin COMMENT '静态资源信息', 86 | PRIMARY KEY (`id`), 87 | KEY `idx_project_day` (`project`,`day`), 88 | KEY `idx_project_day_hour` (`project`,`day`,`hour`) 89 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='性能监控'; 90 | 91 | DROP TABLE IF EXISTS `web_performance_project`; 92 | CREATE TABLE `web_performance_project` ( 93 | `id` int(11) NOT NULL AUTO_INCREMENT, 94 | `key` varchar(20) COLLATE utf8mb4_bin NOT NULL COMMENT '唯一值', 95 | `name` varchar(45) COLLATE utf8mb4_bin NOT NULL COMMENT '项目名称', 96 | `ctime` timestamp NULL DEFAULT CURRENT_TIMESTAMP, 97 | `status` tinyint(1) NOT NULL DEFAULT '1' COMMENT '1：正常 0：删除', 98 | PRIMARY KEY (`id`), 99 | UNIQUE KEY `name_UNIQUE` (`name`) 100 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='性能监控项目'; 101 | 102 | DROP TABLE IF EXISTS `web_performance_statis`; 103 | CREATE TABLE `web_performance_statis` ( 104 | `id` int(11) NOT NULL AUTO_INCREMENT, 105 | `date` int(11) NOT NULL COMMENT '日期，格式为20210318', 106 | `statis` mediumtext COLLATE utf8mb4_bin NOT NULL COMMENT '以JSON格式保存的统计信息', 107 | `project` varchar(20) COLLATE utf8mb4_bin NOT NULL COMMENT '项目关键字，关联 web_performance_project 表中的key', 108 | PRIMARY KEY (`id`) 109 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='性能监控统计'; 110 | 111 | DROP TABLE IF EXISTS `web_monitor_record`; 112 | CREATE TABLE `web_monitor_record` ( 113 | `id` INT NOT NULL AUTO_INCREMENT, 114 | `monitor_id` BIGINT NOT NULL COMMENT '监控日志的ID', 115 | `record` MEDIUMTEXT NOT NULL COMMENT '回放信息，包括各类元素和DOM操作记录', 116 | `ctime` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, 117 | PRIMARY KEY (`id`), 118 | UNIQUE INDEX `monitor_id_UNIQUE` (`monitor_id` ASC)) 119 | DEFAULT CHARACTER SET = utf8mb4 120 | COLLATE = utf8mb4_bin 121 | COMMENT = '直播监控回放'; -------------------------------------------------------------------------------- /docs/assets/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pwstrick/shin-server/3bdb181185a0bf58aa6e18034cb8640a0731a877/docs/assets/1.png -------------------------------------------------------------------------------- /docs/assets/architecture.odg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pwstrick/shin-server/3bdb181185a0bf58aa6e18034cb8640a0731a877/docs/assets/architecture.odg -------------------------------------------------------------------------------- /docs/assets/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pwstrick/shin-server/3bdb181185a0bf58aa6e18034cb8640a0731a877/docs/assets/architecture.png -------------------------------------------------------------------------------- /docs/assets/shin.odg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pwstrick/shin-server/3bdb181185a0bf58aa6e18034cb8640a0731a877/docs/assets/shin.odg -------------------------------------------------------------------------------- /docs/assets/shin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pwstrick/shin-server/3bdb181185a0bf58aa6e18034cb8640a0731a877/docs/assets/shin.png -------------------------------------------------------------------------------- /index-worker.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2020-12-16 19:17:57 5 | * @LastEditTime: 2021-02-03 15:01:36 6 | * @Description: 定时任务配置文件 7 | * @FilePath: /strick/shin-server/index-worker.js 8 | */ 9 | require('babel-core/register'); 10 | require('babel-polyfill'); 11 | 12 | const bunyan = require('bunyan'); 13 | 14 | global.logger = bunyan.createLogger({ 15 | name: 'Task', 16 | level: 'trace', 17 | }); 18 | 19 | global.env = process.env.NODE_ENV || 'development'; 20 | 21 | const agenda = require('./worker/agenda.js'); 22 | //读取命令中带的任务名称（用于调试），例如 JOB_TYPES=? npm run worker ( ? 代表任务名称，即文件名) 23 | const jobTypes = process.env.JOB_TYPES ? process.env.JOB_TYPES.split(',') : []; 24 | 25 | /** 26 | * 基于时间的定时任务，采用 node-schedule 库 27 | * https://github.com/node-schedule/node-schedule 28 | */ 29 | if (jobTypes.length) { 30 | jobTypes.forEach((type) => { 31 | try { 32 | require('./worker/cronJobs/' + type)(); 33 | } catch (ex) { 34 | 35 | } 36 | }); 37 | } else { 38 | require('./worker/cronJobs/demo')(); 39 | } 40 | 41 | /** 42 | * 触发型的定时任务，采用 agenda 库 43 | * https://github.com/agenda/agenda 44 | */ 45 | agenda.on('ready', () => { 46 | if (jobTypes.length) { 47 | jobTypes.forEach((type) => { 48 | try { 49 | require('./worker/triggerJobs/' + type)(agenda); 50 | } catch (ex) { 51 | 52 | } 53 | }); 54 | } else { 55 | require('./worker/triggerJobs/demo')(agenda); 56 | } 57 | agenda.start().catch(error => logger.error('job start error', error)); 58 | }); 59 | 60 | async function graceful() { 61 | await agenda.stop(); 62 | process.exit(0); 63 | } 64 | 65 | process.on('SIGTERM', graceful); 66 | process.on('SIGINT', graceful); 67 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-02 16:10:13 4 | * @LastEditTime: 2021-02-02 16:13:08 5 | * @LastEditors: strick 6 | * @Description: 入口文件 7 | * @FilePath: /strick/shin-server/index.js 8 | */ 9 | require('babel-core/register'); 10 | require('babel-polyfill'); 11 | 12 | global.env = process.env.NODE_ENV || 'development'; 13 | 14 | require('./app.js'); -------------------------------------------------------------------------------- /init.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-09-06 12:49:20 5 | * @LastEditTime: 2021-09-06 12:53:00 6 | * @Description: 7 | * @FilePath: /strick/shin-server/init.js 8 | */ 9 | import queue from './utils/queue'; 10 | import services from './services'; 11 | 12 | export default () => { 13 | //处理性能数据 14 | queue.process('handlePerformance', (job, done) => { 15 | services.webMonitor.createPerformance(job.data.performance) 16 | .then(() => done()) 17 | .catch((err) => { 18 | logger.error(err); 19 | done(err); 20 | }); 21 | }); 22 | 23 | //处理监控数据 24 | const taskName = 'handleMonitor'; 25 | queue.process(taskName, (job, done) => { 26 | logger.trace(`${taskName} job begin`, job.id); 27 | services.webMonitor.handleMonitor(job.data.monitor) 28 | .then(() => { 29 | done(); 30 | logger.trace(`${taskName} job completed`, job.id); 31 | job.remove(function(){//手动移除 32 | logger.trace(`${taskName} job removed`, job.id); 33 | }); 34 | }) 35 | .catch((err) => { 36 | logger.error(err); 37 | done(err); 38 | }); 39 | }); 40 | } -------------------------------------------------------------------------------- /middlewares/checkAuth.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-02 16:24:43 4 | * @LastEditTime: 2021-02-03 18:34:31 5 | * @LastEditors: strick 6 | * @Description: 验证当前用户的操作权限 7 | * @FilePath: /strick/shin-server/middlewares/checkAuth.js 8 | */ 9 | import redis from '../db/redis'; 10 | 11 | export default authority => async (ctx, next) => { 12 | const { id } = ctx.state.user; 13 | const res = await redis.aws.get(`backend:user:account:authorities:${id}`); 14 | if (!res) { 15 | ctx.status = 409; 16 | ctx.body = { error: '服务升级，请重新登录' }; 17 | return; 18 | } 19 | const authorities = res.split(','); 20 | if (authorities.includes(authority) || authorities[0] === '*') { 21 | await next(); 22 | } else { 23 | ctx.status = 403; 24 | ctx.body = { error: '您没有操作权限' }; 25 | } 26 | }; 27 | -------------------------------------------------------------------------------- /middlewares/checkExport.js: -------------------------------------------------------------------------------- 1 | import config from 'config'; 2 | import jwt from 'jsonwebtoken'; 3 | 4 | export default () => async (ctx, next) => { 5 | const { token } = ctx.query; 6 | try { 7 | jwt.verify(token, config.get('jwtSecret')); 8 | await next(); 9 | } catch (error) { 10 | ctx.status = 400; 11 | ctx.body = { error }; 12 | } 13 | }; 14 | -------------------------------------------------------------------------------- /middlewares/errorHandle.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-02-02 16:24:43 5 | * @LastEditTime: 2023-04-25 17:18:34 6 | * @Description: 所有路由错误处理 7 | * @FilePath: /strick/shin-server/middlewares/errorHandle.js 8 | */ 9 | export default () => async (ctx, next) => { 10 | try { 11 | await next(); 12 | // 在响应头中自定义 req-id，实现全链路监控 13 | ctx.set('Access-Control-Expose-Headers', 'req-id'); 14 | ctx.set('req-id', ctx.req.id); 15 | } catch (error) { 16 | logger.error(ctx.path, error, ctx.path); 17 | ctx.status = 500; 18 | ctx.body = { error: String(error), reqId: ctx.req.id }; 19 | } 20 | }; 21 | -------------------------------------------------------------------------------- /middlewares/index.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-02-02 16:22:30 5 | * @LastEditTime: 2023-04-24 18:11:27 6 | * @Description: 中间件配置 7 | * @FilePath: /strick/shin-server/middlewares/index.js 8 | */ 9 | import checkAuth from './checkAuth'; 10 | import checkExport from './checkExport'; 11 | import errorHandle from './errorHandle'; 12 | 13 | const middleswares = { 14 | checkAuth, 15 | checkExport, 16 | errorHandle 17 | }; 18 | 19 | export default middleswares; 20 | -------------------------------------------------------------------------------- /models/AppGlobalConfig.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2020-12-24 11:49:07 5 | * @LastEditTime: 2021-02-03 13:50:48 6 | * @Description: 全局通用配置 7 | * @FilePath: /strick/shin-server/models/AppGlobalConfig.js 8 | */ 9 | export default ({ mysql }) => 10 | mysql.backend.define("AppGlobalConfig", 11 | { 12 | id: { 13 | type: Sequelize.INTEGER, 14 | field: "id", 15 | autoIncrement: true, 16 | primaryKey: true 17 | }, 18 | // 标题 19 | title: { 20 | type: Sequelize.STRING, 21 | field: "title" 22 | }, 23 | // 内容 24 | content: { 25 | type: Sequelize.STRING, 26 | field: "content" 27 | }, 28 | // 唯一标识，用于读取 29 | key: { 30 | type: Sequelize.STRING, 31 | field: "key" 32 | }, 33 | ctime: { 34 | type: Sequelize.DATE, 35 | field: "ctime" 36 | }, 37 | mtime: { 38 | type: Sequelize.DATE, 39 | field: "mtime" 40 | }, 41 | status: { 42 | type: Sequelize.INTEGER, 43 | field: "status" 44 | } 45 | }, 46 | { 47 | tableName: "app_global_config", 48 | timestamps: false 49 | } 50 | ); 51 | -------------------------------------------------------------------------------- /models/BackendUserAccount.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-02 16:52:54 4 | * @LastEditTime: 2021-02-05 16:16:43 5 | * @LastEditors: strick 6 | * @Description: 管理后台账号 7 | * @FilePath: /strick/shin-server/models/BackendUserAccount.js 8 | */ 9 | export default ({ mongodb }) => 10 | mongodb.model( 11 | "BackendUserAccount", 12 | { 13 | // 用户名 14 | userName: { 15 | type: String, 16 | index: true 17 | }, 18 | // 密码 19 | password: String, 20 | // 真实姓名 21 | realName: String, 22 | // 手机号 23 | cellphone: String, 24 | // 创建时间 25 | createTime: { 26 | type: Date, 27 | default: Date.now 28 | }, 29 | // 更新时间 30 | updateTime: { 31 | type: Date, 32 | default: Date.now 33 | }, 34 | // 密码过期时间 35 | passwordExpireTime: Date, 36 | // 角色 37 | roles: Array, 38 | // 加密秘钥 39 | salt: String, 40 | // 状态 41 | status: { 42 | type: Number, 43 | default: 1 44 | }, 45 | // 在线状态 46 | online: { 47 | type: Number, 48 | default: 0 49 | } 50 | }, 51 | "user_account" 52 | ); 53 | 54 | -------------------------------------------------------------------------------- /models/BackendUserRole.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-02 16:52:31 4 | * @LastEditTime: 2021-02-02 16:53:08 5 | * @LastEditors: strick 6 | * @Description: 账号角色 7 | * @FilePath: /strick/shin-server/models/BackendUserRole.js 8 | */ 9 | export default ({ mongodb }) => mongodb.model('BackendUserRole', { 10 | // 角色名称 11 | roleName: String, 12 | // 角色简介 13 | roleDesc: String, 14 | // 角色拥有的权限 15 | rolePermission: Array, 16 | // 创建时间 17 | createTime: { 18 | type: Date, 19 | default: Date.now, 20 | }, 21 | // 更新时间 22 | updateTime: { 23 | type: Date, 24 | default: Date.now, 25 | }, 26 | // 状态 27 | status: { 28 | type: Number, 29 | default: 1, 30 | }, 31 | }, 'user_role'); 32 | -------------------------------------------------------------------------------- /models/WebMonitor.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-24 16:17:40 4 | * @LastEditTime: 2023-05-08 17:20:55 5 | * @LastEditors: strick 6 | * @Description: 前端监控日志表 7 | * @FilePath: /strick/shin-server/models/WebMonitor.js 8 | */ 9 | module.exports = ({ mysql }) => 10 | mysql.backend.define( 11 | "WebMonitor", 12 | { 13 | id: { 14 | type: Sequelize.BIGINT, 15 | field: "id", 16 | autoIncrement: true, 17 | primaryKey: true 18 | }, 19 | project: { 20 | type: Sequelize.STRING, 21 | field: "project" 22 | }, 23 | project_subdir: { 24 | type: Sequelize.STRING, 25 | field: "project_subdir" 26 | }, 27 | digit: { 28 | type: Sequelize.INTEGER, 29 | field: "digit" 30 | }, 31 | message: { 32 | type: Sequelize.TEXT, 33 | field: "message" 34 | }, 35 | key: { 36 | type: Sequelize.STRING, 37 | field: "key" 38 | }, 39 | ua: { 40 | type: Sequelize.STRING(600), 41 | field: "ua" 42 | }, 43 | source: { 44 | type: Sequelize.STRING, 45 | field: "source" 46 | }, 47 | category: { 48 | type: Sequelize.STRING, 49 | field: "category" 50 | }, 51 | ctime: { 52 | type: Sequelize.DATE, 53 | field: "ctime" 54 | }, 55 | identity: { 56 | type: Sequelize.STRING, 57 | field: "identity" 58 | }, 59 | day: { 60 | type: Sequelize.INTEGER, 61 | field: "day" 62 | }, 63 | hour: { 64 | type: Sequelize.INTEGER, 65 | field: "hour" 66 | }, 67 | minute: { 68 | type: Sequelize.INTEGER, 69 | field: "minute" 70 | }, 71 | message_status: { 72 | type: Sequelize.INTEGER, 73 | field: "message_status" 74 | }, 75 | message_path: { 76 | type: Sequelize.STRING, 77 | field: "message_path" 78 | }, 79 | message_type: { 80 | type: Sequelize.STRING, 81 | field: "message_type" 82 | }, 83 | referer: { 84 | type: Sequelize.STRING, 85 | field: 'referer', 86 | }, 87 | }, 88 | { 89 | tableName: "web_monitor", 90 | timestamps: false 91 | } 92 | ); 93 | 94 | -------------------------------------------------------------------------------- /models/WebMonitorRecord.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2022-12-21 17:15:12 5 | * @LastEditTime: 2022-12-21 17:15:18 6 | * @Description: 行为记录表 7 | * @FilePath: /strick/shin-server/models/WebMonitorRecord.js 8 | */ 9 | module.exports = ({ mysql }) => mysql.backend.define( 10 | 'WebMonitorRecord', 11 | { 12 | monitor_id: Sequelize.BIGINT, 13 | record: Sequelize.TEXT('medium'), 14 | ctime: { 15 | type: Sequelize.DATE, 16 | field: 'ctime', 17 | }, 18 | }, 19 | { 20 | tableName: 'web_monitor_record', 21 | timestamps: false, 22 | }, 23 | ); -------------------------------------------------------------------------------- /models/WebMonitorStatis.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-03-17 13:08:09 5 | * @LastEditTime: 2021-09-06 12:18:57 6 | * @Description: 监控统计 7 | * @FilePath: /strick/shin-server/models/WebMonitorStatis.js 8 | */ 9 | module.exports = ({ mysql }) => 10 | mysql.backend.define( 11 | "WebMonitorStatis", 12 | { 13 | id: { 14 | type: Sequelize.INTEGER, 15 | field: "id", 16 | autoIncrement: true, 17 | primaryKey: true 18 | }, 19 | date: { 20 | type: Sequelize.INTEGER, 21 | field: "date" 22 | }, 23 | statis: { 24 | type: Sequelize.TEXT, 25 | field: "statis" 26 | }, 27 | }, 28 | { 29 | tableName: "web_monitor_statis", 30 | timestamps: false 31 | } 32 | ); -------------------------------------------------------------------------------- /models/WebPerformance.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-03-23 11:48:52 5 | * @LastEditTime: 2022-07-12 16:29:05 6 | * @Description: 性能监控日志 7 | * @FilePath: /strick/shin-server/models/WebPerformance.js 8 | */ 9 | module.exports = ({ mysql }) => 10 | mysql.backend.define( 11 | "WebPerformance", 12 | { 13 | id: { 14 | type: Sequelize.BIGINT, 15 | field: 'id', 16 | autoIncrement: true, 17 | primaryKey: true, 18 | }, 19 | load: { 20 | type: Sequelize.INTEGER, 21 | field: 'load', 22 | }, 23 | ready: { 24 | type: Sequelize.INTEGER, 25 | field: 'ready', 26 | }, 27 | paint: { 28 | type: Sequelize.INTEGER, 29 | field: 'paint', 30 | }, 31 | screen: { 32 | type: Sequelize.INTEGER, 33 | field: 'screen', 34 | }, 35 | ua: { 36 | type: Sequelize.STRING(600), 37 | field: 'ua', 38 | }, 39 | measure: { 40 | type: Sequelize.STRING(1000), 41 | field: 'measure', 42 | }, 43 | day: { 44 | type: Sequelize.INTEGER, 45 | field: 'day', 46 | }, 47 | hour: { 48 | type: Sequelize.INTEGER, 49 | field: 'hour', 50 | }, 51 | minute: { 52 | type: Sequelize.INTEGER, 53 | field: 'minute', 54 | }, 55 | project: { 56 | type: Sequelize.STRING, 57 | field: 'project', 58 | }, 59 | ctime: { 60 | type: Sequelize.DATE, 61 | field: 'ctime', 62 | }, 63 | identity: { 64 | type: Sequelize.STRING, 65 | field: 'identity', 66 | }, 67 | referer: { 68 | type: Sequelize.STRING, 69 | field: 'referer', 70 | }, 71 | referer_path: { 72 | type: Sequelize.STRING, 73 | field: 'referer_path', 74 | }, 75 | timing: { 76 | type: Sequelize.TEXT, 77 | field: 'timing', 78 | }, 79 | resource: { 80 | type: Sequelize.TEXT, 81 | field: 'resource', 82 | }, 83 | }, 84 | { 85 | tableName: "web_performance", 86 | timestamps: false 87 | } 88 | ); 89 | 90 | -------------------------------------------------------------------------------- /models/WebPerformanceProject.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-03-22 14:23:36 5 | * @LastEditTime: 2021-09-06 12:19:18 6 | * @Description: 性能监控项目 7 | * @FilePath: /strick/shin-server/models/WebPerformanceProject.js 8 | */ 9 | module.exports = ({ mysql }) => 10 | mysql.backend.define( 11 | "WebPerformanceProject", 12 | { 13 | id: { 14 | type: Sequelize.INTEGER, 15 | field: "id", 16 | autoIncrement: true, 17 | primaryKey: true 18 | }, 19 | name: { 20 | type: Sequelize.STRING, 21 | field: "name" 22 | }, 23 | key: { 24 | type: Sequelize.STRING, 25 | field: "key" 26 | }, 27 | status: { 28 | type: Sequelize.INTEGER, 29 | field: "status" 30 | }, 31 | ctime: { 32 | type: Sequelize.DATE, 33 | field: "ctime" 34 | }, 35 | }, 36 | { 37 | tableName: "web_performance_project", 38 | timestamps: false 39 | } 40 | ); 41 | -------------------------------------------------------------------------------- /models/WebPerformanceStatis.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-03-23 18:06:44 5 | * @LastEditTime: 2021-09-06 12:19:23 6 | * @Description: 性能数据统计 7 | * @FilePath: /strick/shin-server/models/WebPerformanceStatis.js 8 | */ 9 | module.exports = ({ mysql }) => 10 | mysql.backend.define( 11 | "WebPerformanceStatis", 12 | { 13 | id: { 14 | type: Sequelize.INTEGER, 15 | field: "id", 16 | autoIncrement: true, 17 | primaryKey: true 18 | }, 19 | date: { 20 | type: Sequelize.INTEGER, 21 | field: "date" 22 | }, 23 | statis: { 24 | type: Sequelize.TEXT, 25 | field: "statis" 26 | }, 27 | project: { 28 | type: Sequelize.TEXT, 29 | field: "project" 30 | }, 31 | }, 32 | { 33 | tableName: "web_performance_statis", 34 | timestamps: false 35 | } 36 | ); -------------------------------------------------------------------------------- /models/WebShortChain.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-01-19 10:01:09 4 | * @LastEditTime: 2021-02-05 13:42:30 5 | * @LastEditors: strick 6 | * @Description: 短链存储 7 | * @FilePath: /strick/shin-server/models/WebShortChain.js 8 | */ 9 | module.exports = ({ mysql }) => 10 | mysql.backend.define("WebShortChain", 11 | { 12 | id: { 13 | type: Sequelize.INTEGER, 14 | field: "id", 15 | autoIncrement: true, 16 | primaryKey: true 17 | }, 18 | // 短链中的6个字符 19 | short: { 20 | type: Sequelize.STRING, 21 | field: "short" 22 | }, 23 | // 原始地址 24 | url: { 25 | type: Sequelize.STRING, 26 | field: "url" 27 | }, 28 | ctime: { 29 | type: Sequelize.DATE, 30 | field: "ctime" 31 | }, 32 | mtime: { 33 | type: Sequelize.DATE, 34 | field: "mtime" 35 | }, 36 | status: { 37 | type: Sequelize.INTEGER, 38 | field: "status" 39 | } 40 | }, 41 | { 42 | tableName: "web_short_chain", 43 | timestamps: false 44 | } 45 | ); 46 | 47 | -------------------------------------------------------------------------------- /models/index.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-02-02 16:22:56 5 | * @LastEditTime: 2023-04-24 18:11:37 6 | * @Description: model文件配置 7 | * @FilePath: /strick/shin-server/models/index.js 8 | */ 9 | import mysql from '../db/mysql'; 10 | import mongodb from '../db/mongodb'; 11 | import redis from '../db/redis'; 12 | import AppGlobalConfig from './AppGlobalConfig'; 13 | import BackendUserAccount from './BackendUserAccount'; 14 | import BackendUserRole from './BackendUserRole'; 15 | import WebMonitor from './WebMonitor'; 16 | import WebMonitorRecord from './WebMonitorRecord'; 17 | import WebMonitorStatis from './WebMonitorStatis'; 18 | import WebPerformance from './WebPerformance'; 19 | import WebPerformanceProject from './WebPerformanceProject'; 20 | import WebPerformanceStatis from './WebPerformanceStatis'; 21 | import WebShortChain from './WebShortChain'; 22 | 23 | const models = { 24 | AppGlobalConfig, 25 | BackendUserAccount, 26 | BackendUserRole, 27 | WebMonitor, 28 | WebMonitorRecord, 29 | WebMonitorStatis, 30 | WebPerformance, 31 | WebPerformanceProject, 32 | WebPerformanceStatis, 33 | WebShortChain 34 | }; 35 | Object.keys(models).forEach(key => { 36 | models[key] = models[key]({ mysql, mongodb, redis }); 37 | }); 38 | 39 | // Object.keys(models).forEach((modelName) => { 40 | // if (models[modelName].associate) { 41 | // models[modelName].associate(models); 42 | // } 43 | // }); 44 | export default models; 45 | 46 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "shin-server", 3 | "version": "1.0.0", 4 | "description": "后台管理系统", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "NODE_ENV=development nodemon index.js | pino-pretty", 8 | "worker": "NODE_ENV=development nodemon index-worker.js | bunyan", 9 | "test": "./node_modules/mocha/bin/mocha | bunyan -l warn" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "git+https://github.com/pwstrick/shin-server.git" 14 | }, 15 | "author": "pwstrick", 16 | "license": "ISC", 17 | "bugs": { 18 | "url": "https://github.com/pwstrick/shin-server/issues" 19 | }, 20 | "homepage": "https://github.com/pwstrick/shin-server#readme", 21 | "dependencies": { 22 | "agenda": "^2.0.2", 23 | "axios": "^0.16.2", 24 | "babel-core": "^6.24.1", 25 | "babel-polyfill": "^6.23.0", 26 | "bunyan": "^1.8.12", 27 | "cls-rtracer": "^2.6.2", 28 | "config": "^1.30.0", 29 | "es6-requireindex": "^0.3.10", 30 | "ioredis": "^4.2.0", 31 | "koa": "^2.13.1", 32 | "koa-bodyparser": "^4.2.0", 33 | "koa-bunyan-logger": "^2.1.0", 34 | "koa-compress": "^2.0.0", 35 | "koa-jwt": "^3.2.2", 36 | "koa-logger": "^3.0.0", 37 | "koa-multer": "^1.0.2", 38 | "koa-pino-logger": "^4.0.0", 39 | "koa-router": "7.1.1", 40 | "koa-static": "^4.0.1", 41 | "koa-validate": "^1.0.7", 42 | "koa2-formidable": "^1.0.2", 43 | "kue": "^0.11.6", 44 | "lodash": "^4.17.4", 45 | "moment": "^2.18.1", 46 | "mongodb": "^2.2.27", 47 | "mongoose": "^5.4.17", 48 | "mysql": "^2.13.0", 49 | "node-schedule": "^1.3.2", 50 | "pino": "^8.11.0", 51 | "redis": "^2.7.1", 52 | "redis-clustr": "^1.6.0", 53 | "sequelize": "^3.30.4", 54 | "ua-parser-js": "^0.7.28", 55 | "zipkin": "^0.22.0", 56 | "zipkin-instrumentation-koa": "^0.22.0" 57 | }, 58 | "devDependencies": { 59 | "babel-plugin-transform-object-rest-spread": "^6.26.0", 60 | "babel-preset-env": "^1.7.0", 61 | "babel-preset-es2015": "^6.24.1", 62 | "chai": "^4.0.0", 63 | "mocha": "^3.4.2", 64 | "nodemon": "^1.17.5", 65 | "pino-pretty": "^10.0.0", 66 | "supertest": "^3.0.0" 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /public/blank.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pwstrick/shin-server/3bdb181185a0bf58aa6e18034cb8640a0731a877/public/blank.gif -------------------------------------------------------------------------------- /routers/common.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-03 15:15:01 4 | * @LastEditTime: 2023-05-08 17:21:20 5 | * @LastEditors: strick 6 | * @Description: 通用路由 7 | * @FilePath: /strick/shin-server/routers/common.js 8 | */ 9 | import _ from "lodash"; 10 | import config from 'config'; 11 | const fs = require('fs'); 12 | const path = require('path'); 13 | import crypto from 'crypto'; 14 | import moment from 'moment'; 15 | import queue from '../utils/queue'; 16 | import { MONITOR_PROJECT } from '../utils/constant'; 17 | import services from '../services/'; 18 | 19 | 20 | export default (router) => { 21 | /** 22 | * 上传逻辑 23 | */ 24 | async function upload(ctx) { 25 | const { files, body } = ctx.request; 26 | const { dir } = body; //上传的目录 27 | const { file } = files; 28 | const stringLib = 'abcdefghijklmnopqrstuvwxyz'; 29 | const randomString = _.sampleSize(stringLib, 6); 30 | const ext = file.name.split('.').pop(); 31 | // 文件的上传目录 32 | let upDir = "upload"; 33 | if(dir) { 34 | upDir = `upload/${dir}`; 35 | } 36 | // 文件的上传路径 37 | let filePath = `${upDir}/${Date.now()}${randomString.join('')}.${ext}`; 38 | // 创建目录 39 | const absdir = path.join(__dirname, `../static/${upDir}`), 40 | absFilePath = path.join(__dirname, `../static/${filePath}`); 41 | return new Promise( resolve => { 42 | // 创建可读流 43 | const reader = fs.createReadStream(file.path); 44 | fs.mkdirSync(absdir, { recursive: true }); 45 | // 创建可写流 46 | const upStream = fs.createWriteStream(absFilePath); 47 | // 可读流通过管道写入可写流 48 | reader.pipe(upStream); 49 | reader.on('end', () => { 50 | resolve(filePath); 51 | }); 52 | }); 53 | } 54 | /** 55 | * 文件上传 56 | */ 57 | router.post( 58 | "/common/upload", 59 | async (ctx) => { 60 | // 内部走的是异步逻辑，用Promise来实现同步 61 | const filePath = await upload(ctx); 62 | ctx.body = { code: 0, key: filePath }; 63 | } 64 | ); 65 | 66 | /** 67 | * 会员详情 68 | */ 69 | router.get( 70 | "/appuser/detail", 71 | async (ctx) => { 72 | const { type, id } = ctx.query; 73 | ctx.body = { code: 0 }; 74 | } 75 | ); 76 | 77 | /** 78 | * 读取表名，查询条件或字段 79 | */ 80 | function getTableAndWhere(body) { 81 | if(Object.keys(body).length == 0) { 82 | return { tableName: null }; 83 | } 84 | const tableName = Object.keys(body)[0]; //表名 85 | const where = body[tableName]; //查询条件 86 | return { tableName, where }; 87 | } 88 | 89 | /** 90 | * 验证表名 91 | */ 92 | function validateTableName(ctx, tableName) { 93 | if(!tableName || tableName == 'undefined') { 94 | ctx.body = { code: 1, msg: '请传输表名' }; 95 | return false; 96 | } 97 | return true; 98 | } 99 | /** 100 | * 读取一条记录 101 | * { 102 | * TableName : { 查询条件 } 103 | * } 104 | * TableName是Model文件的名称，并非数据库表名 105 | */ 106 | router.post('/get', 107 | async (ctx) => { 108 | const { body } = ctx.request; 109 | const { tableName, where } = getTableAndWhere(body); 110 | if(!validateTableName(ctx, tableName)) { 111 | return; 112 | } 113 | // 将表名和查询条件传递给数据库方法 114 | const data = await services.common.getOne(tableName, where); 115 | ctx.body = { code: 0, data }; 116 | }); 117 | 118 | /** 119 | * 读取多条记录 120 | * { 121 | * TableName : { 查询条件 }, 122 | * limit, order, curPage 123 | * } 124 | */ 125 | router.post('/gets', 126 | async (ctx) => { 127 | const { body } = ctx.request; 128 | // 分页，排序，页数 129 | let { limit, order, curPage } = body; 130 | delete body.limit; 131 | delete body.order; 132 | delete body.curPage; 133 | limit = limit && (+limit); 134 | const { tableName, where } = getTableAndWhere(body); 135 | if(!validateTableName(ctx, tableName)) { 136 | return; 137 | } 138 | const { count, rows } = await services.common.getList({ 139 | tableName, 140 | where, 141 | limit, 142 | order, 143 | curPage, 144 | }); 145 | ctx.body = { code: 0, data: rows, count }; 146 | }); 147 | 148 | /** 149 | * 聚合数据 150 | * 例如count，max，min 和 sum 151 | * { 152 | * TableName : { 查询条件 }, 153 | * aggregation 聚合函数 154 | * field 聚合字段（除了count之外，都必传） 155 | * } 156 | */ 157 | router.post('/head', 158 | async (ctx) => { 159 | const { body } = ctx.request; 160 | // 聚合动作 161 | const { aggregation, field } = body; 162 | delete body.aggregation; 163 | delete body.field; 164 | const { tableName, where } = getTableAndWhere(body); 165 | if(!validateTableName(ctx, tableName)) { 166 | return; 167 | } 168 | if(aggregation != "count" && !field) { 169 | ctx.body = { code: 1, msg: '请传输聚合字段' }; 170 | return; 171 | } 172 | const data = await services.common.aggregation({ tableName, where, func: aggregation, field }); 173 | ctx.body = { code: 0, data }; 174 | }); 175 | 176 | /** 177 | * 新增 178 | * { 179 | * TableName : { 新增的字段 } 180 | * } 181 | */ 182 | router.post('/post', 183 | async (ctx) => { 184 | const { body } = ctx.request; 185 | const { tableName, where } = getTableAndWhere(body); 186 | if(!validateTableName(ctx, tableName)) { 187 | return; 188 | } 189 | if(Object.keys(where).length === 0) { 190 | ctx.body = { code: 1, msg: '请传输新增字段' }; 191 | return; 192 | } 193 | const data = await services.common.create(tableName, where); 194 | ctx.body = { code: 0, data }; 195 | }); 196 | 197 | /** 198 | * 修改 199 | * { 200 | * TableName : { 查询条件 } 201 | * set : { 更新的字段 } 202 | * } 203 | */ 204 | router.post('/put', 205 | async (ctx) => { 206 | const { body } = ctx.request; 207 | // 字段更新 208 | const { set } = body; 209 | delete body.set; 210 | if(!set) { 211 | ctx.body = { code: 1, msg: '请传输更新字段' }; 212 | return; 213 | } 214 | if(Object.keys(set).length === 0) { 215 | ctx.body = { code: 1, msg: '请传输更新字段' }; 216 | return; 217 | } 218 | const { tableName, where } = getTableAndWhere(body); 219 | if(!validateTableName(ctx, tableName)) { 220 | return; 221 | } 222 | const affected = await services.common.update(tableName, set, where); 223 | ctx.body = { code: (affected > 0 ? 0 : 1) }; 224 | }); 225 | 226 | /** 227 | * 提取路径中的地址 228 | */ 229 | function extractPath(url) { 230 | /** 231 | * 只提取路径信息，去除协议、域名和端口 232 | * 加 {2,4} 是为了解决 https://// 无法匹配的问题 233 | */ 234 | return url ? url.split('?')[0].replace(/(\w*):?\/{2,4}([^/:]+)(:\d*)?/, '').substring(1).trim() : null; 235 | } 236 | 237 | /** 238 | * 监控信息搜集 239 | */ 240 | router.post('/ma.gif', async (ctx) => { 241 | const { m, r } = JSON.parse(ctx.request.body); 242 | const params = JSON.parse(m); 243 | let { subdir = '', token, category, data, identity, referer } = params; 244 | let { type, status, url } = data; 245 | const projectSubdir = subdir; // 子目录 提前缓存 246 | // 对 Promise 的错误做特殊处理 247 | if (type === 'promise') { 248 | status = data.desc.status; 249 | url = data.desc.url; 250 | } else if (data.desc && data.desc.url) { 251 | // React Vue Runtime Crash Image 等错误也要提取 url 252 | url = data.desc.url; 253 | } 254 | const message = JSON.stringify(data); 255 | // MD5加密 256 | const key = crypto.createHash('md5').update(identity + token + category + message).digest('hex'); 257 | // 读取当前最新的 Source Map 文件 258 | let source = ''; 259 | const dir = `${process.env.NODE_ENV}-${token}`; // 存放 map 文件的目录 260 | const absDir = path.resolve(__dirname, config.get('sourceMapPath'), dir); 261 | logger.trace('sourceMapPath', absDir); 262 | // 目录存在，并且当前是错误类型的日志 263 | if (fs.existsSync(absDir) && category === 'error') { 264 | let readDir = fs.readdirSync(absDir); 265 | // 如果是 chunk-vendors 的错误，需做特殊处理 266 | if (type == 'runtime' && data.desc.prompt.indexOf('chunk-vendors') >= 0) { 267 | subdir = 'chunk-vendors'; 268 | readDir = readDir.filter(name => name.split('.')[0] === subdir); 269 | subdir += '.'; 270 | } else if (subdir) { // 当传递的subdir非空时，需要过滤进行过滤 271 | // map 文件第一个点号之前的前缀必须与 subdir 相同 272 | readDir = readDir.filter(name => name.split('.')[0] === subdir); 273 | subdir += '.'; // 用于后续的去除前缀 274 | } 275 | readDir = readDir.sort((a, b) => b.replace(subdir, '').split('.')[0] - a.replace(subdir, '').split('.')[0]); 276 | source = readDir.length > 0 ? readDir[0] : ''; 277 | } 278 | 279 | // UA信息解析 280 | // const ua = JSON.stringify(uaParser(ctx.headers['user-agent'])); 281 | const ua = ctx.headers['user-agent']; 282 | const monitor = { 283 | project: token, 284 | project_subdir: projectSubdir, 285 | category, 286 | message, 287 | key, 288 | ua, 289 | source, 290 | identity, 291 | referer, 292 | message_type: type && type.toLowerCase(), 293 | message_status: status, 294 | message_path: extractPath(url), // 提取路径 295 | day: moment().format('YYYYMMDD'), 296 | hour: moment().format('HH'), 297 | minute: moment().format('mm'), 298 | ctime: new Date(), // 当前日期 299 | }; 300 | if (r) { 301 | monitor.record = r; 302 | } 303 | const taskName = 'handleMonitor';// + Math.ceil(randomNum(0, 10) / 3); 304 | // 新增队列任务 生存时间60秒 305 | const job = queue.create(taskName, { monitor }).ttl(60000) 306 | .removeOnComplete(true); 307 | // job.on('failed', function(errorMessage){ 308 | // logger.trace(`${taskName} job faild`, errorMessage); 309 | // }); 310 | job.save((err) => { 311 | if (err) { 312 | logger.trace(`${taskName} job failed!`); 313 | } 314 | logger.trace(`${taskName} job saved!`, job.id); 315 | }); 316 | 317 | // queue.on('error', function( err ) { 318 | // logger.trace('handleMonitor queue error', err); 319 | // }); 320 | 321 | const blankUrl = path.resolve(__dirname, '../public/blank.gif'); 322 | ctx.body = fs.readFileSync(blankUrl); // 空白gif图 323 | }); 324 | 325 | /** 326 | * 性能信息搜集 327 | */ 328 | router.post('/pe.gif', async (ctx) => { 329 | let params; 330 | try { 331 | params = JSON.parse(ctx.request.body); 332 | } catch (e) { 333 | params = null; 334 | } 335 | if (!params) { 336 | ctx.body = {}; 337 | return; 338 | } 339 | // UA信息解析 340 | // const ua = JSON.stringify(uaParser(ctx.headers['user-agent'])); 341 | const ua = ctx.headers['user-agent']; 342 | const performance = { 343 | project: params.pkey, 344 | load: params.loadTime, 345 | ready: params.domReadyTime, 346 | paint: params.firstPaint, 347 | screen: params.firstScreen, 348 | identity: params.identity, 349 | ua, 350 | day: moment().format('YYYYMMDD'), 351 | hour: moment().format('HH'), 352 | minute: moment().format('mm'), 353 | referer: params.referer, // 来源地址 354 | referer_path: extractPath(params.referer), // 来源地址路径 355 | timing: params.timing ? JSON.stringify(params.timing) : null, 356 | resource: params.resource ? JSON.stringify(params.resource) : null, 357 | }; 358 | delete params.pkey; 359 | delete params.loadTime; 360 | delete params.domReadyTime; 361 | delete params.firstPaint; 362 | delete params.firstScreen; 363 | delete params.identity; 364 | delete params.referer; 365 | delete params.timing; 366 | delete params.resource; 367 | performance.measure = JSON.stringify(params); 368 | // 新增队列任务 生存时间60秒 369 | const job = queue.create('handlePerformance', { performance }).ttl(60000) 370 | .removeOnComplete(true).save((err) => { 371 | if (err) { 372 | logger.trace('handlePerformance job failed!'); 373 | } 374 | logger.trace('handlePerformance job saved!', job.id); 375 | }); 376 | job.on('failed', (errorMessage) => { 377 | logger.trace('handlePerformance job faild', errorMessage); 378 | }); 379 | // queue.on('error', function( err ) { 380 | // // console.log('handlePerformance queue error', err); 381 | // }); 382 | 383 | // // console.log(performance); 384 | // await services.common.createPerformance(); 385 | ctx.body = {}; 386 | }); 387 | 388 | /** 389 | * 删除过期的 Source Map 日志文件 390 | */ 391 | router.get('/smap/del', async (ctx) => { 392 | const { day = 21 } = ctx.query; 393 | // 删除21天前的文件 394 | const threeWeek = ~~moment().add(-day, 'days').startOf('day').format('YYYYMMDDHHmm'); 395 | // 删除文件 需要调用web-api的接口 396 | const mapPath = path.resolve(__dirname, config.get('sourceMapPath')); 397 | logger.info(`source map目录：${mapPath}`); 398 | if (!fs.existsSync(mapPath)) { 399 | logger.info('source map目录不存在'); 400 | } 401 | // 遍历项目 402 | MONITOR_PROJECT.forEach((dir) => { 403 | // 指定目录 404 | const currentDir = path.resolve(mapPath, `${process.env.NODE_ENV}-${dir}`); 405 | if (!fs.existsSync(currentDir)) { 406 | return; 407 | } 408 | const readDir = fs.readdirSync(currentDir); 409 | readDir.forEach((name) => { 410 | const num = ~~name.replace(/[^0-9]/ig, ''); 411 | // 删除过期文件 412 | if (num <= threeWeek) { 413 | const filePath = path.resolve(currentDir, name); 414 | fs.unlinkSync(filePath); 415 | } 416 | }); 417 | }); 418 | ctx.body = {}; 419 | }); 420 | } -------------------------------------------------------------------------------- /routers/index.js: -------------------------------------------------------------------------------- 1 | import requireIndex from 'es6-requireindex'; 2 | 3 | export default (router) => { 4 | const dir = requireIndex(__dirname); 5 | Object.keys(dir).forEach((item) => { 6 | dir[item](router); 7 | }); 8 | }; -------------------------------------------------------------------------------- /routers/template.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-01-04 15:08:42 4 | * @LastEditTime: 2023-04-24 17:46:53 5 | * @LastEditors: strick 6 | * @Description: 模板组件示例 7 | * @FilePath: /strick/shin-server/routers/template.js 8 | */ 9 | // import services from '../services/'; 10 | // import middlewares from '../middlewares'; 11 | export default (router) => { 12 | /** 13 | * 创建 14 | */ 15 | router.post('/template/create', async (ctx) => { 16 | ctx.body = { code: 0, msg: "错误原因" }; 17 | }); 18 | 19 | /** 20 | * 处理 21 | */ 22 | router.post('/template/handle', async (ctx) => { 23 | ctx.body = { code: 0 }; 24 | }); 25 | 26 | /** 27 | * 查询 28 | */ 29 | router.get('/template/query', async (ctx) => { 30 | ctx.body = { code: 0, data: [ 31 | { 32 | "id": "123456", 33 | "url": "http://localhost:6060/img/avatar.png", 34 | "name": "freedom" + Math.round(Math.random() * 10), 35 | "status": 0, 36 | "price": 9.8, 37 | "date": "2021-01-05T15:19:30.000Z", 38 | "udate": "2021-01-05T15:19:30.000Z", 39 | "urls": ["http://www.pwstrick.com", "https://www.cnblogs.com/strick"], 40 | "icon": [ 41 | '//www.pwstrick.com/upload/avatar.png', 42 | '//www.pwstrick.com/usr/uploads/2020/02/4250591636.jpg', 43 | ], 44 | "csv": [ 45 | {nick: "justify", uid: "1"}, 46 | {nick: "freedom", uid: "2"} 47 | ], 48 | "file": [ 49 | 'http://localhost:6060/img/avatar.png' 50 | ] 51 | }, { 52 | "id": "234567", 53 | "url": "http://localhost:6060/img/cover.jpg", 54 | "name": "justify" + Math.round(Math.random() * 10), 55 | "status": 1, 56 | "price": 18, 57 | "date": "2021-01-05T15:19:29.000Z", 58 | "udate": "2021-01-05T15:19:30.000Z", 59 | "urls": [], 60 | "icon": [], 61 | "csv": [], 62 | }, { 63 | "id": "345678", 64 | "url": "http://localhost:6060/img/avatar.png", 65 | "name": "盐汽水真好喝", 66 | "status": 2, 67 | "price": 12, 68 | "date": "2021-01-05T15:17:52.000Z", 69 | "udate": "2021-01-05T15:19:30.000Z", 70 | "urls": [], 71 | "icon": [], 72 | "csv": [], 73 | }, 74 | { 75 | "id": "4", 76 | "url": "http://localhost:6060/img/cover.jpg", 77 | "name": "jane", 78 | "status": 2, 79 | "price": 12, 80 | "date": "2021-01-05T15:17:52.000Z", 81 | "udate": "2021-01-05T15:19:30.000Z", 82 | "urls": [], 83 | "icon": [], 84 | "csv": [], 85 | }, { 86 | "id": "5", 87 | "url": "http://localhost:6060/img/avatar.png", 88 | "name": "小靖轩", 89 | "status": 2, 90 | "price": 12, 91 | "date": "2021-01-05T15:17:52.000Z", 92 | "udate": "2021-01-05T15:19:30.000Z", 93 | "urls": [], 94 | "icon": [], 95 | "csv": [], 96 | }, { 97 | "id": "6", 98 | "url": "http://localhost:6060/img/cover.jpg", 99 | "name": "凯文", 100 | "status": 2, 101 | "price": 12, 102 | "date": "2021-01-05T15:17:52.000Z", 103 | "udate": "2021-01-05T15:19:30.000Z", 104 | "urls": [], 105 | "icon": [], 106 | "csv": [], 107 | }, { 108 | "id": "7", 109 | "url": "http://localhost:6060/img/avatar.png", 110 | "name": "超级飞侠", 111 | "status": 2, 112 | "price": 12, 113 | "date": "2021-01-05T15:17:52.000Z", 114 | "udate": "2021-01-05T15:19:30.000Z", 115 | "urls": [], 116 | "icon": [], 117 | "csv": [], 118 | }, { 119 | "id": "8", 120 | "url": "http://localhost:6060/img/cover.jpg", 121 | "name": "乐迪", 122 | "status": 2, 123 | "price": 12, 124 | "date": "2021-01-05T15:17:52.000Z", 125 | "udate": "2021-01-05T15:19:30.000Z", 126 | "urls": [], 127 | "icon": [], 128 | "csv": [], 129 | }, { 130 | "id": "9", 131 | "url": "http://localhost:6060/img/avatar.png", 132 | "name": "小爱", 133 | "status": 2, 134 | "price": 12, 135 | "date": "2021-01-05T15:17:52.000Z", 136 | "udate": "2021-01-05T15:19:30.000Z", 137 | "urls": [], 138 | "icon": [], 139 | "csv": [], 140 | } 141 | ], count: Math.round(Math.random() * 100) }; 142 | }); 143 | 144 | } -------------------------------------------------------------------------------- /routers/tool.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2020-12-22 14:54:23 4 | * @LastEditTime: 2023-04-24 17:47:12 5 | * @Description: 业务工具接口 6 | * @FilePath: /strick/shin-server/routers/tool.js 7 | */ 8 | import crypto from "crypto"; 9 | import { string10to62 } from "../utils"; 10 | import services from '../services/'; 11 | import middlewares from '../middlewares'; 12 | 13 | const murmurhash = require('../utils/murmurhash'); 14 | 15 | export default (router) => { 16 | /** 17 | * 创建和编辑通用配置 18 | */ 19 | router.post( 20 | "/tool/config/create", 21 | middlewares.checkAuth("backend.tool.config"), 22 | async (ctx) => { 23 | const { id, title, content } = ctx.request.body; 24 | let result; 25 | //MD5加密 26 | const key = crypto.createHash("md5").update(title).digest("hex"); 27 | const keyData = await services.tool.getOneConfig({ key }); 28 | //编辑 29 | if (id) { 30 | const data = await services.tool.getOneConfig({ id }); 31 | //当根据key可以找到数据，并且本次修改了 title，则报错 32 | if (keyData && data.title != title) { 33 | ctx.body = { code: 1, msg: "标题已存在" }; 34 | return; 35 | } 36 | result = await services.tool.editConfig({ id, title, content }); 37 | ctx.body = { code: 0 }; 38 | return; 39 | } 40 | 41 | //当根据key可以找到数据，则报错 42 | if (keyData) { 43 | ctx.body = { code: 1, msg: "标题已存在" }; 44 | return; 45 | } 46 | //创建 47 | result = await services.tool.createConfig({ title, content, key }); 48 | ctx.body = { code: 0 }; 49 | } 50 | ); 51 | 52 | /** 53 | * 创建和编辑通用配置 54 | */ 55 | router.get( 56 | "/tool/config/query", 57 | middlewares.checkAuth("backend.tool.config"), 58 | async (ctx) => { 59 | const { rows, count } = await services.tool.getConfigList(); 60 | ctx.body = { code: 0, data: rows, count }; 61 | } 62 | ); 63 | 64 | /** 65 | * 删除通用配置 66 | */ 67 | router.post( 68 | "/tool/config/del", 69 | middlewares.checkAuth("backend.tool.config"), 70 | async (ctx) => { 71 | const { id } = ctx.request.body; 72 | await services.tool.delConfig({ id }); 73 | ctx.body = { code: 0 }; 74 | } 75 | ); 76 | 77 | /** 78 | * 创建和修改短链 79 | */ 80 | router.post('/tool/short/create', 81 | middlewares.checkAuth("backend.tool.shortChain"), 82 | async (ctx) => { 83 | const { url, id } = ctx.request.body; 84 | //存在ID 85 | if(id) { //修改 86 | let result = await services.tool.getOneShortChain({ id }); 87 | await services.tool.updateShortChain({ 88 | id, 89 | url, 90 | short: result.short 91 | }); 92 | ctx.body = { code: 0 }; 93 | return; 94 | } 95 | //murmurhash算法 96 | const short = string10to62(murmurhash.v3(url)); 97 | //根据 key 查询短链数据 98 | let result = await services.tool.getOneShortChain({ short }); 99 | if(result) { 100 | ctx.body = { code: 1, msg: "短链已存在" }; 101 | return; 102 | } 103 | //创建 104 | result = await services.tool.createShortChain({ 105 | url, 106 | short, 107 | }); 108 | ctx.body = { code: 0, data: result }; 109 | }); 110 | 111 | /** 112 | * 短链查询 113 | */ 114 | router.get('/tool/short/query', 115 | middlewares.checkAuth("backend.tool.shortChain"), 116 | async (ctx) => { 117 | const { curPage = 1, short, url } = ctx.query; 118 | const { rows, count } = await services.tool.getShortChainList({ curPage, short, url}); 119 | ctx.body = { code:0, data: rows, count }; 120 | }); 121 | 122 | /** 123 | * 短链删除 124 | */ 125 | router.post('/tool/short/del', 126 | middlewares.checkAuth("backend.tool.shortChain"), 127 | async (ctx) => { 128 | const { id } = ctx.request.body; 129 | //读取数据 130 | const data = await services.tool.getOneShortChain({ id }); 131 | const result = await services.tool.delShortChain(data); 132 | ctx.body = result > 0 ? { code: 0 } : { code: 1 }; 133 | }); 134 | }; 135 | -------------------------------------------------------------------------------- /routers/user.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 管理后台用户 3 | * 用户的账号和角色管理 4 | */ 5 | import crypto from 'crypto'; 6 | import jwt from 'jsonwebtoken'; 7 | import _ from 'lodash'; 8 | import moment from 'moment'; 9 | import config from 'config'; 10 | import { randomString } from '../utils'; 11 | import redis from '../db/redis'; 12 | import services from '../services/'; 13 | import middlewares from '../middlewares'; 14 | 15 | //5分钟内登录次数超过此值账号将被禁用 16 | const loginCountLimit = 3; 17 | 18 | /** 19 | * 加密明文密码 20 | * @param {string} userName 用户名 21 | * @param {string} password 密码 22 | * @param {string} salt 加密秘钥，每个账户都不同 23 | */ 24 | const cryptoPassword = (userName, password, salt) => { 25 | const str = userName + password + salt; 26 | return crypto.createHash('md5').update(str).digest('hex'); 27 | }; 28 | 29 | export default (router) => { 30 | // 查询当前用户信息 31 | router.get( 32 | '/user', 33 | async (ctx) => { 34 | const { realName } = ctx.state.user; 35 | ctx.body = { 36 | username: realName, 37 | }; 38 | }, 39 | ); 40 | 41 | /** 42 | * 获取用户角色列表 43 | */ 44 | router.get( 45 | '/user/role/list', 46 | middlewares.checkAuth('backend.user.role.list'), 47 | async (ctx) => { 48 | const data = ctx.query; 49 | const cursor = parseInt(data.cursor || 1); 50 | const limit = parseInt(data.limit || 10); 51 | const role = data.role; 52 | const res = await services.backendUserRole.getList(role, cursor, limit); 53 | ctx.body = { 54 | list: res.list, 55 | page: { 56 | cursor: Number(cursor), 57 | total: res.count, 58 | }, 59 | }; 60 | }, 61 | ); 62 | 63 | /** 64 | * 添加用户角色 65 | */ 66 | router.post( 67 | '/user/role', 68 | middlewares.checkAuth('backend.user.role.list'), 69 | async (ctx) => { 70 | ctx.checkBody('roleName').notEmpty('角色名称不能为空'); 71 | ctx.checkBody('roleDesc').notEmpty('角色描述不能为空'); 72 | ctx.checkBody('rolePermission').notEmpty('角色权限不能为空'); 73 | if (ctx.errors) { 74 | ctx.status = 400; 75 | ctx.body = { error: _.values(ctx.errors[0])[0] }; 76 | } else { 77 | const { roleName, roleDesc, rolePermission } = ctx.request.body; 78 | const res = await services.backendUserRole.add(roleName, roleDesc, rolePermission); 79 | ctx.body = res; 80 | } 81 | }, 82 | ); 83 | 84 | /** 85 | * 修改用户角色信息 86 | */ 87 | router.put( 88 | '/user/role', 89 | middlewares.checkAuth('backend.user.role.list'), 90 | async (ctx) => { 91 | ctx.checkBody('roleId').notEmpty('角色id不能为空'); 92 | if (ctx.errors) { 93 | ctx.status = 400; 94 | ctx.body = { error: _.values(ctx.errors[0])[0] }; 95 | } else { 96 | const data = ctx.request.body; 97 | const roleId = data.roleId; 98 | delete data.roleId; 99 | await services.backendUserRole.update(roleId, data); 100 | ctx.body = Object.assign({ roleId }, data); 101 | } 102 | }, 103 | ); 104 | 105 | /** 106 | * 禁用角色 107 | */ 108 | router.post( 109 | '/user/role/disable', 110 | middlewares.checkAuth('backend.user.role.list'), 111 | async (ctx) => { 112 | ctx.checkBody('roleId').notEmpty('角色id不能为空'); 113 | if (ctx.errors) { 114 | ctx.status = 400; 115 | ctx.body = { error: _.values(ctx.errors[0])[0] }; 116 | } else { 117 | const { roleId } = ctx.request.body; 118 | await services.backendUserRole.update(roleId, { status: 0 }); 119 | ctx.body = { roleId, status: 0 }; 120 | } 121 | }, 122 | ); 123 | 124 | /** 125 | * 启用角色 126 | */ 127 | router.post( 128 | '/user/role/enable', 129 | middlewares.checkAuth('backend.user.role.list'), 130 | async (ctx) => { 131 | ctx.checkBody('roleId').notEmpty('角色id不能为空'); 132 | if (ctx.errors) { 133 | ctx.status = 400; 134 | ctx.body = { error: _.values(ctx.errors[0])[0] }; 135 | } else { 136 | const { roleId } = ctx.request.body; 137 | await services.backendUserRole.update(roleId, { status: 1 }); 138 | ctx.body = { roleId, status: 1 }; 139 | } 140 | }, 141 | ); 142 | 143 | /** 144 | * 删除用户角色 145 | */ 146 | router.del( 147 | '/user/role', 148 | middlewares.checkAuth('backend.user.role.list'), 149 | async (ctx) => { 150 | ctx.checkBody('roleId').notEmpty('角色id不能为空'); 151 | if (ctx.errors) { 152 | ctx.status = 400; 153 | ctx.body = { error: _.values(ctx.errors[0])[0] }; 154 | return; 155 | } 156 | const { roleId } = ctx.request.body; 157 | await services.backendUserRole.remove(roleId); 158 | ctx.body = { roleId }; 159 | }, 160 | ); 161 | 162 | /** 163 | * 注册 164 | */ 165 | router.post( 166 | '/user', 167 | async (ctx) => { 168 | ctx.checkBody('userName').isEmail('请填写正确的邮箱'); 169 | ctx.checkBody('password').notEmpty('密码不能为空').len(6); 170 | ctx.checkBody('realName').notEmpty('真实姓名不能为空'); 171 | ctx.checkBody('cellphone').isMobilePhone('请填写正确的手机号码', ['zh-CN']); 172 | ctx.checkBody('roles').notEmpty('用户角色不能为空'); 173 | if (ctx.errors) { 174 | ctx.status = 400; 175 | ctx.body = { error: _.values(ctx.errors[0])[0] }; 176 | return; 177 | } 178 | const { userName, password, realName, cellphone, roles } = ctx.request.body; 179 | // 检查邮箱是否已注册 180 | const emailCheck = await services.backendUserAccount.checkEmailExist(userName); 181 | if (emailCheck) { 182 | ctx.status = 400; 183 | ctx.body = { error: '邮箱已注册' }; 184 | return; 185 | } 186 | // 检查手机是否已注册 187 | const cellphoneCheck = await services.backendUserAccount.checkCelphoneExist(cellphone); 188 | if (cellphoneCheck) { 189 | ctx.status = 400; 190 | ctx.body = { error: '手机号已注册' }; 191 | return; 192 | } 193 | const data = ctx.request.body; 194 | const salt = randomString(32); 195 | data.salt = salt; 196 | data.password = cryptoPassword(userName, password, salt); 197 | const res = await services.backendUserAccount.register(data); 198 | ctx.body = { 199 | _id: res._id, 200 | userName, 201 | realName, 202 | status: 1, 203 | roles, 204 | cellphone, 205 | }; 206 | }, 207 | ); 208 | 209 | /** 210 | * 登录 211 | */ 212 | router.post( 213 | '/user/login', 214 | async (ctx) => { 215 | // 验证参数 216 | ctx.checkBody('__userName__').notEmpty('用户名不能为空'); 217 | ctx.checkBody('__password__').notEmpty('密码不能为空'); 218 | if (ctx.errors) { 219 | ctx.body = { code: 400, msg: ctx.errors }; 220 | return; 221 | } 222 | 223 | // 获取用户信息 224 | const { __userName__, __password__ } = ctx.request.body; 225 | const userName = __userName__; 226 | const password = __password__; 227 | const res = await services.backendUserAccount.getInfoByUserName(userName); 228 | 229 | if (!res) { 230 | ctx.body = { code: 400, msg: '用户名不存在' }; 231 | return; 232 | } 233 | 234 | if (res.status === 0) { 235 | ctx.body = { code: 400, msg: '您的账号已被禁用' }; 236 | return; 237 | } 238 | 239 | // 检查登录计数 240 | const loginCount = await services.backendUserAccount.getLoginCount(userName); 241 | if (loginCount > loginCountLimit) { 242 | await services.backendUserAccount.updateStatus(res._id, 0); 243 | await services.backendUserAccount.resetLoginCount(userName); 244 | ctx.body = { code: 400, msg: '登陆错误次数超过限制，请联系管理员解封' }; 245 | return; 246 | } 247 | if (loginCount === null) { 248 | await services.backendUserAccount.setLoginCount(userName); 249 | } 250 | // 登录计数 +1 251 | await services.backendUserAccount.increaseLoginCount(userName); 252 | 253 | // 验证密码 254 | const pwd = cryptoPassword(userName, password, res.salt); 255 | if (pwd === res.password) { 256 | // 获取当前账号的角色权限 257 | const promises = res.roles.map(item => services.backendUserRole.getInfoById(item)); 258 | let roles = await Promise.all(promises); 259 | roles = _.filter(roles, role => role); 260 | // 检查角色状态 261 | let disabledPermissions = []; 262 | roles.forEach((role) => { 263 | if (role.status === 0) { 264 | disabledPermissions = disabledPermissions.concat(role.rolePermission); 265 | } 266 | }); 267 | const authorities = _.chain(roles.map(item => item.rolePermission)) 268 | .flatten(true) 269 | .filter(role => !disabledPermissions.includes(role)) 270 | .uniq() 271 | .value(); 272 | 273 | // 生成token 274 | const token = jwt.sign({ 275 | id: res._id, 276 | // authorities, 277 | userName: res.userName, 278 | realName: res.realName, 279 | }, config.get('jwtSecret'), { 280 | expiresIn: '12h', 281 | }); 282 | 283 | //测试环境 将token放到Redis中 284 | const nodeEnv = process.env.NODE_ENV; 285 | nodeEnv === "development" && (await redis.aws.set(`backend:user:account:token:development`, token)); 286 | 287 | // 权限放入redis存储 288 | await redis.aws.set(`backend:user:account:authorities:${res._id}`, authorities.toString()); 289 | 290 | // 密码过期时间通知 291 | let expireDays = 90; 292 | // const passwordExpireTime = res.passwordExpireTime; 293 | // if (passwordExpireTime) { 294 | // if (moment().isBefore(moment(passwordExpireTime), 'd')) { 295 | // const days = moment(passwordExpireTime).diff(moment(), 'd'); 296 | // expireDays = days > 0 ? days : 0; 297 | // } else { 298 | // ctx.body = { code: 400, msg: '密码过期' }; 299 | // return; 300 | // } 301 | // } else { 302 | // await services.backendUserAccount.updateInfo(res._id, { passwordExpireTime: moment().add(90, 'd').endOf('d') }); 303 | // } 304 | ctx.body = { code:0, token, authorities, nodeEnv, expireDays }; 305 | } else { 306 | ctx.body = { code: 400, msg: '密码错误' }; 307 | } 308 | }, 309 | ); 310 | 311 | /** 312 | * 注销 313 | */ 314 | router.post( 315 | '/user/logout', 316 | async (ctx) => { 317 | const { id, realName } = ctx.state.user; 318 | // 删除权限redis 319 | await redis.aws.del(`backend:user:account:authorities:${id}`); 320 | ctx.body = { 321 | code: 0, 322 | }; 323 | }, 324 | ); 325 | 326 | /** 327 | * 获取用户列表 328 | * 通过角色id获取角色名称 329 | */ 330 | router.get( 331 | '/user/list', 332 | middlewares.checkAuth('backend.user.account.list'), 333 | async (ctx) => { 334 | const data = ctx.query; 335 | const curPage = data.curPage || 1; 336 | const limit = data.limit || 10; 337 | const keywords = data.keywords; 338 | const roleId = data.roleId; 339 | const res = await services.backendUserAccount.getList(curPage, limit, keywords, roleId); 340 | ctx.body = { 341 | data: res.list, 342 | count: res.count, 343 | }; 344 | }, 345 | ); 346 | 347 | /** 348 | * 查询指定用户详情 349 | */ 350 | router.get( 351 | '/user/detail/:id', 352 | middlewares.checkAuth('backend.user.account.list'), 353 | async (ctx) => { 354 | const userId = ctx.params.id; 355 | const userData = await services.backendUserAccount.getDetail(userId); 356 | ctx.body = userData; 357 | }, 358 | ); 359 | 360 | /** 361 | * 更新用户信息 362 | * 不包括密码修改 363 | */ 364 | router.put( 365 | '/user', 366 | middlewares.checkAuth('backend.user.account'), 367 | async (ctx) => { 368 | ctx.checkBody('id').notEmpty('用户id不能为空'); 369 | ctx.checkBody('realName').notEmpty('真实姓名不能为空'); 370 | ctx.checkBody('cellphone').isMobilePhone('请填写正确的手机号码', ['zh-CN']); 371 | ctx.checkBody('roles').notEmpty('用户角色不能为空'); 372 | if (ctx.errors) { 373 | ctx.status = 400; 374 | ctx.body = { error: _.values(ctx.errors[0])[0] }; 375 | return; 376 | } 377 | const data = ctx.request.body; 378 | const { id, userName, cellphone } = data; 379 | // 检查邮箱是否重复 380 | const emailCheck = await services.backendUserAccount.checkEmailExist(userName); 381 | if (emailCheck && (emailCheck._id.toString() !== id)) { 382 | ctx.status = 400; 383 | ctx.body = { error: '邮箱已注册' }; 384 | return; 385 | } 386 | // 检查手机是否已注册 387 | const cellphoneCheck = await services.backendUserAccount.checkCelphoneExist(cellphone); 388 | if (cellphoneCheck && (cellphoneCheck._id.toString() !== id)) { 389 | ctx.status = 400; 390 | ctx.body = { error: '手机号已注册' }; 391 | return; 392 | } 393 | delete data.id; 394 | await services.backendUserAccount.updateInfo(id, data); 395 | ctx.body = Object.assign({ id }, data); 396 | }, 397 | ); 398 | 399 | /** 400 | * 启用用户 401 | */ 402 | router.post( 403 | '/user/enable', 404 | middlewares.checkAuth('backend.user.account'), 405 | async (ctx) => { 406 | ctx.checkBody('id').notEmpty('id不能为空'); 407 | if (ctx.errors) { 408 | ctx.status = 400; 409 | ctx.body = { error: _.values(ctx.errors[0])[0] }; 410 | return; 411 | } 412 | const id = ctx.request.body.id; 413 | await services.backendUserAccount.updateStatus(id, 1); 414 | ctx.body = { 415 | id, 416 | status: 1, 417 | }; 418 | }, 419 | ); 420 | 421 | /** 422 | * 禁用用户 423 | */ 424 | router.post( 425 | '/user/disable', 426 | middlewares.checkAuth('backend.user.account'), 427 | async (ctx) => { 428 | ctx.checkBody('id').notEmpty('id不能为空'); 429 | if (ctx.errors) { 430 | ctx.status = 400; 431 | ctx.body = { error: _.values(ctx.errors[0])[0] }; 432 | return; 433 | } 434 | const id = ctx.request.body.id; 435 | await services.backendUserAccount.updateStatus(id, 0); 436 | ctx.body = { 437 | id, 438 | status: 0, 439 | }; 440 | }, 441 | ); 442 | 443 | /** 444 | * 修改密码 445 | */ 446 | router.put( 447 | '/user/password', 448 | async (ctx) => { 449 | ctx.checkBody('password').notEmpty('密码不能为空'); 450 | if (ctx.errors) { 451 | ctx.body = { code: 400, msg: _.values(ctx.errors[0])[0] }; 452 | return; 453 | } 454 | const { password } = ctx.request.body; 455 | let { id, userName } = ctx.request.body; 456 | if (!id || !userName) { 457 | id = ctx.state.user.id; 458 | userName = ctx.state.user.userName; 459 | } 460 | // 增加密码是否重复判断 461 | const res = await services.backendUserAccount.getInfoByUserName(userName); 462 | if (!res) { 463 | ctx.body = { code: 400, msg: '用户名不存在' }; 464 | return; 465 | } 466 | 467 | if (res.status === 0) { 468 | ctx.body = { code: 400, msg: '您的账号已被禁用' }; 469 | return; 470 | } 471 | // 验证密码 472 | const pwdRes = cryptoPassword(userName, password, res.salt); 473 | if (pwdRes === res.password) { 474 | ctx.body = { code: 400, msg: '新密码与原密码相同' }; 475 | return; 476 | } 477 | const salt = randomString(32); 478 | const pwd = cryptoPassword(userName, password, salt); 479 | // 修改密码后 有效期重置设置90天 480 | await services.backendUserAccount.updateInfo(id, { salt, password: pwd, passwordExpireTime: moment().add(90, 'd').endOf('d') }); 481 | ctx.body = { code: 0 }; 482 | }, 483 | ); 484 | 485 | /** 486 | * 删除用户 487 | */ 488 | router.del( 489 | '/user', 490 | middlewares.checkAuth('backend.user.account'), 491 | async (ctx) => { 492 | ctx.checkBody('userId').notEmpty('id不能为空'); 493 | if (ctx.errors) { 494 | ctx.status = 400; 495 | ctx.body = { error: _.values(ctx.errors[0])[0] }; 496 | return; 497 | } 498 | const id = ctx.request.body.userId; 499 | await services.backendUserAccount.delete(id); 500 | ctx.body = { 501 | id, 502 | }; 503 | }, 504 | ); 505 | 506 | /** 507 | * 初始化管理员用户 508 | */ 509 | router.get( 510 | '/user/init', 511 | async (ctx) => { 512 | // 检查当前数据库状态 513 | const users = await services.backendUserAccount.find(); 514 | if (users.length > 0) { 515 | ctx.status = 400; 516 | ctx.body = { error: '已经初始化过了', users }; 517 | return; 518 | } 519 | // 创建管理员角色 520 | const roleName = '超级管理员'; 521 | const roleDesc = '超级管理员'; 522 | const rolePermission = '*'; 523 | // console.log('创建管理员角色...'); 524 | const res = await services.backendUserRole.add(roleName, roleDesc, rolePermission); 525 | const roleId = res._id.toString(); 526 | // console.log(`创建管理员角色成功, roleId = ${roleId}`); 527 | // 创建管理员账号 528 | // console.log('创建管理员账号...'); 529 | const initPassword = 'admin'; //可自定义密码 530 | const userName = 'admin@shin.com'; //可自定义初始化 531 | const salt = randomString(32); 532 | const createData = { 533 | userName, 534 | realName: '超级管理员', 535 | password: cryptoPassword(userName, initPassword, salt), 536 | salt, 537 | roles: [roleId], 538 | }; 539 | await services.backendUserAccount.register(createData); 540 | // console.log(`创建管理员账号成功, accountId = ${account._id}`); 541 | ctx.body = `创建管理员账号成功, 请用 ${userName} : ${initPassword} 登录`; 542 | }, 543 | ); 544 | }; 545 | -------------------------------------------------------------------------------- /routers/webMonitor.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-25 15:40:31 4 | * @LastEditTime: 2023-04-24 17:47:37 5 | * @LastEditors: strick 6 | * @Description: 7 | * @FilePath: /strick/shin-server/routers/webMonitor.js 8 | */ 9 | import config from 'config'; 10 | import moment from 'moment'; 11 | import path from 'path'; 12 | import crypto from "crypto"; 13 | const sourceMap = require("source-map"); 14 | const fs = require("fs"); 15 | import { PERFORMANCT_RATE } from "../utils/constant"; 16 | import { formatDate } from '../utils/tools'; 17 | import uaParser from "ua-parser-js"; 18 | import services from '../services/'; 19 | import middlewares from '../middlewares'; 20 | 21 | export default (router) => { 22 | /** 23 | * 读取指定的Source-Map文件 24 | */ 25 | function readSourceMap(filePath) { 26 | let parsedData = null; 27 | try { 28 | parsedData = fs.readFileSync(filePath, 'utf8'); 29 | parsedData = JSON.parse(parsedData); 30 | }catch(e) { 31 | logger.info(`sourceMap：error`); 32 | } 33 | return parsedData; 34 | } 35 | /** 36 | * 对Vue的错误做特殊处理 37 | */ 38 | function handleVue(message) { 39 | const stacks = message.stack.split("\n"); 40 | if(stacks.length <= 1) 41 | return; 42 | let coordinate 43 | if(stacks[0] === message.desc) { 44 | stacks[1] = stacks[1].replace(/\)/g, ""); 45 | coordinate = stacks[1].split(":"); 46 | }else { 47 | stacks[0] = stacks[0].replace(/\)/g, ""); 48 | coordinate = stacks[0].split(":"); 49 | } 50 | if(coordinate.length <= 2) { 51 | return; 52 | } 53 | message.lineno = ~~coordinate[coordinate.length - 2]; 54 | message.colno = ~~coordinate[coordinate.length - 1]; 55 | } 56 | /** 57 | * 处理映射逻辑 58 | */ 59 | async function getSourceMap(row) { 60 | // 拼接映射文件的地址 61 | // const url = config.get("services").webHttpsApi + '/smap/' + process.env.NODE_ENV + '-' + row.project + '/' + row.source; 62 | const filePath = path.resolve(__dirname, config.get("sourceMapPath"), process.env.NODE_ENV + '-' + row.project + '/' + row.source); 63 | logger.info(`filePath：${filePath}`); 64 | // logger.info(`sourceMap：${url}`); 65 | let { message } = row; 66 | message = JSON.parse(message); 67 | // logger.info(`sourceMap：${message}`); 68 | // VUE 错误需要特殊处理 69 | if(message.type === "vue") { 70 | handleVue(message); 71 | row.message = JSON.stringify(message); 72 | } 73 | // 不存在行号或列号 74 | if(!message.lineno || !message.colno) { 75 | return row; 76 | } 77 | logger.info(`sourceMap`); 78 | // 打包后的sourceMap文件 79 | // const rawSourceMap = await httpSourceMap(url); 80 | const rawSourceMap = readSourceMap(filePath); 81 | if(!rawSourceMap) { 82 | return row; 83 | } 84 | const errorPos = { 85 | line: message.lineno, 86 | column: message.colno, 87 | }; 88 | // 过传入打包后的代码位置来查询源代码的位置 89 | const consumer = await new sourceMap.SourceMapConsumer(rawSourceMap); 90 | // 获取出错代码在哪一个源文件及其对应位置 91 | const originalPosition = consumer.originalPositionFor({ 92 | line: errorPos.line, 93 | column: errorPos.column, 94 | }); 95 | // 根据源文件名寻找对应源文件 96 | const sourceIndex = consumer.sources.findIndex( 97 | (item) => item === originalPosition.source 98 | ); 99 | const sourceCode = consumer.sourcesContent[sourceIndex]; 100 | logger.trace('sourceCode', !!sourceCode); 101 | if(sourceCode) { 102 | row.sourceInfo = { 103 | code: sourceCode, 104 | lineno: originalPosition.line, 105 | path: originalPosition.source 106 | }; 107 | // row.sourceCode = sourceCode; 108 | // row.lineno = originalPosition.line; 109 | } 110 | // 销毁，否则会报内存访问超出范围 111 | consumer.destroy(); 112 | return row; 113 | } 114 | function initMonitorQuery(ctx) { 115 | // 图表默认已小时计算 116 | const { id, category, project, start, end, curPage = 1, 117 | pageSize = 10, identity, match=1, chart=2, status, path } = ctx.query; 118 | const other = {}; 119 | const messages = []; 120 | // if(msg) { 121 | // // 全文检索时还需处理特殊字符的转义 122 | // match == 1 ? 123 | // messages.push(`+*${msg.toLocaleLowerCase().replace(/"/g, "")}*`) : 124 | // messages.push(`%${msg}%`); 125 | // } 126 | let messageType; 127 | let currentCategory = category; 128 | //传递类别以及其子类别 129 | if(Array.isArray(category)) { 130 | currentCategory = category[0]; 131 | messageType = category[1]; 132 | } 133 | return { 134 | id, currentCategory, messages, messageType, messageStatus: status, messagePath: path, 135 | project, start, end, curPage, pageSize, identity, match, chart, other 136 | } 137 | } 138 | /** 139 | * 解析代理信息 140 | */ 141 | function parseAgent(item) { 142 | if(item.ua.indexOf("{") == -1) 143 | item.ua = JSON.stringify(uaParser(item.ua)); 144 | } 145 | /** 146 | * 监控日志明细 147 | */ 148 | router.get( 149 | '/monitor/list', 150 | middlewares.checkAuth('backend.monitor.log'), 151 | async (ctx) => { 152 | const { id, currentCategory, project, start, end, 153 | curPage, pageSize, identity, match, other, 154 | messageType, messageStatus, messagePath } = initMonitorQuery(ctx); 155 | const { rows, count } = await services.webMonitor.getMonitorList({ 156 | id, 157 | category: currentCategory, 158 | msg: ctx.query.msg, 159 | messageType, messageStatus, messagePath, 160 | match, 161 | project, 162 | start, 163 | end, 164 | curPage, 165 | pageSize, 166 | identity, 167 | other 168 | }); 169 | // 查找出错误的原始代码 170 | const sourceRows = await Promise.all(rows.map(item => { 171 | // UA转换 172 | parseAgent(item); 173 | //错误类型，有映射文件 174 | if(item.category === 'error' && item.source) { 175 | return getSourceMap(item); 176 | } 177 | return item; 178 | })); 179 | ctx.body = { code: 0, data: sourceRows, count, query: ctx.query }; 180 | }, 181 | ); 182 | /** 183 | * 监控日志明细图表 184 | */ 185 | router.get( 186 | '/monitor/list/chart', 187 | middlewares.checkAuth('backend.monitor.log'), 188 | async (ctx) => { 189 | const { id, currentCategory, project, start, end, 190 | identity, chart, messageType, messageStatus, messagePath } = initMonitorQuery(ctx); 191 | let attribute, days=[]; 192 | switch(+chart) { 193 | case 1: //按天 194 | attribute = "day"; 195 | break; 196 | case 2: //按小时 197 | //以开始时间作为 day 的值 198 | if(start) { 199 | //先计算出两天的相差的天数，只取年月日 200 | let diff = moment(end.split(" ")[0]).diff(moment(start.split(" ")[0]), 'days') 201 | for(let i=0; i<=diff; i++) { 202 | days.push(moment(start).add(i, "days").format("YYYYMMDD")); 203 | } 204 | }else { 205 | //默认处理 206 | days.push(moment().format("YYYYMMDD")); 207 | } 208 | attribute = "hour"; 209 | break; 210 | } 211 | if(days.length === 0) { 212 | const counts = await services.webMonitor.getMonitorListChart({ 213 | id, 214 | category: currentCategory, 215 | msg: ctx.query.msg, 216 | messageType, messageStatus, messagePath, 217 | project, 218 | start, 219 | end, 220 | identity, 221 | attribute 222 | }); 223 | const x = counts.map(value => value.hour); 224 | const y = counts.map(value => value.count); 225 | ctx.body = { code: 0, data: { 226 | x, y 227 | }}; 228 | return; 229 | } 230 | const coor = { 231 | x: [], 232 | y: [] 233 | }; 234 | //遍历 235 | for(let day of days) { 236 | const counts = await services.webMonitor.getMonitorListChart({ 237 | id, 238 | category: currentCategory, 239 | msg: ctx.query.msg, 240 | messageType, messageStatus, messagePath, 241 | project, 242 | start, 243 | end, 244 | identity, 245 | day, 246 | attribute 247 | }); 248 | // console.log(counts) 249 | const substr = day.substr(4); 250 | const x = counts.map((value, index) => (index === 0 ? (substr + value.hour) : value.hour)); 251 | const y = counts.map(value => value.count); 252 | coor.x = coor.x.concat(x); 253 | coor.y = coor.y.concat(y); 254 | } 255 | ctx.body = { code: 0, data: coor}; 256 | }, 257 | ); 258 | 259 | /** 260 | * 日志上下文查询 261 | */ 262 | router.get( 263 | '/monitor/context', 264 | middlewares.checkAuth('backend.monitor.log'), 265 | async (ctx) => { 266 | let { prevId, nextId } = ctx.query; 267 | prevId = +prevId; //类型转换 268 | nextId = +nextId; //类型转换 269 | let rows = []; 270 | if(prevId) { 271 | // 读取前10条 272 | rows = await services.webMonitor.getMonitorContext({ from: prevId, to: prevId+9 }); 273 | }else if(nextId) { 274 | // 读取后10条 275 | rows = await services.webMonitor.getMonitorContext({ from: nextId-9, to: nextId }); 276 | } 277 | ctx.body = { code: 0, data: rows }; 278 | }, 279 | ); 280 | 281 | /** 282 | * 统计日志趋势图 283 | */ 284 | router.get( 285 | '/monitor/chart', 286 | middlewares.checkAuth('backend.monitor.log'), 287 | async (ctx) => { 288 | let { project, start, end } = ctx.query; 289 | //默认展示前面7天的数据 290 | if(!start || !end) { 291 | start = moment().add(-8, 'days').format("YYYYMMDD"); 292 | end = moment().add(-1, 'days').format("YYYYMMDD"); 293 | } 294 | const list = await services.webMonitor.getStatisList({ start, end }); 295 | if(!list) { 296 | ctx.body = { code: 0, data: {}}; 297 | return; 298 | } 299 | //保存趋势数值 300 | const days = []; 301 | const daysErrorHash = {}, 302 | days500ErrorHash = {}, 303 | days502ErrorHash = {}, 304 | days504ErrorHash = {}, 305 | daysAllCountHash = {}; 306 | list.forEach(current => { 307 | const digits = current.date.toString().split("").splice(-4); 308 | const today = digits.slice(0, 2).join("") + '-' + digits.slice(-2).join(""); 309 | days.push(today); //将数字格式化成日期 例如03-18 310 | let statis = JSON.parse(current.statis); //格式化统计数据 311 | statis = caculateFiled(project, statis); 312 | daysErrorHash[today] = statis.errorCount; 313 | days500ErrorHash[today] = statis.error500Count; 314 | days502ErrorHash[today] = statis.error502Count; 315 | days504ErrorHash[today] = statis.error504Count; 316 | daysAllCountHash[today] = statis.allCount || (statis.errorCount + statis.ajaxCount + statis.consoleCount + statis.eventCount + statis.redirectCount); 317 | }); 318 | ctx.body = { code: 0, data: { 319 | daysErrorHash, 320 | days500ErrorHash, 321 | days502ErrorHash, 322 | days504ErrorHash, 323 | daysAllCountHash 324 | }}; 325 | } 326 | ); 327 | 328 | /** 329 | * 按天读取监控日志 330 | */ 331 | router.get( 332 | '/monitor/date', 333 | middlewares.checkAuth('backend.monitor.log'), 334 | async (ctx) => { 335 | let { project, date } = ctx.query; 336 | const row = await services.webMonitor.getOneStatis({ date }); 337 | if(!row) { 338 | return ctx.body = { code: 0, data: {}}; 339 | } 340 | row.statis = JSON.parse(row.statis); 341 | ctx.body = { code: 0, data: row.statis[project]}; 342 | } 343 | ); 344 | 345 | /** 346 | * 计算字段的累加值 347 | */ 348 | function caculateFiled(project, statis) { 349 | if(project) { 350 | statis = statis[project]; //若选择了项目，则展示项目信息 351 | }else { 352 | //否则将各个项目的数值累加 353 | const calc = {}; 354 | Object.keys(statis).forEach(pro => { 355 | Object.keys(statis[pro]).forEach(field => { 356 | // 若不存在就重新赋值 357 | if(!calc[field]) { 358 | calc[field] = statis[pro][field]; 359 | return; 360 | } 361 | // 否则累加 362 | calc[field] += statis[pro][field]; 363 | }); 364 | }); 365 | statis = calc; 366 | } 367 | return statis; 368 | } 369 | /** 370 | * 统计日志信息 371 | */ 372 | router.get( 373 | '/monitor/statistic', 374 | middlewares.checkAuth('backend.monitor.log'), 375 | async (ctx) => { 376 | const { project } = ctx.query; 377 | //昨日的统计信息 378 | const yesterdayDate = moment().add(-1, 'days').format("YYYYMMDD"); //格式化的昨天日期 379 | let yesterdayStatis = await services.webMonitor.getOneStatis({ date: yesterdayDate }); 380 | if(!yesterdayStatis) { 381 | yesterdayStatis = {}; 382 | }else { 383 | yesterdayStatis = JSON.parse(yesterdayStatis.statis); //格式化存储的数据 384 | yesterdayStatis = caculateFiled(project, yesterdayStatis); 385 | } 386 | const { 387 | allCount=0, 388 | errorCount=0, 389 | errorSum=0, 390 | error500Count=0, 391 | error502Count=0, 392 | error504Count=0, 393 | ajaxCount=0, 394 | consoleCount=0, 395 | eventCount=0, 396 | redirectCount=0 397 | } = yesterdayStatis; 398 | //其他日期 399 | const yesterday = moment().add(-1, 'days').startOf('day').format("YYYY-MM-DD HH:mm"); //昨天凌晨 400 | const yesterdayNow = moment().add(-1, 'days').format("YYYY-MM-DD HH:mm"); //昨天现在的时间 401 | const today = moment().startOf('day').format("YYYY-MM-DD HH:mm"); //今天凌晨 402 | const tomorrow = moment().add(1, 'days').startOf('day').format("YYYY-MM-DD HH:mm"); //明天凌晨 403 | const todayFilter = { project, from: today, to: tomorrow }; 404 | const yesterdayFilter = { project, from: yesterday, to: yesterdayNow }; 405 | // const yesterdayAllDayFilter = { project, from: yesterday, to: today }; 406 | 407 | //日志数 408 | const todayCount = await services.webMonitor.statisticCount(todayFilter); 409 | const yesterdayCount = allCount ? allCount : (errorCount + ajaxCount + consoleCount + eventCount + redirectCount); 410 | 411 | //错误数 412 | const yesterdayErrorCount = await services.webMonitor.statisticCount({ category: "error", ...yesterdayFilter }); 413 | const yesterdayErrorSum = await services.webMonitor.statisticSum({ field: "digit", category: "error", ...yesterdayFilter }); 414 | const todayErrorCount = await services.webMonitor.statisticCount({ category: "error", ...todayFilter }); 415 | const todayErrorSum = await services.webMonitor.statisticSum({ field: "digit", category: "error", ...todayFilter }); 416 | const todayErrorCountRate = yesterdayErrorCount ? (todayErrorCount - yesterdayErrorCount) / yesterdayErrorCount * 100 : 0; 417 | const todayErrorSumRate = yesterdayErrorSum ? (todayErrorSum - yesterdayErrorSum) / yesterdayErrorSum * 100 : 0; 418 | 419 | //通信错误数 420 | // const yesterday50XErrorCountFilter = { category: "error", ...yesterdayAllDayFilter }; 421 | const today50XErrorCountFilter = { category: "error", ...todayFilter }; 422 | const message500 = { message_status: 500 }; 423 | const message502 = { message_status: 502 }; 424 | const message504 = { message_status: 504 }; 425 | const yesterday500ErrorCount = error500Count; 426 | const yesterday502ErrorCount = error502Count; 427 | const yesterday504ErrorCount = error504Count; 428 | const today500ErrorCount = await services.webMonitor.statisticCount({ ...today50XErrorCountFilter, other: message500 }); 429 | const today502ErrorCount = await services.webMonitor.statisticCount({ ...today50XErrorCountFilter, other: message502 }); 430 | const today504ErrorCount = await services.webMonitor.statisticCount({ ...today50XErrorCountFilter, other: message504 }); 431 | 432 | //通信、打印、事件、跳转数 433 | const todayAjaxCount = await services.webMonitor.statisticCount({ category: "ajax", ...todayFilter }); 434 | const todayConsoleCount = await services.webMonitor.statisticCount({ category: "console", ...todayFilter }); 435 | const todayEventCount = await services.webMonitor.statisticCount({ category: "event", ...todayFilter }); 436 | const todayRedirectCount = await services.webMonitor.statisticCount({ category: "redirect", ...todayFilter }); 437 | 438 | ctx.body = { code: 0, data: { 439 | todayCount, 440 | yesterdayCount, 441 | todayErrorCount, 442 | todayErrorSum, 443 | todayAjaxCount, 444 | todayConsoleCount, 445 | todayEventCount, 446 | todayRedirectCount, 447 | todayErrorCountRate: todayErrorCountRate.toFixed(2), 448 | todayErrorSumRate: todayErrorSumRate.toFixed(2), 449 | yesterday500ErrorCount, 450 | yesterday502ErrorCount, 451 | yesterday504ErrorCount, 452 | today500ErrorCount, 453 | today502ErrorCount, 454 | today504ErrorCount, 455 | }}; 456 | }, 457 | ); 458 | 459 | 460 | /** 461 | * 创建性能监控项目 462 | */ 463 | router.post( 464 | "/monitor/performance/project/create", 465 | middlewares.checkAuth("backend.monitor.performance.project"), 466 | async (ctx) => { 467 | const { name, id } = ctx.request.body; 468 | if(id) { 469 | const exist = await services.webMonitor.getOnePerformanceProject({ 470 | name, 471 | id: { 472 | $not: id 473 | } 474 | }); 475 | if(exist) { 476 | ctx.body = { code: 1, msg: '项目已存在' }; 477 | return; 478 | } 479 | // 更新操作 480 | await services.webMonitor.updatePerformanceProject(id, { name }); 481 | ctx.body = { code: 0 }; 482 | return; 483 | } 484 | // MD5加密 取前面的16位 485 | const key = crypto.createHash('md5').update(name).digest('hex').substring(0, 16); 486 | const keyData = await services.webMonitor.getOnePerformanceProject({ key }); 487 | // 当根据key可以找到数据，则报错 488 | if (keyData) { 489 | ctx.body = { code: 1, msg: '项目已存在' }; 490 | return; 491 | } 492 | // 创建 493 | await services.webMonitor.createPerformanceProject({ name, key }); 494 | ctx.body = { code: 0 }; 495 | } 496 | ); 497 | 498 | /** 499 | * 性能项目列表 500 | */ 501 | router.get( 502 | "/monitor/performance/project/list", 503 | middlewares.checkAuth("backend.monitor.performance.project"), 504 | async (ctx) => { 505 | const { curPage = 1, pageSize = 10, name } = ctx.query; 506 | const { count, rows } = await services.webMonitor.getPerformanceProjectList({ name, curPage, pageSize}); 507 | ctx.body = { 508 | code: 0, 509 | data: rows.map((item) => { 510 | item.ctime = formatDate(item.ctime); 511 | return item; 512 | }), 513 | count 514 | }; 515 | } 516 | ); 517 | 518 | /** 519 | * 删除性能项目 520 | */ 521 | router.post( 522 | "/monitor/performance/project/del", 523 | middlewares.checkAuth("backend.monitor.performance.project"), 524 | async (ctx) => { 525 | const { id } = ctx.request.body; 526 | await services.webMonitor.delPerformanceProject({ id }); 527 | ctx.body = { code: 0 }; 528 | } 529 | ); 530 | 531 | function getFloorNumber(count) { 532 | return Math.floor(count * PERFORMANCT_RATE); 533 | } 534 | 535 | /** 536 | * 读取性能的历史记录 537 | */ 538 | async function getHistoryList({ project, start, end, hour }) { 539 | const list = []; 540 | for(let day = start; day <= end; day++) { 541 | const statistic = { 542 | x: [], 543 | load: [], 544 | ready: [], 545 | paint: [], 546 | screen: [], 547 | loadZero: 0, //load时间为0的个数 548 | all: 0, //日志总数 549 | day 550 | }; 551 | const history = await services.webMonitor.getOnePerformanceStatis({ date: day, project }); 552 | //不存在历史记录就返回默认信息 553 | if(!history) { 554 | continue; 555 | } 556 | const historyStatis = JSON.parse(history.statis); 557 | let loadList, readyList, paintList, screenList; 558 | //如果未传递小时 559 | if(!hour) { 560 | const { x, load, ready, paint, screen, loadZero, all=0 } = historyStatis.hour; 561 | //补全24小时 562 | for(let i=0; i<=23; i++) { 563 | statistic.x.push(i); 564 | statistic.load.push(0); 565 | statistic.ready.push(0); 566 | statistic.paint.push(0); 567 | statistic.screen.push(0); 568 | } 569 | loadList = await services.webMonitor.getPerformanceListByIds(load); 570 | readyList = await services.webMonitor.getPerformanceListByIds(ready); 571 | paintList = await services.webMonitor.getPerformanceListByIds(paint); 572 | screenList = await services.webMonitor.getPerformanceListByIds(screen); 573 | x.forEach(value => { 574 | //UA转换 575 | parseAgent(loadList[0]); 576 | parseAgent(readyList[0]); 577 | parseAgent(paintList[0]); 578 | parseAgent(screenList[0]); 579 | statistic.load[value] = loadList[0]; 580 | statistic.ready[value] = readyList[0]; 581 | statistic.paint[value] = paintList[0]; 582 | statistic.screen[value] = screenList[0]; 583 | loadList.shift(); 584 | readyList.shift(); 585 | paintList.shift(); 586 | screenList.shift(); 587 | }); 588 | statistic.loadZero = loadZero; 589 | statistic.all = all; 590 | }else { 591 | //包含小时数据 592 | if(historyStatis.minute[hour]) { 593 | //补全60小时 594 | for(let i=0; i<=59; i++) { 595 | statistic.x.push(i); 596 | statistic.load.push(0); 597 | statistic.ready.push(0); 598 | statistic.paint.push(0); 599 | statistic.screen.push(0); 600 | } 601 | const { x, load, ready, paint, screen, loadZero, all=0 } = historyStatis.minute[hour]; 602 | loadList = await services.webMonitor.getPerformanceListByIds(load); 603 | readyList = await services.webMonitor.getPerformanceListByIds(ready); 604 | paintList= await services.webMonitor.getPerformanceListByIds(paint); 605 | screenList = await services.webMonitor.getPerformanceListByIds(screen); 606 | x.forEach(value => { 607 | //UA转换 608 | parseAgent(loadList[0]); 609 | parseAgent(readyList[0]); 610 | parseAgent(paintList[0]); 611 | parseAgent(screenList[0]); 612 | statistic.load[value] = loadList[0]; 613 | statistic.ready[value] = readyList[0]; 614 | statistic.paint[value] = paintList[0]; 615 | statistic.screen[value] = screenList[0]; 616 | loadList.shift(); 617 | readyList.shift(); 618 | paintList.shift(); 619 | screenList.shift(); 620 | }); 621 | statistic.loadZero = loadZero; 622 | statistic.all = all; 623 | } 624 | } 625 | list.push(statistic); 626 | } 627 | return list; 628 | } 629 | /** 630 | * 读取性能的历史记录 631 | */ 632 | async function getTodayList({ project, day, hour }) { 633 | let counts; 634 | const statistic = { 635 | x: [], 636 | load: [], 637 | ready: [], 638 | paint: [], 639 | screen: [], 640 | loadZero: 0, //load时间为0的个数 641 | all: 0, //日志总数 642 | day 643 | }; 644 | statistic.loadZero = await services.webMonitor.statisticPerformanceCount({ 645 | project, day, hour, 646 | other: { 647 | load: 0 648 | } 649 | }); 650 | statistic.all = await services.webMonitor.statisticPerformanceCount({ 651 | project, day, hour, 652 | }); 653 | if(!hour) { 654 | //补全24小时 655 | for(let i=0; i<=23; i++) { 656 | statistic.x.push(i); 657 | statistic.load.push(0); 658 | statistic.ready.push(0); 659 | statistic.paint.push(0); 660 | statistic.screen.push(0); 661 | } 662 | counts = await services.webMonitor.statisticPerformanceCount({ 663 | project, 664 | day, 665 | group: 'hour', 666 | attributes: ['hour'] 667 | }); 668 | counts = counts.map(value => { 669 | return { 670 | ...value, 671 | offset: getFloorNumber(value.count) 672 | } 673 | }); 674 | // console.log(counts) 675 | //逐条查询排序后排在95%位置的记录 676 | for(let i=0; i { 749 | let { project, start, end, hour } = ctx.query; 750 | //今日的统计信息 751 | const today = moment().format("YYYYMMDD"); //格式化的日期 752 | start = +start; 753 | end = +end; 754 | //结束时间非今天，就查找历史记录 755 | if(today != end) { 756 | const data = await getHistoryList({ project, start, end, hour }); 757 | ctx.body = { code: 0, data: data}; 758 | return; 759 | } 760 | let statistic = []; 761 | //开始时间和结束时间不是同一天 762 | if(start != end) { 763 | statistic = await getHistoryList({ project, start, end: (end-1), hour }); 764 | } 765 | //结束时间是今天，就计算今天的统计信息 766 | const current = await getTodayList({ project, day: end, hour }); 767 | statistic.push(current); 768 | ctx.body = { code: 0, data: statistic}; 769 | } 770 | ); 771 | /** 772 | * 读取一条性能日志 773 | */ 774 | router.get( 775 | '/monitor/performance/get', 776 | middlewares.checkAuth('backend.monitor.performance.dashboard'), 777 | async (ctx) => { 778 | const { 779 | id, 780 | } = ctx.query; 781 | // 结束时间是今天，就计算今天的统计信息 782 | const row = await services.webMonitor.getOnePerformance({ id }); 783 | ctx.body = { code: 0, data: row }; 784 | }, 785 | ); 786 | /** 787 | * 读取性能时序 788 | */ 789 | router.get( 790 | '/monitor/performance/flow', 791 | middlewares.checkAuth('backend.monitor.performance.dashboard'), 792 | async (ctx) => { 793 | const { 794 | id, 795 | type, 796 | range, 797 | identity, 798 | curPage = 1, 799 | pageSize = 10, 800 | start, 801 | end, 802 | path 803 | } = ctx.query; 804 | const { count, rows } = await services.webMonitor.getPerformanceFlow({ 805 | id, type, range, identity, curPage, pageSize, start, end, path 806 | }); 807 | ctx.body = { code: 0, data: rows, count }; 808 | }, 809 | ); 810 | /** 811 | * 飞书告警 demo演示 812 | */ 813 | router.post( 814 | "/monitor/send/warn", 815 | async (ctx) => { 816 | let { email, text } = ctx.request.body; 817 | await services.webMonitor.tenantSend({ email, text }); 818 | ctx.body = { code: 0 }; 819 | } 820 | ); 821 | } 822 | 823 | -------------------------------------------------------------------------------- /scripts/demo.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-03 14:06:51 4 | * @LastEditTime: 2021-02-05 16:14:49 5 | * @LastEditors: strick 6 | * @Description: 脚本测试 7 | * @FilePath: /strick/shin-server/scripts/demo.js 8 | */ 9 | import services from '../services'; 10 | 11 | async function test() { 12 | const result = await services.backendUserAccount.find(); 13 | console.log("script test", result); 14 | } 15 | test(); 16 | -------------------------------------------------------------------------------- /scripts/index.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-03 14:06:38 4 | * @LastEditTime: 2021-02-03 15:06:39 5 | * @LastEditors: strick 6 | * @Description: 脚本入口 7 | * @FilePath: /strick/shin-server/scripts/index.js 8 | */ 9 | require('babel-core/register'); 10 | require('babel-polyfill'); 11 | 12 | global.env = process.env.NODE_ENV; 13 | global.logger = { 14 | trace: console.log, 15 | info: console.log, 16 | debug: console.log, 17 | error: console.log, 18 | warn: console.log, 19 | }; 20 | 21 | /** 22 | * 执行命令，当前位置如果与 scripts 目录平级，则 NODE_ENV=development node scripts/index.js 23 | * 其中 NODE_ENV 为环境常量，test、pre 和 production 24 | */ 25 | require('./demo'); 26 | 27 | 28 | -------------------------------------------------------------------------------- /services/backendUserAccount.js: -------------------------------------------------------------------------------- 1 | import redis from '../db/redis'; 2 | const KEY_LOGIN_COUNT = 'backend:login:count'; 3 | import models from '../models'; 4 | 5 | /** 6 | * 管理后台用户账号 7 | */ 8 | class BackendUserAccount { 9 | /** 10 | * 注册 11 | */ 12 | async register(...data) { 13 | const entity = new models.BackendUserAccount(data[0]); 14 | const res = await entity.save(); 15 | return res; 16 | } 17 | 18 | /** 19 | * 通过id查询用户 20 | */ 21 | async getInfoById(id) { 22 | try { 23 | const res = await models.BackendUserAccount.findOne({ _id: id }); 24 | return res; 25 | } catch (err) { 26 | logger.error(err.message); 27 | return ''; 28 | } 29 | } 30 | 31 | /** 32 | * 通过用户名查询账号信息 33 | */ 34 | async getInfoByUserName(userName) { 35 | const res = await models.BackendUserAccount.findOne({ userName }); 36 | return res; 37 | } 38 | 39 | /** 40 | * 更新账号信息 41 | */ 42 | async updateInfo(id, data) { 43 | const res = await models.BackendUserAccount.update({ 44 | _id: id, 45 | }, { 46 | $set: data, 47 | updateTime: new Date(), 48 | }); 49 | return res; 50 | } 51 | 52 | /** 53 | * 获取用户列表 54 | */ 55 | async getList(curPage, limit, keywords, roleId) { 56 | const whereCondition = { 57 | status: { 58 | $lt: 2, 59 | }, 60 | }; 61 | if (keywords) { 62 | whereCondition.$or = [ 63 | { 64 | realName: new RegExp(keywords), 65 | }, 66 | { 67 | userName: new RegExp(keywords), 68 | }, 69 | ]; 70 | } 71 | if (roleId && roleId !== 'all') { 72 | whereCondition.roles = { 73 | $in: [roleId], 74 | }; 75 | } 76 | const list = await models.BackendUserAccount.find( 77 | whereCondition 78 | , { 79 | password: 0, 80 | salt: 0, 81 | }).skip((curPage - 1) * limit).limit(limit).sort({ createTime: 'desc' }); 82 | const count = await models.BackendUserAccount 83 | .find(whereCondition, { 84 | password: 0, 85 | salt: 0, 86 | }) 87 | .count(); 88 | return {list, count}; 89 | } 90 | 91 | /** 92 | * 查询指定用户详情 93 | */ 94 | async getDetail(userId) { 95 | const res = await models.BackendUserAccount.findOne({ 96 | _id: userId, 97 | }) 98 | return res; 99 | } 100 | 101 | /** 102 | * 检查邮箱是否已被注册 103 | */ 104 | async checkEmailExist(email) { 105 | const res = await models.BackendUserAccount.findOne({ 106 | userName: email, 107 | }); 108 | return res; 109 | } 110 | 111 | /** 112 | * 检查手机号是否已注册 113 | */ 114 | async checkCelphoneExist(cellphone) { 115 | const res = await models.BackendUserAccount.findOne({ 116 | cellphone, 117 | }); 118 | return res; 119 | } 120 | 121 | /** 122 | * 更新用户状态 123 | */ 124 | async updateStatus(_id, status) { 125 | const res = await models.BackendUserAccount.update({ 126 | _id, 127 | }, { 128 | $set: { 129 | status, 130 | }, 131 | updateTime: new Date(), 132 | }); 133 | return res; 134 | } 135 | 136 | /** 137 | * 删除用户 138 | */ 139 | async delete(_id) { 140 | const res = await models.BackendUserAccount.remove({ 141 | _id, 142 | }); 143 | return res; 144 | } 145 | 146 | /** 147 | * 查询所有账号 148 | */ 149 | async find() { 150 | const res = await models.BackendUserAccount.find(); 151 | return res; 152 | } 153 | 154 | /** 155 | * 读取缓存中的登录次数 156 | */ 157 | async getLoginCount(userName) { 158 | const res = await redis.aws.get(`${KEY_LOGIN_COUNT}:${userName}`); 159 | return res; 160 | } 161 | 162 | /** 163 | * 增加缓存中的登录次数 164 | */ 165 | async increaseLoginCount(userName) { 166 | const res = await redis.aws.incr(`${KEY_LOGIN_COUNT}:${userName}`); 167 | return res; 168 | } 169 | 170 | /** 171 | * 写入缓存中的登录次数 172 | */ 173 | async setLoginCount(userName) { 174 | const res = await redis.aws.set(`${KEY_LOGIN_COUNT}:${userName}`, 0, 'EX', 300); 175 | return res; 176 | } 177 | 178 | /** 179 | * 移除缓存中的登录次数 180 | */ 181 | async resetLoginCount(userName) { 182 | const res = await redis.aws.del(`${KEY_LOGIN_COUNT}:${userName}`); 183 | return res; 184 | } 185 | 186 | /** 187 | * 根据真实姓名查询账户 188 | */ 189 | async getAccountOfName(realName) { 190 | return models.BackendUserAccount.findOne({ 191 | realName, 192 | }); 193 | } 194 | 195 | /** 196 | * 查询指定角色id的账号 197 | */ 198 | async getListOfRole(id) { 199 | return models.BackendUserAccount.find({ 200 | roles: id.toString(), 201 | }); 202 | } 203 | } 204 | 205 | export default BackendUserAccount; 206 | -------------------------------------------------------------------------------- /services/backendUserRole.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-02 16:51:51 4 | * @LastEditTime: 2023-04-24 18:00:36 5 | * @LastEditors: strick 6 | * @Description: 权限角色 7 | * @FilePath: /strick/shin-server/services/backendUserRole.js 8 | */ 9 | import models from '../models'; 10 | class BackendUserRole { 11 | 12 | /** 13 | * 获取角色列表 14 | */ 15 | async getList(role, cursor, limit) { 16 | const list = await models.BackendUserRole 17 | .find({roleName: {$regex: (role ? role : "")}}) 18 | .skip((cursor - 1) * limit) 19 | .limit(limit) 20 | .sort({ createTime: -1 }); 21 | const count = await models.BackendUserRole 22 | .find({roleName: {$regex: (role ? role : "")}}) 23 | .count(); 24 | return { list, count }; 25 | } 26 | 27 | /** 28 | * 获取角色权限列表 29 | */ 30 | async getInfoById(roleId) { 31 | const res = await models.BackendUserRole.findOne({ _id: roleId }); 32 | return res; 33 | } 34 | 35 | /** 36 | * 添加角色 37 | */ 38 | async add(roleName, roleDesc, rolePermission) { 39 | const entity = new models.BackendUserRole({ 40 | roleName, 41 | roleDesc, 42 | rolePermission, 43 | }); 44 | const res = await entity.save(); 45 | return res; 46 | } 47 | 48 | /** 49 | * 更新角色信息 50 | */ 51 | async update(id, data) { 52 | const res = await models.BackendUserRole.update({ 53 | _id: id, 54 | }, { 55 | $set: data, 56 | updateTime: new Date(), 57 | }); 58 | return res; 59 | } 60 | 61 | /** 62 | * 删除角色 63 | */ 64 | async remove(roleId) { 65 | const res = await models.BackendUserRole.remove({ _id: roleId }); 66 | return res; 67 | } 68 | 69 | /** 70 | * 获取某个角色的所有账号 71 | */ 72 | async getAccountOfRoleName(roleName) { 73 | const role = await models.BackendUserRole.findOne({ 74 | roleName, 75 | }); 76 | if (!role) { 77 | return []; 78 | } 79 | const roleId = role._id; 80 | return models.BackendUserAccount.find({ 81 | roles: roleId.toString(), 82 | }); 83 | } 84 | 85 | /** 86 | * 查询匹配关键字的角色列表 87 | */ 88 | async getRolesOfKeywords(keywords) { 89 | return models.BackendUserRole.find({ 90 | roleName: new RegExp(keywords), 91 | }); 92 | } 93 | } 94 | 95 | export default BackendUserRole; 96 | -------------------------------------------------------------------------------- /services/common.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-07-21 15:47:03 5 | * @LastEditTime: 2023-04-24 18:01:08 6 | * @Description: 通用数据处理 7 | * @FilePath: /strick/shin-server/services/common.js 8 | */ 9 | import models from '../models'; 10 | class Common { 11 | /** 12 | * 数据库查询一条记录 13 | */ 14 | async getOne(tableName, where = {}) { 15 | return models[tableName].findOne({ 16 | where, 17 | raw: true 18 | }); 19 | } 20 | 21 | /** 22 | * 数据库查询多条记录 23 | * 默认提供页码、页数和排序规则 24 | */ 25 | async getList({ 26 | tableName, 27 | where = {}, 28 | curPage = 1, 29 | limit = 20, 30 | order = [["id", "DESC"]] 31 | }) { 32 | return models[tableName].findAndCountAll({ 33 | where, 34 | limit, 35 | offset: (curPage - 1) * limit, 36 | order, 37 | raw: true 38 | }); 39 | } 40 | 41 | /** 42 | * 聚合 43 | */ 44 | async aggregation({ tableName, where = {}, func = "count", field }) { 45 | if (func === "count") 46 | return models[tableName][func]({ 47 | where 48 | }); 49 | return models[tableName][func](field, { 50 | where 51 | }); 52 | } 53 | 54 | /** 55 | * 新增 56 | */ 57 | async create(tableName, data) { 58 | return models[tableName].create(data); 59 | } 60 | 61 | /** 62 | * 修改 63 | */ 64 | async update(tableName, set, where) { 65 | return models[tableName].update(set, { where }); 66 | } 67 | } 68 | export default Common; -------------------------------------------------------------------------------- /services/index.js: -------------------------------------------------------------------------------- 1 | import backendUserAccount from './backendUserAccount'; 2 | import backendUserRole from './backendUserRole'; 3 | import common from './common'; 4 | import tool from './tool'; 5 | import webMonitor from './webMonitor'; 6 | 7 | const services = { 8 | backendUserAccount: new backendUserAccount(), 9 | backendUserRole: new backendUserRole(), 10 | common: new common(), 11 | tool: new tool(), 12 | webMonitor: new webMonitor() 13 | }; 14 | 15 | export default services; 16 | 17 | -------------------------------------------------------------------------------- /services/tool.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2020-12-16 19:17:57 5 | * @LastEditTime: 2023-04-24 18:01:29 6 | * @Description: 后台工具服务 7 | * @FilePath: /strick/shin-server/services/tool.js 8 | */ 9 | import redis from '../db/redis'; 10 | const shortChainKey = 'cache:aws:shortChain'; 11 | import models from '../models'; 12 | 13 | class Tool { 14 | /** 15 | * MongoDB查询 16 | */ 17 | async mongoQuery({ name, options, cursor }) { 18 | let result = models[name].find(options); 19 | for (let key in cursor) { 20 | if (cursor[key] !== undefined) { 21 | result[key](cursor[key]); 22 | continue; 23 | } 24 | result[key](); 25 | } 26 | return result.exec(); 27 | } 28 | 29 | /** 30 | * 创建通用配置 31 | */ 32 | async createConfig({ title, content, key }) { 33 | return models.AppGlobalConfig.create({ title, content, key }); 34 | } 35 | /** 36 | * 编辑通用配置 37 | */ 38 | async editConfig({ title, content, id }) { 39 | return models.AppGlobalConfig.update( 40 | { title, content }, 41 | { 42 | where: { 43 | id 44 | } 45 | } 46 | ); 47 | } 48 | /** 49 | * 删除通用配置 50 | */ 51 | async delConfig({ id }) { 52 | return models.AppGlobalConfig.update( 53 | { status: 0 }, 54 | { 55 | where: { 56 | id 57 | } 58 | } 59 | ); 60 | } 61 | /** 62 | * 通用配置列表 63 | */ 64 | async getConfigList() { 65 | return models.AppGlobalConfig.findAndCountAll({ 66 | where: { 67 | status: 1 68 | } 69 | }); 70 | } 71 | /** 72 | * 读取一条通用配置 73 | */ 74 | async getOneConfig(where) { 75 | return models.AppGlobalConfig.findOne({ 76 | where, 77 | raw: true 78 | }); 79 | } 80 | /** 81 | * 读取通用配置解析后的内容 82 | */ 83 | async getConfigContent(where) { 84 | const result = await models.AppGlobalConfig.findOne({ 85 | where, 86 | raw: true 87 | }); 88 | if (!result) { 89 | return result; 90 | } 91 | const { content } = result; 92 | return JSON.parse(content); 93 | } 94 | /** 95 | * 添加短链 96 | */ 97 | async createShortChain(data) { 98 | const result = models.WebShortChain.create(data); 99 | if(result) { 100 | this.redisShortChainSet(data.short, data.url); 101 | } 102 | return result; 103 | } 104 | /** 105 | * 更新短链 106 | */ 107 | async updateShortChain(data) { 108 | //更新的返回值是一个数组，包含受影响的函数 109 | const [affected] = await models.WebShortChain.update({ 110 | url: data.url 111 | }, { 112 | where: { 113 | id: data.id, 114 | }, 115 | }); 116 | if(affected > 0) { 117 | this.redisShortChainSet(data.short, data.url); 118 | } 119 | return affected; 120 | } 121 | /** 122 | * 查询短链的一条记录 123 | */ 124 | async getOneShortChain(where) { 125 | return models.WebShortChain.findOne({ 126 | where, 127 | raw: true //格式化返回值，只包含表的字段 128 | }); 129 | } 130 | /** 131 | * 查询短链列表 132 | */ 133 | async getShortChainList({ curPage, short, url }) { 134 | const where = { 135 | status: 1 136 | }; 137 | if(short) { 138 | where.short = { 139 | $like: `%${short}%`, 140 | }; 141 | } 142 | if(url) { 143 | where.url = { 144 | $like: `%${url}%`, 145 | }; 146 | } 147 | return models.WebShortChain.findAndCountAll({ 148 | where, 149 | limit: 20, 150 | offset: (curPage - 1) * 20, 151 | order: [['ctime', 'desc']], 152 | }); 153 | } 154 | /** 155 | * 删除短链 156 | * 若有删除，就需要有恢复的操作，否则就得物理删除 157 | * 因为 key 是唯一的，不允许重复 158 | */ 159 | async delShortChain({ id, short }) { 160 | const [affected] = await models.WebShortChain.update({ 161 | status: 0 162 | }, { 163 | where: { 164 | id: id, 165 | }, 166 | }); 167 | if(affected == 0) 168 | return 0; 169 | //删除缓存 170 | this.redisShortChainDel(short); 171 | return 1; 172 | } 173 | /** 174 | * 更新短链缓存 175 | * 默认存7天 176 | */ 177 | async redisShortChainSet(key, url) { 178 | const result = redis.aws.hset(shortChainKey, key, url); 179 | redis.aws.expire(shortChainKey, 604800); //超时时间为7天 180 | return result; 181 | } 182 | //读取redis 183 | async redisShortChainGet(key) { 184 | return redis.aws.hget(shortChainKey, key); 185 | } 186 | //删除redis 187 | async redisShortChainDel(key) { 188 | return redis.aws.hdel(shortChainKey, key); 189 | } 190 | } 191 | export default Tool; 192 | -------------------------------------------------------------------------------- /services/webMonitor.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-02-25 15:32:43 5 | * @LastEditTime: 2023-04-24 18:01:39 6 | * @Description: 前端监控 7 | * @FilePath: /strick/shin-server/services/webMonitor.js 8 | */ 9 | import config from "config"; 10 | import models from '../models'; 11 | import redis from '../db/redis'; 12 | import xfetch from '../utils/xfetch'; 13 | const tokenRedis = 'tenant:access:token'; 14 | class WebMonitor { 15 | /** 16 | * 初始化数据 17 | */ 18 | initMonitorWhere({id, category, msg, project, start, 19 | end, identity, other, messageType, messageStatus, messagePath}) { 20 | let where = { 21 | ...other 22 | }; 23 | if (id) { 24 | where.id = id; 25 | } 26 | if (category) { 27 | where.category = category; 28 | } 29 | if (identity) { 30 | where.identity = identity; 31 | } 32 | if (project) { 33 | where.project = project; 34 | } 35 | if (messageType) { 36 | where.message_type = messageType; 37 | } 38 | if (messageStatus) { 39 | where.message_status = messageStatus; 40 | } 41 | if (messagePath) { 42 | where.message_path = messagePath; 43 | } 44 | if (start && end) { 45 | where.ctime = { 46 | $gte: start, 47 | $lte: end, 48 | }; 49 | } 50 | if (msg && msg.length > 0) { 51 | const messages = msg.map(value => ({ 52 | message: { 53 | $like: `%${value}%`, 54 | } 55 | })); 56 | where = Sequelize.and(where, ...messages); 57 | } 58 | return where; 59 | } 60 | /** 61 | * 查询日志 62 | */ 63 | async getMonitorList({ curPage, pageSize, id, category, msg, 64 | project, start, end, identity, other={}, match, messageType, messageStatus, messagePath }) { 65 | const where = this.initMonitorWhere({ id, category, msg, 66 | project, start, end, identity, other, match, messageType, messageStatus, messagePath }); 67 | return models.WebMonitor.findAndCountAll({ 68 | where, 69 | order: [['id', 'DESC']], 70 | raw: true, 71 | limit: parseInt(pageSize), 72 | offset: (curPage - 1) * pageSize, 73 | }); 74 | } 75 | 76 | 77 | /** 78 | * 统计异常通信 79 | */ 80 | async countErrorAjax({ project, day, messageStatus }) { 81 | const where = { 82 | project, 83 | day, 84 | message_status: messageStatus 85 | } 86 | const field = "message_path"; 87 | return models.WebMonitor.count({ 88 | where, 89 | group: [ field ], 90 | attributes: [ field ] 91 | }); 92 | } 93 | 94 | /** 95 | * 查询日志图表生成 96 | */ 97 | async getMonitorListChart({ id, category, msg, project, start, end, 98 | identity, other={}, match, day, hour, attribute, messageType, messageStatus, messagePath }) { 99 | const where = this.initMonitorWhere({ id, category, msg, project, start, end, 100 | identity, other, match, messageType, messageStatus, messagePath }); 101 | if(day) { 102 | where.day = day; 103 | } 104 | if(hour !== undefined) { 105 | where.hour = hour; 106 | } 107 | const field = attribute; 108 | return models.WebMonitor.count({ 109 | where, 110 | group: [ attribute ], 111 | attributes: [ attribute ], 112 | having: { [field]: { 113 | $ne: null 114 | }} 115 | }); 116 | } 117 | /** 118 | * 查询指定日志的上下文 119 | */ 120 | async getMonitorContext({ from, to }) { 121 | const where = { 122 | id: { 123 | $gte: from, 124 | $lte: to, 125 | } 126 | }; 127 | return models.WebMonitor.findAll({ 128 | where, 129 | raw: true, 130 | }); 131 | } 132 | /** 133 | * 删除过期日志 134 | */ 135 | async delExpiredMonitor({ deadline }) { 136 | const where = { 137 | ctime: { 138 | $lte: deadline , 139 | } 140 | }; 141 | return models.WebMonitor.destroy({ 142 | where, 143 | }); 144 | } 145 | /** 146 | * 删除过期的map文件 147 | */ 148 | async delExpiredMap({ day }) { 149 | return xfetch({ 150 | url: '/smap/del', 151 | method: 'GET', 152 | params: { day }, 153 | baseURL: config.get('services').adminApi, 154 | }); 155 | } 156 | /** 157 | * 统计数量 158 | */ 159 | async statisticCount({ project, category, from, to, other={} }) { 160 | const where = { 161 | ...other 162 | }; 163 | if(project) { 164 | where.project = project; 165 | } 166 | if(category) { 167 | where.category = category; 168 | } 169 | where.ctime = { 170 | $gte: from, 171 | $lt: to, 172 | }; 173 | const count = await models.WebMonitor.count({ 174 | where, 175 | }); 176 | return count ? count : 0; 177 | } 178 | /** 179 | * 统计总量 180 | */ 181 | async statisticSum({ field, project, category, from, to }) { 182 | const where = { 183 | ctime: { 184 | $gte: from, 185 | $lte: to, 186 | } 187 | }; 188 | if(project) { 189 | where.project = project; 190 | } 191 | if(category) { 192 | where.category = category; 193 | } 194 | const sum = await models.WebMonitor.sum(field, { 195 | where 196 | }); 197 | return sum ? sum : 0; 198 | } 199 | /** 200 | * 添加一条统计记录 201 | */ 202 | async createStatis(data) { 203 | return models.WebMonitorStatis.create(data); 204 | } 205 | /** 206 | * 读取一条统计记录 207 | */ 208 | async getOneStatis(where) { 209 | return models.WebMonitorStatis.findOne({ 210 | where, 211 | raw: true 212 | }); 213 | } 214 | /** 215 | * 读取多条统计记录 216 | */ 217 | async getStatisList({ start, end, other={} }) { 218 | const where = { 219 | ...other 220 | }; 221 | if (start && end) { 222 | where.date = { 223 | $gte: start, 224 | $lte: end, 225 | }; 226 | } 227 | return models.WebMonitorStatis.findAll({ 228 | where, 229 | raw: true, 230 | }); 231 | } 232 | /** 233 | * 创建性能监控项目 234 | */ 235 | async createPerformanceProject(data) { 236 | return models.WebPerformanceProject.create(data); 237 | } 238 | /** 239 | * 获取一条性能监控项目 240 | */ 241 | async getOnePerformanceProject(where) { 242 | return models.WebPerformanceProject.findOne({ 243 | where, 244 | raw: true 245 | }); 246 | } 247 | /** 248 | * 获取正常的性能监控项目 249 | */ 250 | async getPerformanceProjectList({ name, curPage, pageSize }) { 251 | const where = { 252 | status: 1, 253 | }; 254 | if(name) { 255 | where.name = { 256 | $like: `%${name}%` 257 | } 258 | } 259 | return models.WebPerformanceProject.findAndCountAll({ 260 | where, 261 | raw: true, 262 | order: 'ctime DESC', 263 | limit: parseInt(pageSize), 264 | offset: (curPage - 1) * pageSize, 265 | }); 266 | } 267 | /** 268 | * 更新性能监控项目 269 | */ 270 | async updatePerformanceProject(id, data) { 271 | return models.WebPerformanceProject.update( 272 | data, 273 | { 274 | where: { id }, 275 | } 276 | ); 277 | } 278 | /** 279 | * 删除性能监控项目 280 | */ 281 | async delPerformanceProject({ id }) { 282 | return models.WebPerformanceProject.update( 283 | { status: 0 }, 284 | { 285 | where: { id } 286 | } 287 | ); 288 | } 289 | /** 290 | * 统计数量 291 | */ 292 | async statisticPerformanceCount({ project, day, hour, other={}, group, attributes }) { 293 | const where = { 294 | ...other 295 | }; 296 | if(project) { 297 | where.project = project; 298 | } 299 | if(day) { 300 | where.day = day; 301 | } 302 | if(hour !== undefined) { 303 | where.hour = hour; 304 | } 305 | const count = await models.WebPerformance.count({ 306 | attributes, 307 | where, 308 | group 309 | }); 310 | return count ? count : 0; 311 | } 312 | /** 313 | * 获取一条性能数据 314 | */ 315 | async getOnePerformance(where) { 316 | return models.WebPerformance.findOne({ 317 | where, 318 | raw: true, 319 | }); 320 | } 321 | /** 322 | * 获取一条性能数据 323 | */ 324 | async getPerformanceFlow({ 325 | id, type, range, identity, curPage, pageSize, start, end, path 326 | }) { 327 | const where = {}; 328 | const types = { 329 | 1: 'paint', 330 | 2: 'screen', 331 | }; 332 | const ranges = { 333 | 1: { 334 | $lte: 1000, 335 | }, 336 | 2: { 337 | $gt: 1000, 338 | $lte: 2000, 339 | }, 340 | 3: { 341 | $gt: 2000, 342 | $lte: 3000, 343 | }, 344 | 4: { 345 | $gt: 3000, 346 | $lte: 4000, 347 | }, 348 | 5: { 349 | $gt: 4000, 350 | }, 351 | }; 352 | if(identity) where.identity = identity; 353 | if (id) where.id = id; 354 | if (type && range) { 355 | where[types[type]] = ranges[range]; 356 | } 357 | if (start && end) { 358 | where.ctime = { 359 | $gte: start, 360 | $lte: end, 361 | }; 362 | } 363 | if(path) where.referer_path = path; 364 | return models.WebPerformance.findAndCountAll({ 365 | where, 366 | raw: true, 367 | order: 'ctime DESC', 368 | limit: parseInt(pageSize), 369 | offset: (curPage - 1) * pageSize, 370 | }); 371 | } 372 | /** 373 | * 获取一条排序处在95%位置的性能数据 374 | */ 375 | async getOneOrderPerformance(where, order, offset) { 376 | return models.WebPerformance.findOne({ 377 | where, 378 | order, 379 | offset, 380 | raw: true 381 | }); 382 | } 383 | /** 384 | * 获取多条性能数据 385 | */ 386 | async getPerformanceListByIds(ids) { 387 | const where = { 388 | id: ids 389 | }; 390 | return models.WebPerformance.findAll({ 391 | where, 392 | raw: true 393 | }); 394 | } 395 | /** 396 | * 创建性能统计项目 397 | */ 398 | async createPerformanceStatis(data) { 399 | return models.WebPerformanceStatis.create(data); 400 | } 401 | /** 402 | * 读取一条性能统计记录 403 | */ 404 | async getOnePerformanceStatis(where) { 405 | return models.WebPerformanceStatis.findOne({ 406 | where, 407 | raw: true 408 | }); 409 | } 410 | /** 411 | * 删除过期性能日志 412 | */ 413 | async delExpiredPerformance({ deadline }) { 414 | const where = { 415 | day: { 416 | $lte: deadline , 417 | } 418 | }; 419 | return models.WebPerformance.destroy({ 420 | where, 421 | }); 422 | } 423 | /** 424 | * 删除过期性能统计日志 425 | */ 426 | async delExpiredPerformanceStatis({ deadline }) { 427 | const where = { 428 | date: { 429 | $lte: deadline , 430 | } 431 | }; 432 | return models.WebPerformanceStatis.destroy({ 433 | where, 434 | }); 435 | } 436 | /** 437 | * 获取飞书的token 438 | * https://open.feishu.cn/document/ukTMukTMukTM/uIjNz4iM2MjLyYzM 439 | */ 440 | async getTenantAccessToken() { 441 | // 读取缓存中的token，缓存有效时间为 2 小时 442 | let token = await redis.single.get(tokenRedis); 443 | if(token) { 444 | return `Bearer ${token}`; 445 | } 446 | // 调用飞书接口 447 | const { data } = await xfetch({ 448 | baseURL: config.get("feishu").url, 449 | url: "open-apis/auth/v3/tenant_access_token/internal", 450 | data: { 451 | app_id: config.get("feishu").app_id, 452 | app_secret: config.get("feishu").app_secret, 453 | }, 454 | method: 'POST', 455 | }); 456 | token = data.tenant_access_token; 457 | // 超时时间为 2 小时 458 | await redis.single.set(tokenRedis, token, 'EX', data.expire); 459 | return `Bearer ${token}`; 460 | } 461 | 462 | /** 463 | * 飞书发送消息 464 | * https://open.feishu.cn/document/ukTMukTMukTM/uUjNz4SN2MjL1YzM?lang=zh-CN 465 | */ 466 | async tenantSend({ email, text }) { 467 | const token = await this.getTenantAccessToken(); 468 | return xfetch({ 469 | headers: { 470 | Authorization: token 471 | }, 472 | baseURL: config.get("feishu").url, 473 | url: "open-apis/message/v4/send/", 474 | data: { 475 | email, 476 | msg_type: "text", 477 | content: { 478 | text 479 | } 480 | }, 481 | method: 'POST', 482 | }); 483 | } 484 | 485 | /** 486 | * 创建性能日志 487 | */ 488 | async createPerformance(data) { 489 | return models.WebPerformance.create(data); 490 | } 491 | 492 | /** 493 | * 处理日志的更新和新增 494 | */ 495 | async handleMonitor(monitor) { 496 | const exist = await this.getMonitorByKey(monitor); //是否存在监控日志 497 | // 存在 498 | if(exist) 499 | return this.updateMonitorDigit(exist.digit+1, exist.id); //更新出现次数 500 | let row = await this.createMonitor(monitor); //创建记录 501 | row = row.toJSON(); 502 | // 添加行为记录 503 | if (monitor.record) { 504 | await models.WebMonitorRecord.create({ monitor_id: row.id, record: monitor.record }); 505 | } 506 | return row; 507 | } 508 | 509 | /** 510 | * 根据 Key 来搜索日志 511 | */ 512 | async getMonitorByKey({ project, category, key, identity }) { 513 | return models.WebMonitor.findOne({ 514 | raw: true, 515 | where: { 516 | project, 517 | category, 518 | key, 519 | identity 520 | } 521 | }); 522 | } 523 | 524 | /** 525 | * 创建监控日志 526 | */ 527 | async createMonitor(data) { 528 | return models.WebMonitor.create(data); 529 | } 530 | 531 | /** 532 | * 更新监控日志的出现次数 533 | */ 534 | async updateMonitorDigit(digit, id) { 535 | return models.WebMonitor.update({ 536 | digit 537 | }, { 538 | where: { id } 539 | }); 540 | } 541 | } 542 | export default WebMonitor; 543 | -------------------------------------------------------------------------------- /static/img/avatar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pwstrick/shin-server/3bdb181185a0bf58aa6e18034cb8640a0731a877/static/img/avatar.png -------------------------------------------------------------------------------- /static/img/cover.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pwstrick/shin-server/3bdb181185a0bf58aa6e18034cb8640a0731a877/static/img/cover.jpg -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-03 14:17:34 4 | * @LastEditTime: 2021-02-03 14:28:36 5 | * @LastEditors: strick 6 | * @Description: 单元测试入口 7 | * @FilePath: /strick/shin-server/test/index.js 8 | */ 9 | import redis from '../db/redis'; 10 | import 'babel-polyfill'; 11 | /** 12 | * chai 4.0 13 | * https://www.chaijs.com/api/bdd/ 14 | */ 15 | import { expect } from 'chai'; 16 | /** 17 | * supertest 3.0 18 | * https://github.com/visionmedia/supertest 19 | */ 20 | import supertest from 'supertest'; 21 | import models from '../models'; 22 | 23 | global.models = models; 24 | global.expect = expect; 25 | global.api = supertest('http://localhost:6060'); 26 | global.env = process.env.NODE_ENV || 'development'; 27 | global.logger = { 28 | info: function () {}, 29 | warn: function () {}, 30 | debug: function () {}, 31 | trace: function () {}, 32 | error: function () {}, 33 | fatal: function () {} 34 | }; 35 | 36 | //读取账户的登录态缓存 37 | redis.aws.get("backend:user:account:token:development").then((token) => { 38 | global.authToken = `Bearer ${token}`; 39 | }); 40 | 41 | 42 | -------------------------------------------------------------------------------- /test/mocha.opts: -------------------------------------------------------------------------------- 1 | --compilers js:babel-core/register 2 | -t 10000 3 | --recursive -------------------------------------------------------------------------------- /test/routers/user.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-03 14:17:34 4 | * @LastEditTime: 2021-02-05 16:10:19 5 | * @LastEditors: strick 6 | * @Description: 账户的路由层测试 7 | * @FilePath: /strick/shin-server/test/routers/user.js 8 | */ 9 | const userName = `test+${Date.now()}@shin.com`; 10 | const password = '123456'; 11 | 12 | describe('GET /user', () => { 13 | const url = '/user'; 14 | it('注册成功：返回用户id和注册时间', (done) => { 15 | api 16 | .get(url) 17 | .set('Authorization', authToken) 18 | .send({ 19 | userName, 20 | password, 21 | realName: '测试账号', 22 | cellphone: '13800138000', 23 | roles: '*', 24 | }) 25 | .expect(200) 26 | .end((err, res) => { 27 | if (err) done(err); 28 | const { username } = res.body; 29 | expect(username).to.be.not.empty; 30 | // expect(createTime).to.be.not.empty; 31 | // console.log(res.body) 32 | done(); 33 | }); 34 | }); 35 | }); 36 | 37 | describe('GET /user/list', () => { 38 | const url = '/user/list'; 39 | it('获取用户列表成功', (done) => { 40 | api 41 | .get(url) 42 | .set('Authorization', authToken) 43 | .expect(200, done); 44 | }); 45 | }); 46 | -------------------------------------------------------------------------------- /test/services/user.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-03 14:17:34 4 | * @LastEditTime: 2021-02-03 14:27:44 5 | * @LastEditors: strick 6 | * @Description: 账户的服务层测试 7 | * @FilePath: /strick/shin-server/test/services/user.js 8 | */ 9 | import backendUserRole from '../../services/backendUserRole'; 10 | 11 | describe('用户角色', () => { 12 | it('获取指定id的角色信息', async () => { 13 | const service = new backendUserRole(models); 14 | const res = await service.getInfoById('584a4dc24c886205bd771afe'); 15 | // expect(2).toBe(2); 16 | // expect(res.rolePermisson).to.be.an('array'); 17 | }); 18 | }); 19 | -------------------------------------------------------------------------------- /test/utils/tool.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-03 14:17:34 4 | * @LastEditTime: 2021-02-03 14:28:05 5 | * @LastEditors: strick 6 | * @Description: 工具测试 7 | * @FilePath: /strick/shin-server/test/utils/tool.js 8 | */ 9 | import moment from 'moment'; 10 | 11 | describe("各类工具", () => { 12 | it("moment", () => { 13 | // console.log('测试', moment(Date.now()).add(-1, 'day').format('YYYY-MM-DD')) 14 | // console.log('测试', moment('2020-11-12').add(1, 'd').format('YYYY-MM-DD')); 15 | // const tmp = moment(Date.now()).add(-1, 'day'); 16 | // console.log(tmp.add(8, 'hours').format("YYYY-MM-DD HH:mm")) 17 | // console.log(moment("2020-11-13T07:06:57.000Z").format("YYYY-MM-DD HH:mm")) 18 | // console.log(moment(Date.now()).add(-1, 'day').add(8, 'hours').utcOffset("+00:00").format('YYYY-MM-DD HH:mm')); 19 | // console.log(moment("2020-08-17T08:28:08.000Z")); 20 | // const start = moment().week(moment().week() - 1).startOf('isoWeek').format("YYYY-MM-DD HH:mm:ss"); 21 | // const end = moment().week(moment().week() - 1).endOf('isoWeek').format("YYYY-MM-DD HH:mm:ss"); 22 | // console.log(start, end); 23 | // console.log(moment('2020-11-09').add(1, 'd').format('YYYY-MM-DD')) 24 | }); 25 | it("array.map", () => { 26 | let ids = [1, 2].map((element) => { 27 | if (element == 1) return null; 28 | return element; 29 | }); 30 | expect(ids.length).to.equal(2); 31 | }); 32 | }); 33 | -------------------------------------------------------------------------------- /utils/constant.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-09-06 12:16:33 5 | * @LastEditTime: 2021-09-06 12:17:55 6 | * @Description: 常量 7 | * @FilePath: /strick/shin-server/utils/constant.js 8 | */ 9 | export const MONITOR_PROJECT = [ //监控项目 10 | 'shin-app', 11 | 'shin-h5', 12 | 'shin-mini' 13 | ]; 14 | export const PERFORMANCT_RATE = 0.95; //性能分析的用户比例 15 | -------------------------------------------------------------------------------- /utils/index.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-02-02 16:35:59 5 | * @LastEditTime: 2021-09-06 13:24:58 6 | * @Description: 7 | * @FilePath: /strick/shin-server/utils/index.js 8 | */ 9 | import moment from 'moment'; 10 | 11 | /** 12 | * 随机字符串 13 | * @export 14 | * @param {number} len 字符串长度 15 | * @returns 16 | */ 17 | export function randomString(len) { 18 | const length = len || 32; 19 | const $chars = 'ABCDEFGHJKMNPQRSTWXYZabcdefhijkmnprstwxyz2345678'; 20 | const maxPos = $chars.length; 21 | let pwd = ''; 22 | for (let i = 0; i < length; i += 1) { 23 | pwd += $chars.charAt(Math.floor(Math.random() * maxPos)); 24 | } 25 | return pwd; 26 | } 27 | 28 | /** 29 | * 延迟执行 30 | * @export 31 | * @param {number} ms 延迟时间 32 | * @returns 33 | */ 34 | export function sleep(ms) { 35 | return new Promise((resolve) => { 36 | setTimeout(resolve, ms); 37 | }); 38 | } 39 | 40 | /** 41 | * 身份证、 邮箱、 银行卡、 手机号加密 42 | * @param str string 43 | */ 44 | export function encrypt(str, num) { 45 | if (str) { 46 | const number = num || 0; 47 | const length = str.length; 48 | const hideLen = Math.floor(length / 2) - number; 49 | const offset = (length - hideLen) / 2; 50 | const head = str.slice(0, offset); 51 | const tail = str.slice(offset + hideLen, length); 52 | let star = ''; 53 | for (let i = 0; i < hideLen; i += 1) { 54 | star += '*'; 55 | } 56 | return head + star + tail; 57 | } 58 | return str; 59 | } 60 | 61 | /** 62 | * 日期格式 63 | */ 64 | export function utcToDate(date, format="YYYY-MM-DD HH:mm:ss") { 65 | return moment(date).utcOffset(480).format(format); 66 | } 67 | 68 | /** 69 | * 10进制转62进制 70 | */ 71 | export function string10to62(n) { 72 | if (n === 0) { 73 | return "0"; 74 | } 75 | var digits = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; 76 | var result = ""; 77 | while (n > 0) { 78 | result = digits[n % digits.length] + result; 79 | n = parseInt(n / digits.length, 10); 80 | } 81 | return result; 82 | } 83 | 84 | /** 85 | * 62进制转10进制 86 | */ 87 | export function string62to10(s) { 88 | var digits = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; 89 | var result = 0; 90 | for (var i = 0; i < s.length; i++) { 91 | var p = digits.indexOf(s[i]); 92 | if (p < 0) { 93 | return NaN; 94 | } 95 | result += p * Math.pow(digits.length, s.length - i - 1); 96 | } 97 | return result; 98 | } 99 | 100 | /** 101 | * 定时任务的日志格式 102 | */ 103 | export function setCronFormat({name, result='success', startTimestamp}) { 104 | logger.info(`[${moment().format('YYYY-MM-DD HH:mm:ss')}] 任务[${name}] 执行结果[${result}] 执行时长[${Date.now() - startTimestamp}ms]`); 105 | } -------------------------------------------------------------------------------- /utils/murmurhash.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-01-19 10:50:16 4 | * @LastEditTime: 2021-02-03 14:05:26 5 | * @LastEditors: strick 6 | * @Description: murmurhash算法 7 | * @FilePath: /strick/shin-server/utils/murmurhash.js 8 | */ 9 | const util = require('util'); 10 | (function(){ 11 | var _global = this; 12 | 13 | const createBuffer = (val) => new util.TextEncoder().encode(val) 14 | 15 | /** 16 | * JS Implementation of MurmurHash2 17 | * 18 | * @author Gary Court 19 | * @see http://github.com/garycourt/murmurhash-js 20 | * @author Austin Appleby 21 | * @see http://sites.google.com/site/murmurhash/ 22 | * 23 | * @param {Uint8Array | string} str ASCII only 24 | * @param {number} seed Positive integer only 25 | * @return {number} 32-bit positive integer hash 26 | */ 27 | function MurmurHashV2(str, seed) { 28 | if (typeof str === 'string') str = createBuffer(str); 29 | var 30 | l = str.length, 31 | h = seed ^ l, 32 | i = 0, 33 | k; 34 | 35 | while (l >= 4) { 36 | k = 37 | ((str[i] & 0xff)) | 38 | ((str[++i] & 0xff) << 8) | 39 | ((str[++i] & 0xff) << 16) | 40 | ((str[++i] & 0xff) << 24); 41 | 42 | k = (((k & 0xffff) * 0x5bd1e995) + ((((k >>> 16) * 0x5bd1e995) & 0xffff) << 16)); 43 | k ^= k >>> 24; 44 | k = (((k & 0xffff) * 0x5bd1e995) + ((((k >>> 16) * 0x5bd1e995) & 0xffff) << 16)); 45 | 46 | h = (((h & 0xffff) * 0x5bd1e995) + ((((h >>> 16) * 0x5bd1e995) & 0xffff) << 16)) ^ k; 47 | 48 | l -= 4; 49 | ++i; 50 | } 51 | 52 | switch (l) { 53 | case 3: h ^= (str[i + 2] & 0xff) << 16; 54 | case 2: h ^= (str[i + 1] & 0xff) << 8; 55 | case 1: h ^= (str[i] & 0xff); 56 | h = (((h & 0xffff) * 0x5bd1e995) + ((((h >>> 16) * 0x5bd1e995) & 0xffff) << 16)); 57 | } 58 | 59 | h ^= h >>> 13; 60 | h = (((h & 0xffff) * 0x5bd1e995) + ((((h >>> 16) * 0x5bd1e995) & 0xffff) << 16)); 61 | h ^= h >>> 15; 62 | 63 | return h >>> 0; 64 | }; 65 | 66 | /** 67 | * JS Implementation of MurmurHash3 (r136) (as of May 20, 2011) 68 | * 69 | * @author Gary Court 70 | * @see http://github.com/garycourt/murmurhash-js 71 | * @author Austin Appleby 72 | * @see http://sites.google.com/site/murmurhash/ 73 | * 74 | * @param {Uint8Array | string} key ASCII only 75 | * @param {number} seed Positive integer only 76 | * @return {number} 32-bit positive integer hash 77 | */ 78 | function MurmurHashV3(key, seed) { 79 | if (typeof key === 'string') key = createBuffer(key); 80 | 81 | var remainder, bytes, h1, h1b, c1, c1b, c2, c2b, k1, i; 82 | 83 | remainder = key.length & 3; // key.length % 4 84 | bytes = key.length - remainder; 85 | h1 = seed; 86 | c1 = 0xcc9e2d51; 87 | c2 = 0x1b873593; 88 | i = 0; 89 | 90 | while (i < bytes) { 91 | k1 = 92 | ((key[i] & 0xff)) | 93 | ((key[++i] & 0xff) << 8) | 94 | ((key[++i] & 0xff) << 16) | 95 | ((key[++i] & 0xff) << 24); 96 | ++i; 97 | 98 | k1 = ((((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16))) & 0xffffffff; 99 | k1 = (k1 << 15) | (k1 >>> 17); 100 | k1 = ((((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16))) & 0xffffffff; 101 | 102 | h1 ^= k1; 103 | h1 = (h1 << 13) | (h1 >>> 19); 104 | h1b = ((((h1 & 0xffff) * 5) + ((((h1 >>> 16) * 5) & 0xffff) << 16))) & 0xffffffff; 105 | h1 = (((h1b & 0xffff) + 0x6b64) + ((((h1b >>> 16) + 0xe654) & 0xffff) << 16)); 106 | } 107 | 108 | k1 = 0; 109 | 110 | switch (remainder) { 111 | case 3: k1 ^= (key[i + 2] & 0xff) << 16; 112 | case 2: k1 ^= (key[i + 1] & 0xff) << 8; 113 | case 1: k1 ^= (key[i] & 0xff); 114 | 115 | k1 = (((k1 & 0xffff) * c1) + ((((k1 >>> 16) * c1) & 0xffff) << 16)) & 0xffffffff; 116 | k1 = (k1 << 15) | (k1 >>> 17); 117 | k1 = (((k1 & 0xffff) * c2) + ((((k1 >>> 16) * c2) & 0xffff) << 16)) & 0xffffffff; 118 | h1 ^= k1; 119 | } 120 | 121 | h1 ^= key.length; 122 | 123 | h1 ^= h1 >>> 16; 124 | h1 = (((h1 & 0xffff) * 0x85ebca6b) + ((((h1 >>> 16) * 0x85ebca6b) & 0xffff) << 16)) & 0xffffffff; 125 | h1 ^= h1 >>> 13; 126 | h1 = ((((h1 & 0xffff) * 0xc2b2ae35) + ((((h1 >>> 16) * 0xc2b2ae35) & 0xffff) << 16))) & 0xffffffff; 127 | h1 ^= h1 >>> 16; 128 | 129 | return h1 >>> 0; 130 | } 131 | 132 | var murmur = MurmurHashV3; 133 | murmur.v2 = MurmurHashV2; 134 | murmur.v3 = MurmurHashV3; 135 | 136 | if (typeof(module) != 'undefined') { 137 | module.exports = murmur; 138 | } else { 139 | var _previousRoot = _global.murmur; 140 | murmur.noConflict = function() { 141 | _global.murmur = _previousRoot; 142 | return murmur; 143 | } 144 | _global.murmur = murmur; 145 | } 146 | }()); -------------------------------------------------------------------------------- /utils/queue.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-04-21 17:23:23 5 | * @LastEditTime: 2023-04-24 18:13:40 6 | * @Description: 任务队列 7 | * @FilePath: /strick/shin-server/utils/queue.js 8 | */ 9 | import kue from 'kue'; 10 | import config from 'config'; 11 | 12 | const redisConfig = config.get('kueRedis'); 13 | const queue = kue.createQueue({ 14 | prefix: 'q', 15 | redis: redisConfig, 16 | }); 17 | // queue.setMaxListeners(1000) 18 | export default queue; -------------------------------------------------------------------------------- /utils/tools.js: -------------------------------------------------------------------------------- 1 | import moment from 'moment'; 2 | 3 | /** 4 | * 日期格式 5 | */ 6 | export function formatDate(date, format = 'YYYY-MM-DD HH:mm:ss') { 7 | return moment(date).format(format); 8 | } -------------------------------------------------------------------------------- /utils/xfetch.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2020-09-28 11:15:47 5 | * @LastEditTime: 2023-04-24 18:13:21 6 | * @Description: 通信 7 | * @FilePath: /strick/shin-server/utils/xfetch.js 8 | */ 9 | import axios from 'axios'; 10 | import config from 'config'; 11 | 12 | /** 13 | * 请求信息格式 14 | * 15 | * @param {object} req 请求参数 16 | * @param {object} res 返回参数 17 | * @param {number} ms 请求耗时 18 | */ 19 | const createInfo = (req, res, ms) => ` 20 | ******************** request api start ****************** 21 | ${req.method} ${req.baseURL}${req.url} ${ms}ms 22 | request params: ${JSON.stringify(req.params)} 23 | request data: ${JSON.stringify(req.data)} 24 | request body: ${JSON.stringify(res.data)} 25 | ******************** request api end ******************** 26 | `; 27 | 28 | /** 29 | * api请求 30 | * @param {string} url 请求地址 31 | * @param {string} method 请求方法 32 | * @param {string} baseURL 默认域名地址 33 | * @param {object} params 请求参数（GET） 34 | * @param {object} data 请求参数（非GET） 35 | * @param {boolean} disableLog 是否关闭打印日志 36 | */ 37 | export default async ({ url, method, baseURL, params, data, disableLog, headers={} }) => { 38 | const start = Date.now(); 39 | const conf = { 40 | url, 41 | method, 42 | baseURL: baseURL || config.get('services').internalApi, 43 | }; 44 | const _conf = { 45 | headers: { 46 | ...headers 47 | }, 48 | }; 49 | if (method.toUpperCase() === 'GET' || params) { 50 | _conf.params = params; 51 | } else { 52 | !_conf.headers['Content-Type'] && (_conf.headers['Content-Type'] = 'application/json'); 53 | if(typeof data === "string") { 54 | _conf.data = data; 55 | }else { 56 | _conf.data = JSON.stringify(data); 57 | } 58 | } 59 | const req = Object.assign(conf, _conf); 60 | req.validateStatus = status => (status < 500); 61 | const res = await axios(req); 62 | const end = Date.now(); 63 | if (!disableLog) { 64 | logger.info(createInfo(req, res, (end - start))); 65 | global.logMessages && global.logMessages.push(createInfo(req, res, (end - start))); 66 | } 67 | return res; 68 | }; 69 | -------------------------------------------------------------------------------- /worker/agenda.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-03 14:34:44 4 | * @LastEditTime: 2021-02-03 14:51:59 5 | * @LastEditors: strick 6 | * @Description: 7 | * @FilePath: /strick/shin-server/worker/agenda.js 8 | */ 9 | import Agenda from 'agenda'; 10 | import mongodb from '../db/mongodb'; 11 | const connectionOpts = { 12 | mongo: mongodb.connection, 13 | }; 14 | 15 | const agenda = new Agenda(connectionOpts); 16 | module.exports = agenda; 17 | -------------------------------------------------------------------------------- /worker/cronJobs/demo.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-03 14:35:06 4 | * @LastEditTime: 2021-02-05 16:13:43 5 | * @LastEditors: strick 6 | * @Description: 定时任务的demo 7 | * @FilePath: /strick/shin-server/worker/cronJobs/demo.js 8 | */ 9 | import schedule from 'node-schedule'; 10 | import services from '../../services'; 11 | function test(result) { 12 | console.log(result); 13 | } 14 | module.exports = async () => { 15 | const result = await services.backendUserAccount.find(); 16 | //每 30 秒执行一次定时任务 17 | schedule.scheduleJob({ rule: "*/30 * * * * *" }, () => { 18 | test(result); 19 | }); 20 | }; -------------------------------------------------------------------------------- /worker/cronJobs/webMonitorRemove.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-03-17 11:34:10 5 | * @LastEditTime: 2021-09-06 13:55:59 6 | * @Description: 删除过期的监控记录和map文件 7 | * @FilePath: /strick/shin-server/worker/cronJobs/webMonitorRemove.js 8 | */ 9 | import schedule from "node-schedule"; 10 | import moment from "moment"; 11 | import { setCronFormat } from "../../utils"; 12 | import services from "../../services"; 13 | 14 | async function monitor() { 15 | logger.info("开始执行监控日志的删除"); 16 | const startTimestamp = Date.now(); 17 | //删除7天前的记录 18 | const twoWeek = moment().add(-7, 'days').startOf('day').format("YYYY-MM-DD HH:mm"); 19 | await services.webMonitor.delExpiredMonitor({ deadline: twoWeek }); 20 | 21 | //删除21天前的文件 需要调用删除接口 22 | // await services.webMonitor.delExpiredMap({ day:21 }); 23 | 24 | setCronFormat({ name: "监控日志的删除", startTimestamp }); 25 | } 26 | module.exports = () => { 27 | // 每日凌晨3点执行 28 | schedule.scheduleJob("0 0 3 * * *", monitor); 29 | // schedule.scheduleJob("*/10 * * * * *", monitor); 30 | }; 31 | -------------------------------------------------------------------------------- /worker/cronJobs/webMonitorStatis.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-03-17 11:33:41 5 | * @LastEditTime: 2021-09-06 13:26:35 6 | * @Description: 监控统计 7 | * @FilePath: /strick/shin-server/worker/cronJobs/webMonitorStatis.js 8 | */ 9 | import schedule from "node-schedule"; 10 | import moment from "moment"; 11 | import { MONITOR_PROJECT } from "../../utils/constant"; 12 | import { setCronFormat } from "../../utils"; 13 | import services from "../../services"; 14 | 15 | async function monitor() { 16 | logger.info("开始执行监控统计"); 17 | const startTimestamp = Date.now(); 18 | const date = moment().add(-1, 'days').format("YYYYMMDD"); 19 | //判断当前是否已经包含昨日的统计信息 20 | const yesterdayStatis = await services.webMonitor.getOneStatis({ date }); 21 | if(yesterdayStatis) { 22 | logger.info("昨日的监控统计已完成"); 23 | return; 24 | } 25 | const yesterday = moment().add(-1, 'days').startOf('day').format("YYYY-MM-DD HH:mm"); //昨天凌晨 26 | const today = moment().startOf('day').format("YYYY-MM-DD HH:mm"); //今天凌晨 27 | // const message500 = {message: {$like: '%"status":500%'}}; //500的请求 28 | // const message502 = {message: {$like: '%"status":502%'}}; //502的请求 29 | // const message504 = {message: {$like: '%"status":504%'}}; //504的请求 30 | const message500 = { message_status: 500 }; //500的请求 31 | const message502 = { message_status: 502 }; //502的请求 32 | const message504 = { message_status: 504 }; //504的请求 33 | const statis = {}; 34 | for(let i=0; i [item.message_path, item.count]).sort((left, right) => (right[1] - left[1])); //倒序排列 49 | statis[project] = { 50 | allCount: (errorCount + ajaxCount + consoleCount + eventCount + redirectCount), 51 | errorCount, 52 | errorSum, 53 | error500Count, 54 | error502Count, 55 | error504Count, 56 | ajaxCount, 57 | consoleCount, 58 | eventCount, 59 | redirectCount, 60 | error504 61 | }; 62 | } 63 | await services.webMonitor.createStatis({ date, statis: JSON.stringify(statis) }); 64 | setCronFormat({ name: "监控统计", startTimestamp }); 65 | } 66 | module.exports = () => { 67 | // 每日凌晨4点执行 68 | schedule.scheduleJob("0 0 4 * * *", monitor); 69 | // schedule.scheduleJob("*/10 * * * * *", monitor); 70 | }; 71 | -------------------------------------------------------------------------------- /worker/cronJobs/webMonitorWarn.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-07-07 14:28:38 5 | * @LastEditTime: 2021-09-06 13:27:35 6 | * @Description: 监控后台告警 7 | * @FilePath: /strick/shin-server/worker/cronJobs/webMonitorWarn.js 8 | */ 9 | import schedule from "node-schedule"; 10 | import moment from "moment"; 11 | import { setCronFormat } from "../../utils"; 12 | import services from "../../services"; 13 | // 监控对象 14 | const monitor = { 15 | 'test': '测试页面', 16 | }; 17 | // 接收邮箱 18 | const emails = [ 19 | 'test@shin.org', 20 | ]; 21 | async function send(text) { 22 | for(let value of emails) { 23 | await services.webMonitor.tenantSend({ email: value, text }); 24 | } 25 | } 26 | async function warn() { 27 | logger.info("开始执行监控告警"); 28 | const startTimestamp = Date.now(); 29 | const from = moment().add(-5, 'minutes').format("YYYY-MM-DD HH:mm"), //计算前5分钟 30 | to = moment().format("YYYY-MM-DD HH:mm"); 31 | for(let key in monitor) { 32 | const amount = await services.webMonitor.statisticCount({ 33 | from, 34 | to, 35 | other: { 36 | project_subdir: key 37 | } 38 | }); 39 | if(amount < 20) { // 当请求量小于20时，发送白屏警告 40 | const text = `${monitor[key]}白屏异常`; 41 | await send(text); 42 | } 43 | } 44 | 45 | setCronFormat({ name: "监控告警", startTimestamp }); 46 | } 47 | module.exports = () => { 48 | // 每5分钟运行一次 49 | // schedule.scheduleJob("0 */5 * * * *", warn); 50 | // schedule.scheduleJob("*/10 * * * * *", warn); 51 | }; -------------------------------------------------------------------------------- /worker/cronJobs/webPerformanceRemove.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-03-23 18:05:06 5 | * @LastEditTime: 2021-09-06 13:56:31 6 | * @Description: 性能数据清除 7 | * @FilePath: /strick/shin-server/worker/cronJobs/webPerformanceRemove.js 8 | */ 9 | import schedule from "node-schedule"; 10 | import moment from "moment"; 11 | import { setCronFormat } from "../../utils"; 12 | import services from "../../services"; 13 | 14 | async function monitor() { 15 | logger.info("开始执行性能日志的删除"); 16 | const startTimestamp = Date.now(); 17 | //删除28天前的记录 18 | const fourWeek = moment().add(-28, 'days').startOf('day').format("YYYYMMDD"); 19 | await services.webMonitor.delExpiredPerformance({ deadline: fourWeek }); 20 | await services.webMonitor.delExpiredPerformanceStatis({ deadline: fourWeek }); 21 | 22 | setCronFormat({ name: "性能日志的删除", startTimestamp }); 23 | } 24 | module.exports = () => { 25 | // 每日凌晨4点半执行 26 | schedule.scheduleJob("0 30 4 * * *", monitor); 27 | // schedule.scheduleJob("*/10 * * * * *", monitor); 28 | }; 29 | -------------------------------------------------------------------------------- /worker/cronJobs/webPerformanceStatis-func.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @LastEditors: strick 4 | * @Date: 2021-04-01 11:16:38 5 | * @LastEditTime: 2021-09-06 13:25:25 6 | * @Description: 7 | * @FilePath: /strick/shin-server/worker/cronJobs/webPerformanceStatis-func.js 8 | */ 9 | import moment from "moment"; 10 | import { setCronFormat } from "../../utils"; 11 | import { PERFORMANCT_RATE } from "../../utils/constant"; 12 | import services from "../../services"; 13 | 14 | /** 15 | * 计算 95% 的位置 16 | */ 17 | function getFloorNumber(count) { 18 | return Math.floor(count * PERFORMANCT_RATE); 19 | } 20 | /** 21 | * 计算24小时的性能数据 22 | */ 23 | async function calcHour({ project, day }) { 24 | let counts = await services.webMonitor.statisticPerformanceCount({ 25 | project, 26 | day, 27 | group: "hour", 28 | attributes: ["hour"] 29 | }); 30 | if(counts.length == 0) 31 | return null; 32 | const statistic = { 33 | x: [], 34 | load: [], 35 | ready: [], 36 | paint: [], 37 | screen: [], 38 | loadZero: 0, //load时间为0的个数 39 | all: 0, //日志总数 40 | }; 41 | statistic.loadZero = await services.webMonitor.statisticPerformanceCount({ 42 | project, day, 43 | other: { 44 | load: 0 45 | } 46 | }); 47 | statistic.all = await services.webMonitor.statisticPerformanceCount({ 48 | project, day, 49 | }); 50 | counts = counts.map((value) => { 51 | return { 52 | ...value, 53 | offset: getFloorNumber(value.count) 54 | }; 55 | }); 56 | //逐条查询排序后排在95%位置的记录 57 | for(let i=0; i { 108 | return { 109 | ...value, 110 | offset: getFloorNumber(value.count) 111 | } 112 | }); 113 | //逐条查询排序后排在95%位置的记录 114 | for(let i=0; i { 12 | // 每日凌晨3点半执行 13 | schedule.scheduleJob("0 30 3 * * *", monitor); 14 | // schedule.scheduleJob("*/10 * * * * *", monitor); 15 | }; 16 | -------------------------------------------------------------------------------- /worker/triggerJobs/demo.js: -------------------------------------------------------------------------------- 1 | /* 2 | * @Author: strick 3 | * @Date: 2021-02-03 14:35:12 4 | * @LastEditTime: 2021-02-03 15:11:47 5 | * @LastEditors: strick 6 | * @Description: 触发任务的demo 7 | * @FilePath: /strick/shin-server/worker/triggerJobs/demo.js 8 | */ 9 | import services from '../../services'; 10 | 11 | module.exports = (agenda) => { 12 | // 例如满足某种条件触发邮件通知 13 | agenda.define('send email report', (job, done) => { 14 | // 传递进来的数据 15 | const data = job.attrs.data; 16 | console.log(data); 17 | // 触发此任务，需要先引入 agenda.js，然后调用 now() 方法 18 | // import agenda from '../worker/agenda'; 19 | // agenda.now('send email report', { 20 | // username: realName, 21 | // }); 22 | }); 23 | }; 24 | --------------------------------------------------------------------------------