├── .nvmrc ├── .eslintignore ├── .gitignore ├── .prettierrc ├── test ├── setup.js ├── helper.test.js └── express-elasticsearch-logger.test.js ├── .editorconfig ├── .eslintrc.js ├── .travis.yml ├── LICENSE ├── lib ├── readme.hbs ├── elasticsearch.js ├── helper.js ├── config.js └── express-elasticsearch-logger.js ├── express ├── bin │ └── www └── app.js ├── package.json └── README.md /.nvmrc: -------------------------------------------------------------------------------- 1 | 10 -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .idea/ 3 | *.log 4 | **DS_Store 5 | coverage.html 6 | npm-debug.log 7 | lib-cov 8 | dist/tests.js 9 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 80, 3 | "tabWidth": 2, 4 | "useTabs": false, 5 | "semi": false, 6 | "trailingComma": "all", 7 | "jsxBracketSameLine": true 8 | } 9 | -------------------------------------------------------------------------------- /test/setup.js: -------------------------------------------------------------------------------- 1 | const chai = require("chai") 2 | const sinonChai = require("sinon-chai") 3 | const chaiAsPromised = require("chai-as-promised") 4 | const chaid = require("chaid") 5 | 6 | chai.use(sinonChai).use(chaiAsPromised).use(chaid) 7 | chai.should() 8 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = false 10 | insert_final_newline = false 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs") 2 | const path = require("path") 3 | const prettierOptions = JSON.parse( 4 | fs.readFileSync(path.join(__dirname, "./.prettierrc"), "utf8"), 5 | ) 6 | 7 | module.exports = { 8 | parserOptions: { 9 | allowImportExportEverywhere: true, 10 | }, 11 | extends: ["airbnb-base", "prettier"], 12 | env: { 13 | mocha: true, 14 | node: true, 15 | es6: true, 16 | }, 17 | rules: { 18 | "no-underscore-dangle":"off", 19 | "prettier/prettier": [2, prettierOptions], 20 | "no-console": "error", 21 | "func-names": ["error", "never"], 22 | quotes: ["error", "double"], 23 | semi: ["error", "never"], 24 | }, 25 | plugins: ["prettier", "mocha"], 26 | } 27 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 8.9.4 4 | deploy: 5 | provider: npm 6 | email: junrai82@gmail.com 7 | on: 8 | tags: true 9 | api_key: 10 | secure: LyIT8M3N7VlrTydgLwJ0zB6YXwLS8b/4tP9FKi1yDXgazJ2nyJGZqzGFstjmP/FCD8iQ15EVrvWoBU5vIpSsRxHuOX9VVzx9efaKNIEDeaIpvdXw76KHOXLFsoeCG/XTyZq1KnLSzo+9l6ZlkRD3kCZKHpN4S/H/UmXRsmH6TYqimq5hPcLk8ngB3dFFGHH2C00LAV6n4BN1RwY9H6iPNA+iDzrTzakcpk+e2lflJ4K3F/SXAm7u4VAmx5gE5Kwjn3YO3LXIWDgYyRQrSR0JhGOVy3sdx88A8WsvOKsCgqXdV1QmmfkjQWQjQ4KTcqaXhjaO0ndsreQ5bJ4ighqb23efaRLAD/PppZpQb0GkJ3mEc//MxT3MaAU8MhOhoL3xgPLnmrDQshjxfcgGZVrGl0l5FcIxJ7+9cKu0GBM+pikK5VFrN2OPLdkdjja4d/TzTrTJj9ECd/xY3leNu+OSyC1jwRj6K5ivX4PUL6Z+BThy5w5A+eFlpvTr1vB4uhbjKQEFn5X5OmAT/34OetFqh2udk7qGYYY6Mfw+xgyqIKpmaXH4A+Lb4aPvw1eklLzw22i1Cf7oxzDmgnIviOlaGt36IQKGfBAFLnIOrrLQLriXRXtdAoaD7Rb1CQzEHlvATKUonAgucSZmRvWPG3xjIlU/RlFZtFF03/zEoUUfAe4= 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | # Open Works License 2 | 3 | This is version 0.9.4 of the Open Works License 4 | 5 | ## Terms 6 | 7 | Permission is hereby granted by the holder(s) of copyright or other legal 8 | privileges, author(s) or assembler(s), and contributor(s) of this work, to any 9 | person who obtains a copy of this work in any form, to reproduce, modify, 10 | distribute, publish, sell, sublicense, use, and/or otherwise deal in the 11 | licensed material without restriction, provided the following conditions are 12 | met: 13 | 14 | Redistributions, modified or unmodified, in whole or in part, must retain 15 | applicable copyright and other legal privilege notices, the above license 16 | notice, these conditions, and the following disclaimer. 17 | 18 | NO WARRANTY OF ANY KIND IS IMPLIED BY, OR SHOULD BE INFERRED FROM, THIS LICENSE 19 | OR THE ACT OF DISTRIBUTION UNDER THE TERMS OF THIS LICENSE, INCLUDING BUT NOT 20 | LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, 21 | AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS, ASSEMBLERS, OR HOLDERS OF 22 | COPYRIGHT OR OTHER LEGAL PRIVILEGE BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER 23 | LIABILITY, WHETHER IN ACTION OF CONTRACT, TORT, OR OTHERWISE ARISING FROM, OUT 24 | OF, OR IN CONNECTION WITH THE WORK OR THE USE OF OR OTHER DEALINGS IN THE WORK. 25 | -------------------------------------------------------------------------------- /lib/readme.hbs: -------------------------------------------------------------------------------- 1 | # express-elasticsearch-logger [![Build Status](http://img.shields.io/travis/alexmingoia/express-elasticsearch-logger.svg?style=flat)](http://travis-ci.org/alexmingoia/express-elasticsearch-logger) [![Code Coverage](http://img.shields.io/coveralls/alexmingoia/express-elasticsearch-logger.svg?style=flat)](https://coveralls.io/r/alexmingoia/express-elasticsearch-logger) [![NPM version](http://img.shields.io/npm/v/express-elasticsearch-logger.svg?style=flat)](https://www.npmjs.org/package/express-elasticsearch-logger) [![Dependency Status](http://img.shields.io/david/alexmingoia/express-elasticsearch-logger.svg?style=flat)](https://david-dm.org/alexmingoia/express-elasticsearch-logger) 2 | 3 | > Log Express app requests to ElasticSearch. 4 | 5 | {{#module name="express-elasticsearch-logger"}}{{>body}}{{/module}}## Installation 6 | 7 | Install using [npm](https://www.npmjs.org/): 8 | 9 | ```sh 10 | npm install express-elasticsearch-logger 11 | ``` 12 | 13 | ## API Reference 14 | 15 | {{#module name="express-elasticsearch-logger"}}{{>exported}}{{/module}}## Contributing 16 | 17 | Please submit all issues and pull requests to the [alexmingoia/express-elasticsearch-logger](http://github.com/alexmingoia/express-elasticsearch-logger) repository! 18 | 19 | ## Tasks 20 | 21 | List available tasks with `gulp help`. 22 | 23 | ## Tests 24 | 25 | Run tests using `npm test` or `gulp test`. 26 | 27 | ## Support 28 | 29 | If you have any problem or suggestion please open an issue [here](https://github.com/alexmingoia/express-elasticsearch-logger/issues). 30 | -------------------------------------------------------------------------------- /lib/elasticsearch.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | /** 3 | * Log `request` to elasticsearch. 4 | * 5 | * @param {String} index index name 6 | * @param {elasticsearch.Client} client 7 | * @param {Object} doc document to save 8 | * @private 9 | */ 10 | function log(doc, index, client) { 11 | client.index( 12 | { 13 | index, 14 | body: doc, 15 | }, 16 | function (error) { 17 | if (error) { 18 | console.error(error) 19 | } 20 | }, 21 | ) 22 | } 23 | 24 | /** 25 | * Ensure log index and request mapping exist. 26 | * 27 | * @param {Object} config 28 | * @param {elasticsearch.Client} client 29 | * @param {Function} done 30 | * @private 31 | */ 32 | function ensureIndexExists(config, client, done) { 33 | const { settings, mapping: mappings } = config 34 | 35 | client.indices.exists( 36 | { 37 | index: config.index, 38 | }, 39 | (err, response) => { 40 | if (err) { 41 | done(err) 42 | return 43 | } 44 | const { body } = response 45 | if (body) { 46 | client.indices.putMapping( 47 | { 48 | index: config.index, 49 | body: config.mapping, 50 | }, 51 | done, 52 | ) 53 | } else { 54 | client.indices.create( 55 | { 56 | index: config.index, 57 | body: { 58 | mappings, 59 | ...(settings && { settings }), 60 | }, 61 | }, 62 | done, 63 | ) 64 | } 65 | }, 66 | ) 67 | } 68 | 69 | module.exports = { 70 | log, 71 | ensureIndexExists, 72 | } 73 | -------------------------------------------------------------------------------- /express/bin/www: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | /** 4 | * Module dependencies. 5 | */ 6 | 7 | var app = require('../app'); 8 | var debug = require('debug')('express:server'); 9 | var http = require('http'); 10 | 11 | /** 12 | * Get port from environment and store in Express. 13 | */ 14 | 15 | var port = normalizePort(process.env.PORT || '3000'); 16 | app.set('port', port); 17 | 18 | /** 19 | * Create HTTP server. 20 | */ 21 | 22 | var server = http.createServer(app); 23 | 24 | /** 25 | * Listen on provided port, on all network interfaces. 26 | */ 27 | 28 | server.listen(port); 29 | server.on('error', onError); 30 | server.on('listening', onListening); 31 | 32 | /** 33 | * Normalize a port into a number, string, or false. 34 | */ 35 | 36 | function normalizePort(val) { 37 | var port = parseInt(val, 10); 38 | 39 | if (isNaN(port)) { 40 | // named pipe 41 | return val; 42 | } 43 | 44 | if (port >= 0) { 45 | // port number 46 | return port; 47 | } 48 | 49 | return false; 50 | } 51 | 52 | /** 53 | * Event listener for HTTP server "error" event. 54 | */ 55 | 56 | function onError(error) { 57 | if (error.syscall !== 'listen') { 58 | throw error; 59 | } 60 | 61 | var bind = typeof port === 'string' 62 | ? 'Pipe ' + port 63 | : 'Port ' + port; 64 | 65 | // handle specific listen errors with friendly messages 66 | switch (error.code) { 67 | case 'EACCES': 68 | console.error(bind + ' requires elevated privileges'); 69 | process.exit(1); 70 | break; 71 | case 'EADDRINUSE': 72 | console.error(bind + ' is already in use'); 73 | process.exit(1); 74 | break; 75 | default: 76 | throw error; 77 | } 78 | } 79 | 80 | /** 81 | * Event listener for HTTP server "listening" event. 82 | */ 83 | 84 | function onListening() { 85 | var addr = server.address(); 86 | var bind = typeof addr === 'string' 87 | ? 'pipe ' + addr 88 | : 'port ' + addr.port; 89 | debug('Listening on ' + bind); 90 | } 91 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "express-elasticsearch-logger", 3 | "description": "Log Express app requests to ElasticSearch.", 4 | "version": "4.0.0", 5 | "homepage": "https://github.com/alexmingoia/express-elasticsearch-logger", 6 | "author": { 7 | "name": "Alex Mingoia", 8 | "email": "talk@alexmingoia.com" 9 | }, 10 | "contributors": [ 11 | { 12 | "name": "Potsawee Vechpanich", 13 | "email": "potsawee@rentspree.com" 14 | } 15 | ], 16 | "repository": { 17 | "type": "git", 18 | "url": "git://github.com/alexmingoia/express-elasticsearch-logger.git" 19 | }, 20 | "bugs": { 21 | "url": "https://github.com/alexmingoia/express-elasticsearch-logger/issues" 22 | }, 23 | "licenses": [ 24 | { 25 | "type": "Open Works License (OWL)", 26 | "url": "https://github.com/alexmingoia/express-elasticsearch-logger/blob/master/LICENSE" 27 | } 28 | ], 29 | "main": "lib/express-elasticsearch-logger.js", 30 | "files": [ 31 | "dist", 32 | "lib" 33 | ], 34 | "engines": { 35 | "node": ">= 8", 36 | "npm": ">= 5" 37 | }, 38 | "scripts": { 39 | "test": "mocha 'test/**/*.test.js' --require ./test/setup.js", 40 | "lint": "eslint .", 41 | "lint:fix": "npm run lint -- --fix", 42 | "start-express": "DEBUG=* node ./express/bin/www", 43 | "watch": "npm-watch" 44 | }, 45 | "watch": { 46 | "start-express": "{lib,express}/*.js" 47 | }, 48 | "dependencies": { 49 | "@elastic/elasticsearch": "^7.6.1", 50 | "clone-deep": "^0.2.4" 51 | }, 52 | "devDependencies": { 53 | "chai": "^4.2.0", 54 | "chai-as-promised": "^7.1.1", 55 | "chaid": "^1.0.2", 56 | "cookie-parser": "^1.4.5", 57 | "debug": "~2.6.9", 58 | "eslint": "^6.8.0", 59 | "eslint-config-airbnb-base": "^14.1.0", 60 | "eslint-config-prettier": "^6.11.0", 61 | "eslint-plugin-import": "^2.20.2", 62 | "eslint-plugin-mocha": "^6.3.0", 63 | "eslint-plugin-prettier": "^3.1.3", 64 | "express": "^4.17.1", 65 | "http-errors": "^1.7.3", 66 | "jsdoc-to-markdown": "^5.0.3", 67 | "jshint": "^2.9.2", 68 | "jshint-stylish": "^2.1.0", 69 | "mocha": "^7.1.2", 70 | "mocha-lcov-reporter": "1.2.0", 71 | "npm-watch": "^0.6.0", 72 | "prettier": "^2.0.5", 73 | "sinon": "^7.0.0", 74 | "sinon-chai": "^3.5.0", 75 | "supertest": "^1.2.0", 76 | "vinyl-buffer": "^1.0.0", 77 | "vinyl-source-stream": "^1.0.0" 78 | }, 79 | "keywords": [ 80 | "express", 81 | "elasticsearch", 82 | "log", 83 | "logger", 84 | "request" 85 | ], 86 | "publishConfig": { 87 | "access": "public" 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /express/app.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/no-extraneous-dependencies */ 2 | /* eslint-disable no-unused-vars */ 3 | const createError = require("http-errors") 4 | const express = require("express") 5 | const cookieParser = require("cookie-parser") 6 | const expressLogger = require("../lib/express-elasticsearch-logger") 7 | 8 | const app = express() 9 | 10 | app.use(express.json()) 11 | app.use(express.urlencoded({ extended: false })) 12 | app.use(cookieParser()) 13 | 14 | const loggerHost = "http://localhost:9200" 15 | app.use( 16 | expressLogger.requestHandler({ 17 | host: loggerHost, 18 | whitelist: { 19 | request: [ 20 | "body", 21 | "email", 22 | "httpVersion", 23 | "headers", 24 | "method", 25 | "originalUrl", 26 | "path", 27 | "query", 28 | ], 29 | response: ["statusCode", "sent"], 30 | }, 31 | censor: [ 32 | "ssn", 33 | "socialSecurityNumber", 34 | "password", 35 | "form.socialSecurityNumber", 36 | "userInfo.password", 37 | ], 38 | mapping: { 39 | properties: { 40 | response: { 41 | properties: { 42 | sent: { 43 | type: "object", 44 | enabled: false, 45 | }, 46 | }, 47 | }, 48 | request: { 49 | properties: { 50 | email: { 51 | type: "text", 52 | }, 53 | }, 54 | }, 55 | error: { 56 | properties: { 57 | errors: { 58 | type: "object", 59 | enabled: false, 60 | }, 61 | }, 62 | }, 63 | }, 64 | }, 65 | }), 66 | ) 67 | 68 | app.get("/", function (req, res) { 69 | res.send(new Date()) 70 | }) 71 | 72 | const router = express.Router() 73 | 74 | router.get("/", function (req, res) { 75 | res.send(req.query) 76 | }) 77 | 78 | router.post("/", function (req, res) { 79 | res.send(req.body) 80 | }) 81 | 82 | app.use("/route", router) 83 | 84 | app.use(expressLogger.errorHandler) 85 | 86 | // catch 404 and forward to error handler 87 | app.use(function (req, res, next) { 88 | next(createError(404)) 89 | }) 90 | 91 | // error handler 92 | app.use(function (err, req, res, next) { 93 | // set locals, only providing error in development 94 | res.locals.message = err.message 95 | res.locals.error = req.app.get("env") === "development" ? err : {} 96 | 97 | // render the error page 98 | res.status(err.status || 500) 99 | res.send(err) 100 | }) 101 | 102 | module.exports = app 103 | -------------------------------------------------------------------------------- /lib/helper.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-param-reassign */ 2 | const _censorDeep = function (obj, censorKeyArray) { 3 | if (censorKeyArray.length === 0 && typeof obj !== "undefined") { 4 | // this means function reach its base condition, return censor 5 | return "**CENSORED**" 6 | } 7 | const targetKey = censorKeyArray[0] 8 | if (Array.isArray(obj) && obj.length > 0 && targetKey !== undefined) { 9 | if (targetKey === "*") { 10 | const mappedArray = obj.map(function (item) { 11 | const restKey = censorKeyArray.slice(1, censorKeyArray.length) 12 | return _censorDeep(item, restKey) 13 | }) 14 | return mappedArray 15 | } 16 | if (obj[targetKey] !== undefined) { 17 | obj[targetKey] = _censorDeep( 18 | obj[targetKey], 19 | censorKeyArray.splice(1, censorKeyArray.length), 20 | ) 21 | } 22 | return obj 23 | } 24 | if (obj instanceof Object && obj[targetKey] !== "undefined") { 25 | obj[targetKey] = _censorDeep( 26 | obj[targetKey], 27 | censorKeyArray.splice(1, censorKeyArray.length), 28 | ) 29 | return obj 30 | } 31 | return obj 32 | } 33 | 34 | exports.censor = function (obj, censorKeyArray) { 35 | censorKeyArray.forEach(function (key) { 36 | // split with dot notation 37 | const keyArray = key.split(".") 38 | if (keyArray.length >= 1) { 39 | // this mean the key exist 40 | const targetKey = keyArray[0] 41 | if (typeof obj[targetKey] !== "undefined") { 42 | obj[targetKey] = _censorDeep( 43 | obj[targetKey], 44 | keyArray.splice(1, keyArray.length), 45 | ) 46 | } 47 | } 48 | }) 49 | } 50 | 51 | exports._censorDeep = _censorDeep 52 | 53 | const deepMerge = (b, a, mergeArray) => { 54 | if (mergeArray && Array.isArray(a) && Array.isArray(b)) { 55 | return [...b, ...a].filter((v, i, arr) => arr.indexOf(v) === i) 56 | } 57 | if (Array.isArray(a) && Array.isArray(b)) { 58 | return b 59 | } 60 | Object.keys(b).forEach((key) => { 61 | if (typeof b[key] === "object") { 62 | a[key] = deepMerge( 63 | b[key], 64 | typeof a[key] === "object" ? a[key] : {}, 65 | mergeArray, 66 | ) 67 | } else { 68 | a[key] = b[key] 69 | } 70 | }) 71 | return a 72 | } 73 | 74 | exports.deepMerge = deepMerge 75 | 76 | exports.indexDateQuarter = (date) => 77 | `${date.toISOString().substr(0, 4)}-q${Math.ceil( 78 | +date.toISOString().substr(5, 2) / 3, 79 | )}` 80 | 81 | exports.indexDateHalfYear = (date) => 82 | `${date.toISOString().substr(0, 4)}-h${Math.ceil( 83 | +date.toISOString().substr(5, 2) / 6, 84 | )}` 85 | 86 | exports.indexDateMonth = (date) => date.toISOString().substr(0, 7) 87 | -------------------------------------------------------------------------------- /lib/config.js: -------------------------------------------------------------------------------- 1 | const indexSettings = { 2 | index: { 3 | number_of_shards: "3", 4 | number_of_replicas: "2", 5 | refresh_interval: "60s", 6 | analysis: { 7 | normalizer: { 8 | lowercase: { 9 | type: "custom", 10 | char_filter: [], 11 | filter: ["lowercase"], 12 | }, 13 | }, 14 | }, 15 | }, 16 | } 17 | 18 | const defaultMapping = { 19 | dynamic_templates: [ 20 | { 21 | headers_fields: { 22 | match_mapping_type: "string", 23 | path_match: "request.headers.*", 24 | mapping: { 25 | type: "keyword", 26 | ignore_above: 256, 27 | normalizer: "lowercase", 28 | }, 29 | }, 30 | }, 31 | ], 32 | properties: { 33 | env: { 34 | type: "keyword", 35 | index: true, 36 | }, 37 | duration: { 38 | type: "integer", 39 | }, 40 | "@timestamp": { 41 | type: "date", 42 | }, 43 | request: { 44 | properties: { 45 | userId: { 46 | type: "text", 47 | fields: { 48 | keyword: { 49 | type: "keyword", 50 | normalizer: "lowercase", 51 | }, 52 | }, 53 | }, 54 | email: { 55 | type: "text", 56 | fields: { 57 | keyword: { 58 | type: "keyword", 59 | normalizer: "lowercase", 60 | }, 61 | }, 62 | }, 63 | headers: { 64 | properties: { 65 | accept: { 66 | type: "keyword", 67 | normalizer: "lowercase", 68 | }, 69 | acceptencoding: { 70 | type: "keyword", 71 | normalizer: "lowercase", 72 | }, 73 | authorization: { 74 | type: "text", 75 | analyzer: "standard", 76 | }, 77 | cdnloop: { 78 | type: "keyword", 79 | normalizer: "lowercase", 80 | }, 81 | cfconnectingip: { 82 | type: "keyword", 83 | normalizer: "lowercase", 84 | }, 85 | cfipcountry: { 86 | type: "keyword", 87 | normalizer: "lowercase", 88 | }, 89 | cfray: { 90 | type: "keyword", 91 | normalizer: "lowercase", 92 | }, 93 | cfvisitor: { 94 | type: "keyword", 95 | normalizer: "lowercase", 96 | }, 97 | contentlength: { 98 | type: "integer", 99 | }, 100 | contenttype: { 101 | type: "keyword", 102 | normalizer: "lowercase", 103 | }, 104 | host: { 105 | type: "keyword", 106 | normalizer: "lowercase", 107 | }, 108 | useragent: { 109 | type: "text", 110 | analyzer: "standard", 111 | }, 112 | xforwardedfor: { 113 | type: "keyword", 114 | normalizer: "lowercase", 115 | }, 116 | xforwardedhost: { 117 | type: "keyword", 118 | normalizer: "lowercase", 119 | }, 120 | xforwardedport: { 121 | type: "keyword", 122 | normalizer: "lowercase", 123 | }, 124 | xforwardedproto: { 125 | type: "keyword", 126 | normalizer: "lowercase", 127 | }, 128 | xoriginalforwardedfor: { 129 | type: "keyword", 130 | normalizer: "lowercase", 131 | }, 132 | xoriginaluri: { 133 | type: "keyword", 134 | normalizer: "lowercase", 135 | }, 136 | xrealip: { 137 | type: "keyword", 138 | normalizer: "lowercase", 139 | }, 140 | xrequestid: { 141 | type: "keyword", 142 | normalizer: "lowercase", 143 | }, 144 | xscheme: { 145 | type: "keyword", 146 | normalizer: "lowercase", 147 | }, 148 | }, 149 | }, 150 | httpVersion: { 151 | type: "keyword", 152 | }, 153 | method: { 154 | type: "keyword", 155 | }, 156 | originalUrl: { 157 | type: "keyword", 158 | }, 159 | route: { 160 | properties: { 161 | path: { 162 | type: "keyword", 163 | }, 164 | }, 165 | }, 166 | path: { 167 | type: "keyword", 168 | }, 169 | query: { 170 | type: "object", 171 | enabled: false, 172 | }, 173 | body: { 174 | type: "object", 175 | enabled: false, 176 | }, 177 | }, 178 | }, 179 | response: { 180 | properties: { 181 | sent: { 182 | type: "object", 183 | enabled: false, 184 | }, 185 | statusCode: { 186 | type: "integer", 187 | }, 188 | }, 189 | }, 190 | process: { 191 | properties: { 192 | totalmem: { 193 | type: "integer", 194 | }, 195 | freemem: { 196 | type: "integer", 197 | }, 198 | loadavg: { 199 | type: "integer", 200 | }, 201 | }, 202 | }, 203 | error: { 204 | properties: { 205 | errors: { 206 | type: "object", 207 | enabled: false, 208 | }, 209 | code: { 210 | type: "text", 211 | fields: { 212 | keyword: { 213 | type: "keyword", 214 | normalizer: "lowercase", 215 | }, 216 | }, 217 | }, 218 | error: { 219 | type: "text", 220 | }, 221 | error_description: { 222 | type: "text", 223 | }, 224 | message: { 225 | type: "text", 226 | analyzer: "standard", 227 | }, 228 | name: { 229 | type: "keyword", 230 | normalizer: "lowercase", 231 | }, 232 | stack: { 233 | type: "keyword", 234 | }, 235 | type: { 236 | type: "keyword", 237 | }, 238 | }, 239 | }, 240 | }, 241 | } 242 | 243 | const defaultWhiteList = { 244 | request: [ 245 | "userId", 246 | "body", 247 | "email", 248 | "httpVersion", 249 | "headers", 250 | "method", 251 | "originalUrl", 252 | "path", 253 | "query", 254 | ], 255 | response: ["statusCode", "sent", "took"], 256 | error: [ 257 | "message", 258 | "stack", 259 | "type", 260 | "name", 261 | "code", 262 | "errors", 263 | "error", 264 | "error_description", 265 | ], 266 | } 267 | 268 | const defaultCensor = ["password"] 269 | 270 | module.exports = { 271 | defaultMapping, 272 | defaultWhiteList, 273 | defaultCensor, 274 | indexSettings, 275 | } 276 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # express-elasticsearch-logger [![Build Status](http://img.shields.io/travis/alexmingoia/express-elasticsearch-logger.svg?style=flat)](http://travis-ci.org/alexmingoia/express-elasticsearch-logger) [![Code Coverage](http://img.shields.io/coveralls/alexmingoia/express-elasticsearch-logger.svg?style=flat)](https://coveralls.io/r/alexmingoia/express-elasticsearch-logger) [![NPM version](http://img.shields.io/npm/v/express-elasticsearch-logger.svg?style=flat)](https://www.npmjs.org/package/express-elasticsearch-logger) [![Dependency Status](http://img.shields.io/david/alexmingoia/express-elasticsearch-logger.svg?style=flat)](https://david-dm.org/alexmingoia/express-elasticsearch-logger) 2 | 3 | > Log Express app requests to ElasticSearch. 4 | 5 | ## Installation 6 | 7 | Install using [npm](https://www.npmjs.org/): 8 | 9 | ```sh 10 | npm install express-elasticsearch-logger 11 | ``` 12 | 13 | ## API Reference 14 | 15 | 16 | ## express-elasticsearch-logger 17 | 18 | * [express-elasticsearch-logger](#module_express-elasticsearch-logger) 19 | * [.doc](#module_express-elasticsearch-logger.doc) : Object 20 | * [.requestHandler(config, [client])](#module_express-elasticsearch-logger.requestHandler) ⇒ elasticsearchLoggerMiddleware 21 | * [.errorHandler(err, req, res, next)](#module_express-elasticsearch-logger.errorHandler) 22 | * [.skipLog(req, res, next)](#module_express-elasticsearch-logger.skipLog) 23 | 24 | 25 | 26 | ### express-elasticsearch-logger.doc : Object 27 | Document indexed with ElasticSearch. `request` and `response` properties 28 | are included if they are whitelisted by `config.whitelist`. 29 | 30 | **Kind**: static constant of [express-elasticsearch-logger](#module_express-elasticsearch-logger) 31 | **Properties** 32 | 33 | | Name | Type | Description | 34 | | --- | --- | --- | 35 | | env | String | defaults to "development" | 36 | | [error] | Error | error object passed to `next()` | 37 | | duration | Number | milliseconds between request and response | 38 | | request | Object | requst object detail of express | 39 | | request.httpVersion | String | | 40 | | request.headers | Object | | 41 | | request.method | String | | 42 | | request.originalUrl | String | | 43 | | request.route.path | String | | 44 | | request.path | String | | 45 | | request.query | Object | | 46 | | response | Object | | 47 | | response.statusCode | Number | | 48 | | os | Object | | 49 | | os.totalmem | Number | OS total memory in bytes | 50 | | os.freemem | Number | OS free memory in bytes | 51 | | os.loadavg | Array.<Number> | Array of 5, 10, and 15 min averages | 52 | | process | Object | | 53 | | process.memoryUsage | Number | process memory in bytes | 54 | | @timestamp | String | ISO time of request | 55 | 56 | 57 | 58 | ### express-elasticsearch-logger.requestHandler(config, [client]) ⇒ elasticsearchLoggerMiddleware 59 | Returns Express middleware configured according to given `options`. 60 | 61 | Middleware must be mounted before all other middleware to ensure accurate 62 | capture of requests. The error handler must be mounted before other error 63 | handler middleware. 64 | 65 | **Kind**: static method of [express-elasticsearch-logger](#module_express-elasticsearch-logger) 66 | **Returns**: elasticsearchLoggerMiddleware - express middleware 67 | 68 | | Param | Type | Default | Description | 69 | | --- | --- | --- | --- | 70 | | config | Object | | elasticsearch configuration | 71 | | [config.host] | String | "http://localhost:9200" | elasticsearch host to connect | 72 | | [config.index] | String | "log_[YYYY]-h[1\|2]" | elasticsearch index (default: log_YYYY-h1 or log_YYYY-h2 as bi-annually) | 73 | | config.whitelist | Object | | | 74 | | [config.whitelist.request] | Array.<String> | ["userId","body","email","httpVersion","headers","method","originalUrl","path","query"] | request properties to log | 75 | | [config.whitelist.response] | Array.<String> | ["statusCode", "sent", "took"] | response properties to log | 76 | | [config.censor] | Array.<String> | ["password"] | list of request body properties to censor | 77 | | [config.includeDefault] | Boolean | true | include default whitelist and censor the the given config | 78 | | [config.indexPrefix] | String | "log" | elasticsearch index prefix for running index | 79 | | [config.indexSuffixBy] | String | "halfYear" | elasticsearch index suffix for running index, one of m M month (Monthly) q Q quarter (Quarterly) h H halfYear (Bi-annually) | 80 | | [config.indexSettings] | Object | 
{
index: {
    number_of_shards: "3",
    number_of_replicas: "2",
    refresh_interval: "60s",
    analysis: {
      normalizer: {
        lowercase: {
          type: "custom",
          char_filter: [],
          filter: ["lowercase"],
        },
      },
    },
  },
}
| settings in the mapping to be created | 81 | | [client] | elasticsearch.Client | | @elastic/elasticsearch client to be injected | 82 | 83 | **Example** 84 | ```javascript 85 | const express = require('express'); 86 | const logger = require('express-elasticsearch-logger'); 87 | 88 | const app = express(); 89 | 90 | app 91 | .use(logger.requestHandler({ 92 | host: 'http://localhost:9200' 93 | }) 94 | .get('/', function (req, res, next) { 95 | res.sendStatus(204); 96 | }) 97 | .use(logger.errorHandler); 98 | ``` 99 | 100 | * [.requestHandler(config, [client])](#module_express-elasticsearch-logger.requestHandler) ⇒ elasticsearchLoggerMiddleware 101 | 102 | 103 | ### express-elasticsearch-logger.errorHandler(err, req, res, next) 104 | Error handler middleware exposes error to `Response#end` 105 | 106 | This middleware is used in combination with 107 | [requestHandler](#module_express-elasticsearch-logger.requestHandler) to capture request 108 | errors. 109 | 110 | **Kind**: static method of [express-elasticsearch-logger](#module_express-elasticsearch-logger) 111 | 112 | | Param | Type | 113 | | --- | --- | 114 | | err | Error | 115 | | req | express.Request | 116 | | res | express.Response | 117 | | next | express.Request.next | 118 | 119 | 120 | 121 | ### express-elasticsearch-logger.skipLog(req, res, next) 122 | This middleware will mark for skip log 123 | use this middleware for endpoint that is called too often and did not need to log 124 | like healthcheck 125 | 126 | **Kind**: static method of [express-elasticsearch-logger](#module_express-elasticsearch-logger) 127 | 128 | | Param | Type | 129 | | --- | --- | 130 | | req | express.Request | 131 | | res | express.Response | 132 | | next | express.Request.next | 133 | 134 | 135 | ## Contributing 136 | 137 | Please submit all issues and pull requests to the [alexmingoia/express-elasticsearch-logger](http://github.com/alexmingoia/express-elasticsearch-logger) repository! 138 | 139 | ## Tests 140 | 141 | Run tests using `npm test`. 142 | 143 | ## Support 144 | 145 | If you have any problem or suggestion please open an issue [here](https://github.com/alexmingoia/express-elasticsearch-logger/issues). 146 | -------------------------------------------------------------------------------- /lib/express-elasticsearch-logger.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | /*! 3 | * express-elasticsearch-logger 4 | * https://github.com/alexmingoia/express-elasticsearch-logger 5 | */ 6 | 7 | const elasticsearch = require("@elastic/elasticsearch") 8 | const os = require("os") 9 | const cloneDeep = require("clone-deep") 10 | const { 11 | deepMerge, 12 | censor, 13 | indexDateHalfYear, 14 | indexDateQuarter, 15 | indexDateMonth, 16 | } = require("./helper") 17 | const { ensureIndexExists, log } = require("./elasticsearch") 18 | const { 19 | defaultCensor, 20 | defaultMapping, 21 | defaultWhiteList, 22 | indexSettings, 23 | } = require("./config") 24 | 25 | const SUFFIX_MAP = { 26 | m: indexDateMonth, 27 | month: indexDateMonth, 28 | M: indexDateMonth, 29 | q: indexDateQuarter, 30 | quarter: indexDateQuarter, 31 | Q: indexDateQuarter, 32 | h: indexDateHalfYear, 33 | halfYear: indexDateHalfYear, 34 | H: indexDateHalfYear, 35 | } 36 | /** 37 | * @module express-elasticsearch-logger 38 | * @alias logger 39 | */ 40 | 41 | /** 42 | * Returns Express middleware configured according to given `options`. 43 | * 44 | * Middleware must be mounted before all other middleware to ensure accurate 45 | * capture of requests. The error handler must be mounted before other error 46 | * handler middleware. 47 | * 48 | * @example 49 | * 50 | * ```javascript 51 | * const express = require('express'); 52 | * const logger = require('express-elasticsearch-logger'); 53 | * 54 | * const app = express(); 55 | * 56 | * app 57 | * .use(logger.requestHandler({ 58 | * host: 'http://localhost:9200' 59 | * }) 60 | * .get('/', function (req, res, next) { 61 | * res.sendStatus(204); 62 | * }) 63 | * .use(logger.errorHandler); 64 | * ``` 65 | * 66 | * @param {Object} config elasticsearch configuration 67 | * @param {String} [config.host="http://localhost:9200"] elasticsearch host to connect 68 | * @param {String} [config.index="log_[YYYY]-h[1|2]"] elasticsearch index (default: log_YYYY-h1 or log_YYYY-h2 bi-annually) 69 | * @param {Object} config.whitelist 70 | * @param {Array.} [config.whitelist.request=["userId","body","email","httpVersion","headers","method","originalUrl","path","query"]] request properties to log 71 | * @param {Array.} [config.whitelist.response=["statusCode", "sent", "took"]] response properties to log 72 | * @param {Array.} [config.censor=["password"]] list of request body properties to censor 73 | * @param {Boolean} [config.includeDefault=true] include default whitelist and censor the the given config 74 | * @param {String} [config.indexPrefix="log"] elasticsearch index prefix for running index 75 | * @param {String} [config.indexSuffixBy="halfYear"] elasticsearch index suffix for running index, one of m M month (Monthly) q Q quarter (Quarterly) h H halfYear (Bi-annually) 76 | * @param {Object} [config.indexSettings] settings in the mapping to be created 77 | * @param {elasticsearch.Client=} client @elastic/elasticsearch client to be injected 78 | * @returns {elasticsearchLoggerMiddleware} express middleware 79 | */ 80 | exports.requestHandler = function (config = {}, client) { 81 | /** Caching */ 82 | const createdIndexes = new Set() 83 | const isIndexCreated = (name) => createdIndexes.has(name) 84 | const setIndexCreated = (name) => createdIndexes.add(name) 85 | 86 | /** Lazy Evaluation Index Name */ 87 | const suffixFn = SUFFIX_MAP[config.indexSuffixBy] || indexDateHalfYear 88 | const getIndexName = () => 89 | `${config.indexPrefix || "log"}_${suffixFn(new Date())}` 90 | const mergeArray = config.includeDefault !== false 91 | const cfg = deepMerge( 92 | config, 93 | { 94 | node: config.host || "http://localhost:9200", // elastic new client use node 95 | whitelist: defaultWhiteList, 96 | censor: defaultCensor, 97 | mapping: defaultMapping, 98 | indexSettings, 99 | }, 100 | mergeArray, 101 | ) 102 | const esClient = client || new elasticsearch.Client(cfg) 103 | 104 | /** 105 | * Logs request and response information to ElasticSearch. 106 | * 107 | * @param {express.Request} req 108 | * @param {express.Response} res 109 | * @param {express.Request.next} next 110 | */ 111 | return (req, res, next) => { 112 | const { end } = res 113 | const start = Date.now() 114 | const indexName = config.index || getIndexName() 115 | /** 116 | * Document indexed with ElasticSearch. `request` and `response` properties 117 | * are included if they are whitelisted by `config.whitelist`. 118 | * 119 | * @property {String} env defaults to "development" 120 | * @property {Error=} error error object passed to `next()` 121 | * @property {Number} duration milliseconds between request and response 122 | * @property {Object} request requst object detail of express 123 | * @property {String} request.httpVersion 124 | * @property {Object} request.headers 125 | * @property {String} request.method 126 | * @property {String} request.originalUrl 127 | * @property {String} request.route.path 128 | * @property {String} request.path 129 | * @property {Object} request.query 130 | * @property {Object} response 131 | * @property {Number} response.statusCode 132 | * @property {Object} os 133 | * @property {Number} os.totalmem OS total memory in bytes 134 | * @property {Number} os.freemem OS free memory in bytes 135 | * @property {Array.} os.loadavg Array of 5, 10, and 15 min averages 136 | * @property {Object} process 137 | * @property {Number} process.memoryUsage process memory in bytes 138 | * @property {String} @timestamp ISO time of request 139 | * @type {Object} 140 | * @memberof module:express-elasticsearch-logger 141 | */ 142 | const doc = { 143 | env: process.env.NODE_ENV || "development", 144 | request: {}, 145 | "@timestamp": new Date().toISOString(), 146 | } 147 | cfg.whitelist.request.forEach(function (key) { 148 | if (typeof req[key] === "object") { 149 | doc.request[key] = cloneDeep(req[key]) 150 | } else { 151 | doc.request[key] = req[key] 152 | } 153 | }) 154 | 155 | if (doc.request.body) { 156 | censor(doc.request.body, cfg.censor) 157 | } 158 | 159 | // monkey patch Response#end to capture request time 160 | res.end = function (...args) { 161 | res.end = end 162 | end.apply(res, args) 163 | 164 | if (req.skipLog) { 165 | return 166 | } 167 | 168 | doc.response = {} 169 | doc.duration = Date.now() - start 170 | 171 | if (req.route && req.route.path) { 172 | doc.request.route = { 173 | path: req.route.path, 174 | } 175 | } 176 | 177 | if (res.error) { 178 | doc.error = {} 179 | 180 | Object.getOwnPropertyNames(res.error).forEach(function (key) { 181 | if (cfg.whitelist.error.includes(key)) { 182 | doc.error[key] = res.error[key] 183 | } 184 | }) 185 | } 186 | 187 | doc.os = { 188 | totalmem: os.totalmem(), // bytes 189 | freemem: os.freemem(), // bytes 190 | loadavg: os.loadavg(), // array of 5, 10, and 15 min averages 191 | } 192 | 193 | doc.process = { 194 | memory: process.memoryUsage(), // bytes 195 | } 196 | 197 | cfg.whitelist.response.forEach(function (key) { 198 | doc.response[key] = res[key] 199 | }) 200 | if (isIndexCreated(indexName)) { 201 | log(doc, indexName, esClient) 202 | } else { 203 | ensureIndexExists( 204 | { 205 | index: indexName, 206 | settings: cfg.indexSettings, 207 | mapping: cfg.mapping, 208 | }, 209 | esClient, 210 | (error) => { 211 | if (error) { 212 | console.error(error) 213 | return 214 | } 215 | setIndexCreated(indexName) 216 | log(doc, indexName, esClient) 217 | }, 218 | ) 219 | } 220 | } 221 | 222 | next() 223 | } 224 | } 225 | 226 | /** 227 | * Error handler middleware exposes error to `Response#end` 228 | * 229 | * This middleware is used in combination with 230 | * {@link module:express-elasticsearch-logger.requestHandler} to capture request 231 | * errors. 232 | * 233 | * @param {Error} err 234 | * @param {express.Request} req 235 | * @param {express.Response} res 236 | * @param {express.Request.next} next 237 | */ 238 | exports.errorHandler = function (err, req, res, next) { 239 | res.error = err 240 | next(err) 241 | } 242 | 243 | /** 244 | * This middleware will mark for skip log 245 | * use this middleware for endpoint that is called too often and did not need to log 246 | * like healthcheck 247 | * 248 | * @param {express.Request} req 249 | * @param {express.Response} res 250 | * @param {express.Request.next} next 251 | */ 252 | exports.skipLog = function (req, res, next) { 253 | req.skipLog = true 254 | next() 255 | } 256 | -------------------------------------------------------------------------------- /test/helper.test.js: -------------------------------------------------------------------------------- 1 | const { expect } = require("chai") 2 | const { 3 | _censorDeep, 4 | censor, 5 | indexDateHalfYear, 6 | indexDateMonth, 7 | deepMerge, 8 | indexDateQuarter, 9 | } = require("../lib/helper") 10 | 11 | const cencorString = "**CENSORED**" 12 | 13 | describe("censor helpers", function () { 14 | context("#_censorDeep", function () { 15 | it("should be able to censor out 1 level config", function () { 16 | const original = { 17 | a: "should censor me", 18 | b: null, 19 | c: "hello", 20 | d: "haha", 21 | } 22 | const newObj = _censorDeep(original, "a".split(".")) 23 | expect(newObj).to.deep.equal({ 24 | a: cencorString, 25 | b: null, 26 | c: "hello", 27 | d: "haha", 28 | }) 29 | }) 30 | 31 | it("should be able to censor out multi level config", function () { 32 | const original = { 33 | a: { 34 | aa: { aaa: "keep me out" }, 35 | }, 36 | b: null, 37 | c: "hello", 38 | d: "haha", 39 | } 40 | const newObj = _censorDeep(original, "a.aa.aaa".split(".")) 41 | expect(newObj).to.have.nested.property("a.aa.aaa", cencorString) 42 | }) 43 | 44 | it("should remain the same object if the config is at fault", function () { 45 | const original = { 46 | a: { 47 | aa: { aaa: "keep me out" }, 48 | }, 49 | b: null, 50 | c: "hello", 51 | d: "haha", 52 | } 53 | const newObj = _censorDeep(original, "a.azd.ewx.1.2".split(".")) 54 | expect(newObj).to.have.deep.equal(original) 55 | }) 56 | 57 | it("should be able to censor data in array with * position and nested object", function () { 58 | const original = { 59 | Questions: [ 60 | { 61 | question: "this is q1", 62 | answer: "this is a1", 63 | }, 64 | { 65 | question: "this is q2", 66 | answer: "this is a2", 67 | }, 68 | { 69 | question: "this is q3", 70 | answer: "this is a3", 71 | }, 72 | ], 73 | } 74 | const expected = { 75 | Questions: [ 76 | { 77 | question: "this is q1", 78 | answer: cencorString, 79 | }, 80 | { 81 | question: "this is q2", 82 | answer: cencorString, 83 | }, 84 | { 85 | question: "this is q3", 86 | answer: cencorString, 87 | }, 88 | ], 89 | } 90 | const newObj = _censorDeep(original, "Questions.*.answer".split(".")) 91 | expect(newObj).to.have.deep.equal(expected) 92 | }) 93 | 94 | it("should be able to censor data in array with normal position", function () { 95 | const original = { 96 | Questions: [ 97 | { 98 | question: "this is q1", 99 | answer: "this is a1", 100 | }, 101 | { 102 | question: "this is q2", 103 | answer: "this is a2", 104 | }, 105 | { 106 | question: "this is q3", 107 | answer: "this is a3", 108 | }, 109 | ], 110 | } 111 | const expected = { 112 | Questions: [ 113 | { 114 | question: "this is q1", 115 | answer: cencorString, 116 | }, 117 | { 118 | question: "this is q2", 119 | answer: "this is a2", 120 | }, 121 | { 122 | question: "this is q3", 123 | answer: "this is a3", 124 | }, 125 | ], 126 | } 127 | const newObj = _censorDeep(original, "Questions.0.answer".split(".")) 128 | expect(newObj).to.have.deep.equal(expected) 129 | }) 130 | 131 | it("should be able to censor data in array with nested array", function () { 132 | const original = { 133 | Questions: [ 134 | { 135 | question: ["haha", "haha"], 136 | answer: "this is a1", 137 | }, 138 | { 139 | question: ["haha", "haha"], 140 | answer: "this is a2", 141 | }, 142 | ], 143 | } 144 | const expected = { 145 | Questions: [ 146 | { 147 | question: ["haha", cencorString], 148 | answer: "this is a1", 149 | }, 150 | { 151 | question: ["haha", cencorString], 152 | answer: "this is a2", 153 | }, 154 | ], 155 | } 156 | const newObj = _censorDeep(original, "Questions.*.question.1".split(".")) 157 | expect(newObj).to.have.deep.equal(expected) 158 | }) 159 | 160 | it("should be able to censor data in array with nested empty array", function () { 161 | const original = { 162 | Questions: [ 163 | { 164 | question: ["hahe"], 165 | answer: "this is a1", 166 | }, 167 | { 168 | question: ["haha", "haha"], 169 | answer: "this is a2", 170 | }, 171 | ], 172 | } 173 | const expected = { 174 | Questions: [ 175 | { 176 | question: ["hahe"], 177 | answer: "this is a1", 178 | }, 179 | { 180 | question: ["haha", cencorString], 181 | answer: "this is a2", 182 | }, 183 | ], 184 | } 185 | const newObj = _censorDeep(original, "Questions.*.question.1".split(".")) 186 | expect(newObj).to.have.deep.equal(expected) 187 | }) 188 | }) 189 | context("#censor", function () { 190 | it("should be able to censor deep config", function () { 191 | const original = { z: { zz: { zzz: "take_me_out" } } } 192 | censor(original, ["z.zz.zzz"]) 193 | expect(original).to.have.nested.property("z.zz.zzz", cencorString) 194 | }) 195 | it("should be able to censor 1 level config", function () { 196 | const original = { z: { zz: { zzz: "take_me_out" } } } 197 | censor(original, ["z"]) 198 | expect(original).to.have.nested.property("z", cencorString) 199 | }) 200 | 201 | it("should be able to censor multiple configs", function () { 202 | const original = { 203 | z: { zz: { zzz: "take_me_out" } }, 204 | b: "this is to be censored too", 205 | c: { y: "out too", x: "remain cool" }, 206 | } 207 | censor(original, ["z.zz.zzz", "b", "c.y"]) 208 | expect(original).to.deep.equal({ 209 | z: { zz: { zzz: cencorString } }, 210 | b: cencorString, 211 | c: { y: cencorString, x: "remain cool" }, 212 | }) 213 | }) 214 | }) 215 | }) 216 | 217 | describe("indexDate helpers", () => { 218 | context("indexDateMonth", () => { 219 | it("should get the YYYY-MM format for given date", () => { 220 | indexDateMonth(new Date("2020-05-01T10:34:57.685Z")).should.equal( 221 | "2020-05", 222 | ) 223 | indexDateMonth(new Date("2020-04-30T10:34:57.685Z")).should.equal( 224 | "2020-04", 225 | ) 226 | indexDateMonth(new Date("2019-01-01T00:34:57.685Z")).should.equal( 227 | "2019-01", 228 | ) 229 | indexDateMonth(new Date("2020-02-29T00:34:57.685Z")).should.equal( 230 | "2020-02", 231 | ) 232 | indexDateMonth(new Date("2020-12-31T23:59:59.999Z")).should.equal( 233 | "2020-12", 234 | ) 235 | }) 236 | }) 237 | context("indexDateQuarter", () => { 238 | it("should get the YYYY-q{number} format for given date", () => { 239 | indexDateQuarter(new Date("2020-05-01T10:34:57.685Z")).should.equal( 240 | "2020-q2", 241 | ) 242 | indexDateQuarter(new Date("2020-03-30T10:34:57.685Z")).should.equal( 243 | "2020-q1", 244 | ) 245 | indexDateQuarter(new Date("2019-01-01T00:34:57.685Z")).should.equal( 246 | "2019-q1", 247 | ) 248 | indexDateQuarter(new Date("2020-02-29T00:34:57.685Z")).should.equal( 249 | "2020-q1", 250 | ) 251 | indexDateQuarter(new Date("2020-12-31T23:59:59.999Z")).should.equal( 252 | "2020-q4", 253 | ) 254 | indexDateQuarter(new Date("2020-09-30T23:59:59.999Z")).should.equal( 255 | "2020-q3", 256 | ) 257 | indexDateQuarter(new Date("2020-10-01T00:00:00.000Z")).should.equal( 258 | "2020-q4", 259 | ) 260 | }) 261 | }) 262 | context("indexDateHalfYear", () => { 263 | it("should get the YYYY-h{number} format for given date", () => { 264 | indexDateHalfYear(new Date("2020-05-01T10:34:57.685Z")).should.equal( 265 | "2020-h1", 266 | ) 267 | indexDateHalfYear(new Date("2020-03-30T10:34:57.685Z")).should.equal( 268 | "2020-h1", 269 | ) 270 | indexDateHalfYear(new Date("2019-01-01T00:34:57.685Z")).should.equal( 271 | "2019-h1", 272 | ) 273 | indexDateHalfYear(new Date("2020-02-29T00:34:57.685Z")).should.equal( 274 | "2020-h1", 275 | ) 276 | indexDateHalfYear(new Date("2020-12-31T23:59:59.999Z")).should.equal( 277 | "2020-h2", 278 | ) 279 | indexDateHalfYear(new Date("2020-09-30T23:59:59.999Z")).should.equal( 280 | "2020-h2", 281 | ) 282 | indexDateHalfYear(new Date("2020-10-01T00:00:00.000Z")).should.equal( 283 | "2020-h2", 284 | ) 285 | }) 286 | }) 287 | }) 288 | 289 | context("deepMerge", () => { 290 | it("should able to merge object in nested but not array", () => { 291 | const result = deepMerge( 292 | { a: { b: ["green"] }, c: { d: "hi" } }, 293 | { a: { b: ["sorasak", "srirussamee"] }, c: { e: "hi" } }, 294 | ) 295 | result.should.deep.equal({ 296 | a: { b: ["green"] }, 297 | c: { e: "hi", d: "hi" }, 298 | }) 299 | }) 300 | it("should able to merge object in nested object and array if true flag is sent as third argument", () => { 301 | const result = deepMerge( 302 | { a: { b: ["green"] }, c: { d: "hi", f: ["hello", "world"] } }, 303 | { a: { b: ["sorasak"] }, c: { e: "hi", f: ["world", "thailand"] } }, 304 | true, 305 | ) 306 | result.should.deep.equal({ 307 | a: { b: ["green", "sorasak"] }, 308 | c: { e: "hi", d: "hi", f: ["hello", "world", "thailand"] }, 309 | }) 310 | }) 311 | }) 312 | -------------------------------------------------------------------------------- /test/express-elasticsearch-logger.test.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-unused-vars */ 2 | /* eslint-disable import/order */ 3 | /* eslint-disable no-unused-expressions */ 4 | 5 | const { expect } = require("chai") 6 | const { 7 | requestHandler, 8 | errorHandler, 9 | skipLog, 10 | } = require("../lib/express-elasticsearch-logger") 11 | const { defaultMapping } = require("../lib/config") 12 | const express = require("express") 13 | const request = require("supertest") 14 | const sinon = require("sinon") 15 | 16 | describe("express-elasticsearch-logger module", function () { 17 | const config = { 18 | host: "http://localhost:9200", 19 | } 20 | 21 | const client = { 22 | index(params, cb) { 23 | expect(params.body).to.contain.keys([ 24 | "duration", 25 | "request", 26 | "response", 27 | "@timestamp", 28 | "os", 29 | "process", 30 | ]) 31 | expect(params.body.request).to.have.property("method", "GET", "env") 32 | expect(params.body.request).to.have.property("path", "/test") 33 | expect(params.body.request).to.have.property( 34 | "originalUrl", 35 | "/test?test=it", 36 | ) 37 | cb() 38 | }, 39 | indices: { 40 | exists(a, cb) { 41 | cb(null, { body: false }) 42 | }, 43 | create(a, cb) { 44 | cb() 45 | }, 46 | }, 47 | } 48 | 49 | describe("requestHandler", function () { 50 | it("returns express logging middleware", function () { 51 | const middleware = requestHandler() 52 | expect(middleware).to.be.a("function") 53 | }) 54 | 55 | it("logs requests", function (done) { 56 | const app = express() 57 | 58 | app.use(requestHandler(config, client)).get("/test", function (req, res) { 59 | res.sendStatus(200) 60 | }) 61 | 62 | request(app).get("/test").query({ test: "it" }).expect(200).end(done) 63 | }) 64 | }) 65 | 66 | describe("errorHandler", function () { 67 | it("logs errors within requests", function (done) { 68 | const app = express() 69 | 70 | app 71 | .use(requestHandler(config, client)) 72 | .get("/test", function (req, res, next) { 73 | next(new Error("test")) 74 | }) 75 | .use(errorHandler) 76 | .use(function (err, req, res, next) { 77 | expect(err).to.have.property("message", "test") 78 | res.sendStatus(555) 79 | }) 80 | 81 | request(app).get("/test").query({ test: "it" }).expect(555).end(done) 82 | }) 83 | }) 84 | 85 | describe("config", function () { 86 | it("should be able to config error whitelist", function (done) { 87 | const indexFunctionStub = sinon.spy() 88 | const clientTest = { 89 | index: indexFunctionStub, 90 | indices: { 91 | exists(a, cb) { 92 | cb(null, { body: false }) 93 | }, 94 | create(a, cb) { 95 | cb() 96 | }, 97 | }, 98 | } 99 | const app = express() 100 | app 101 | .use(requestHandler(config, clientTest)) 102 | .get("/test", function (req, res, next) { 103 | const e = new Error("test") 104 | e.name = "this-is-the-error" 105 | e.something = "should-not-see-this" 106 | next(e) 107 | }) 108 | .use(errorHandler) 109 | .use(function (err, req, res, next) { 110 | res.sendStatus(555) 111 | }) 112 | 113 | request(app) 114 | .get("/test") 115 | .query({ test: "it" }) 116 | .expect(function () { 117 | expect(indexFunctionStub.calledOnce).to.be.true 118 | const params = indexFunctionStub.firstCall.args[0] 119 | expect(params.body).to.contain.keys(["error", "request"]) 120 | expect(params.body.error).to.have.property( 121 | "name", 122 | "this-is-the-error", 123 | ) 124 | expect(params.body.error).to.not.have.property("something") 125 | }) 126 | .end(done) 127 | }) 128 | }) 129 | 130 | describe("skipLog", function () { 131 | it("should add skipLog to req", function (done) { 132 | const app = express() 133 | app.use(skipLog).get("/test", function (req, res) { 134 | expect(req).to.have.property("skipLog") 135 | res.sendStatus(200) 136 | }) 137 | 138 | request(app).get("/test").expect(200).end(done) 139 | }) 140 | }) 141 | }) 142 | 143 | describe("elasticsearch index creation", () => { 144 | const stubESIndex = sinon.stub().callsArg(1) 145 | const stubESIndicesExists = sinon 146 | .stub() 147 | .callsArgWith(1, null, { body: false }) 148 | const stubESIndicesCreate = sinon.stub().callsArg(1) 149 | const stubESIndicesPutMapping = sinon.stub().callsArg(1) 150 | const config = { 151 | host: "http://localhost:9200", 152 | } 153 | const getClient = (custom) => ({ 154 | index: stubESIndex, 155 | indices: { 156 | exists: stubESIndicesExists, 157 | putMapping: stubESIndicesPutMapping, 158 | create: stubESIndicesCreate, 159 | ...custom, 160 | }, 161 | }) 162 | const callServer = async (cfg = config, client) => { 163 | const app = express() 164 | app 165 | .use(requestHandler(cfg, client || getClient())) 166 | .get("/test", function (req, res) { 167 | res.sendStatus(200) 168 | }) 169 | const rq = request(app) 170 | await rq.get("/test").query({ test: "it" }).expect(200) 171 | return { 172 | callAgain: () => rq.get("/test").query({ test: "it" }).expect(200), 173 | } 174 | } 175 | let clock 176 | const date = new Date("2020-09-30T23:59:59.999Z") 177 | before(() => { 178 | clock = sinon.useFakeTimers(date) 179 | }) 180 | 181 | after(() => { 182 | clock.restore() 183 | }) 184 | describe("default index name", () => { 185 | beforeEach(() => { 186 | stubESIndex.resetHistory() 187 | }) 188 | afterEach(() => { 189 | clock.setSystemTime(date) 190 | }) 191 | it("should create the correct index pattern that default to half year pattern", async () => { 192 | await callServer() 193 | stubESIndex.should.be.called 194 | const { index } = stubESIndex.lastCall.args[0] 195 | index.should.be.equal("log_2020-h2") 196 | }) 197 | }) 198 | describe("configurable index name", () => { 199 | beforeEach(() => { 200 | stubESIndex.resetHistory() 201 | }) 202 | afterEach(() => { 203 | clock.setSystemTime(date) 204 | }) 205 | it("should able to config prefix", async () => { 206 | await callServer({ ...config, indexPrefix: "prod" }) 207 | stubESIndex.should.be.called 208 | const { index } = stubESIndex.lastCall.args[0] 209 | index.should.be.equal("prod_2020-h2") 210 | }) 211 | it("should able to suffix of index name by month", async () => { 212 | await callServer({ ...config, indexSuffixBy: "m" }) 213 | stubESIndex.getCalls()[0].args[0].index.should.be.equal("log_2020-09") 214 | clock.setSystemTime(new Date("2010-02-01T23:59:59.999Z")) 215 | await callServer({ ...config, indexSuffixBy: "month" }) 216 | stubESIndex.getCalls()[1].args[0].index.should.be.equal("log_2010-02") 217 | clock.setSystemTime(new Date("2019-03-01T23:59:59.999Z")) 218 | await callServer({ 219 | ...config, 220 | indexPrefix: "test_month", 221 | indexSuffixBy: "M", 222 | }) 223 | stubESIndex 224 | .getCalls()[2] 225 | .args[0].index.should.be.equal("test_month_2019-03") 226 | }) 227 | it("should able to suffix of index name by quartery", async () => { 228 | await callServer({ ...config, indexSuffixBy: "q" }) 229 | stubESIndex.getCalls()[0].args[0].index.should.be.equal("log_2020-q3") 230 | clock.setSystemTime(new Date("2010-04-01T23:59:59.999Z")) 231 | await callServer({ ...config, indexSuffixBy: "quarter" }) 232 | stubESIndex.getCalls()[1].args[0].index.should.be.equal("log_2010-q2") 233 | clock.setSystemTime(new Date("2019-03-01T23:59:59.999Z")) 234 | await callServer({ 235 | ...config, 236 | indexPrefix: "test_q", 237 | indexSuffixBy: "Q", 238 | }) 239 | stubESIndex.getCalls()[2].args[0].index.should.be.equal("test_q_2019-q1") 240 | }) 241 | it("should able to suffix of index name by bi-annually", async () => { 242 | await callServer({ ...config, indexSuffixBy: "h" }) 243 | stubESIndex.getCalls()[0].args[0].index.should.be.equal("log_2020-h2") 244 | clock.setSystemTime(new Date("2010-04-01T23:59:59.999Z")) 245 | await callServer({ ...config, indexSuffixBy: "halfYear" }) 246 | stubESIndex.getCalls()[1].args[0].index.should.be.equal("log_2010-h1") 247 | clock.setSystemTime(new Date("2019-03-01T23:59:59.999Z")) 248 | await callServer({ 249 | ...config, 250 | indexPrefix: "test_h", 251 | indexSuffixBy: "H", 252 | }) 253 | stubESIndex.getCalls()[2].args[0].index.should.be.equal("test_h_2019-h1") 254 | }) 255 | it("should use default to bi-annually when invalid value is sent", async () => { 256 | await callServer({ ...config, indexSuffixBy: "mmmmm" }) 257 | stubESIndex.getCalls()[0].args[0].index.should.be.equal("log_2020-h2") 258 | clock.setSystemTime(new Date("2019-03-01T23:59:59.999Z")) 259 | await callServer({ 260 | ...config, 261 | indexPrefix: "test_undefined", 262 | indexSuffixBy: undefined, 263 | }) 264 | stubESIndex 265 | .getCalls()[1] 266 | .args[0].index.should.be.equal("test_undefined_2019-h1") 267 | }) 268 | }) 269 | 270 | describe("call create index", () => { 271 | beforeEach(() => { 272 | stubESIndex.resetHistory() 273 | stubESIndicesExists.resetHistory() 274 | stubESIndicesPutMapping.resetHistory() 275 | stubESIndicesCreate.resetHistory() 276 | }) 277 | afterEach(() => { 278 | clock.setSystemTime(date) 279 | }) 280 | it("it should not call createIndex if it is already created", async () => { 281 | const { callAgain } = await callServer(config) 282 | await callAgain() 283 | await callAgain() 284 | await callAgain() 285 | await callAgain() 286 | stubESIndex.getCalls()[0].args[0].index.should.be.equal("log_2020-h2") 287 | stubESIndex.getCalls()[1].args[0].index.should.be.equal("log_2020-h2") 288 | stubESIndex.getCalls()[2].args[0].index.should.be.equal("log_2020-h2") 289 | stubESIndex.getCalls()[3].args[0].index.should.be.equal("log_2020-h2") 290 | stubESIndex.getCalls()[4].args[0].index.should.be.equal("log_2020-h2") 291 | stubESIndicesExists.should.be.calledOnce 292 | stubESIndicesCreate.should.be.calledOnce 293 | }) 294 | it("it should call send log to correct index name if it has change the date of request", async () => { 295 | const { callAgain } = await callServer({ ...config, indexSuffixBy: "m" }) 296 | await callAgain() 297 | await callAgain() 298 | clock.setSystemTime(new Date("2019-03-01T23:59:59.999Z")) 299 | await callAgain() 300 | await callAgain() 301 | stubESIndex.getCalls()[0].args[0].index.should.be.equal("log_2020-09") 302 | stubESIndex.getCalls()[1].args[0].index.should.be.equal("log_2020-09") 303 | stubESIndex.getCalls()[2].args[0].index.should.be.equal("log_2020-09") 304 | stubESIndex.getCalls()[3].args[0].index.should.be.equal("log_2019-03") 305 | stubESIndex.getCalls()[4].args[0].index.should.be.equal("log_2019-03") 306 | stubESIndicesExists.should.be.calledTwice 307 | stubESIndicesCreate.should.be.calledTwice 308 | }) 309 | }) 310 | 311 | describe("merge config", () => { 312 | beforeEach(() => { 313 | stubESIndex.resetHistory() 314 | stubESIndicesExists.resetHistory() 315 | stubESIndicesPutMapping.resetHistory() 316 | stubESIndicesCreate.resetHistory() 317 | }) 318 | afterEach(() => { 319 | clock.setSystemTime(date) 320 | }) 321 | it("it should merge config object/array type if includeDefault is set to true (default)", async () => { 322 | const app = express() 323 | app 324 | .use((req, res, next) => { 325 | req.my_name = "green" 326 | next() 327 | }) 328 | .use( 329 | requestHandler( 330 | { 331 | ...config, 332 | whitelist: { 333 | request: ["my_name"], 334 | }, 335 | mapping: { 336 | properties: { 337 | request: { 338 | properties: { 339 | my_field: { 340 | type: "text", 341 | }, 342 | }, 343 | }, 344 | }, 345 | }, 346 | }, 347 | getClient(), 348 | ), 349 | ) 350 | .get("/test", function (req, res) { 351 | res.sendStatus(200) 352 | }) 353 | const rq = request(app) 354 | await rq.get("/test").query({ test: "it" }).expect(200) 355 | 356 | stubESIndicesCreate.should.be.called 357 | const { 358 | body: { mappings }, 359 | } = stubESIndicesCreate.getCalls()[0].args[0] 360 | mappings.should.be.deep.equal({ 361 | ...defaultMapping, 362 | properties: { 363 | ...defaultMapping.properties, 364 | request: { 365 | properties: { 366 | ...defaultMapping.properties.request.properties, 367 | my_field: { 368 | type: "text", 369 | }, 370 | }, 371 | }, 372 | }, 373 | }) 374 | 375 | stubESIndex.should.be.called 376 | const { body } = stubESIndex.getCalls()[0].args[0] 377 | body.request.should.have.property("query").that.deep.equal({ test: "it" }) 378 | body.request.should.have.property("my_name", "green") 379 | }) 380 | 381 | it("it should merge config object and replace the array type if includeDefault is set to false", async () => { 382 | const app = express() 383 | app 384 | .use((req, res, next) => { 385 | req.my_name = "green" 386 | next() 387 | }) 388 | .use( 389 | requestHandler( 390 | { 391 | ...config, 392 | includeDefault: false, 393 | whitelist: { 394 | request: ["my_name"], 395 | }, 396 | mapping: { 397 | properties: { 398 | request: { 399 | properties: { 400 | my_field: { 401 | type: "text", 402 | }, 403 | }, 404 | }, 405 | }, 406 | }, 407 | }, 408 | getClient(), 409 | ), 410 | ) 411 | .get("/test", function (req, res) { 412 | res.sendStatus(200) 413 | }) 414 | const rq = request(app) 415 | await rq.get("/test").query({ test: "it" }).expect(200) 416 | 417 | stubESIndicesCreate.should.be.called 418 | const { 419 | body: { mappings }, 420 | } = stubESIndicesCreate.getCalls()[0].args[0] 421 | mappings.should.be.deep.equal({ 422 | ...defaultMapping, 423 | properties: { 424 | ...defaultMapping.properties, 425 | request: { 426 | properties: { 427 | ...defaultMapping.properties.request.properties, 428 | my_field: { 429 | type: "text", 430 | }, 431 | }, 432 | }, 433 | }, 434 | }) 435 | stubESIndicesPutMapping.should.not.be.called 436 | stubESIndex.should.be.called 437 | const { body } = stubESIndex.getCalls()[0].args[0] 438 | body.request.should.not.have.property("query") 439 | body.request.should.have.property("my_name", "green") 440 | }) 441 | }) 442 | 443 | describe("edit mapping if index is exists", () => { 444 | beforeEach(() => { 445 | stubESIndex.resetHistory() 446 | stubESIndicesExists.resetHistory() 447 | stubESIndicesPutMapping.resetHistory() 448 | stubESIndicesCreate.resetHistory() 449 | }) 450 | afterEach(() => { 451 | clock.setSystemTime(date) 452 | }) 453 | it("it should call put mapping instead of create when index is exists", async () => { 454 | const stub = sinon.stub().callsArgWith(1, null, { body: true }) // make index exists 455 | const client = getClient({ 456 | exists: stub, 457 | }) 458 | await callServer(config, client) 459 | stubESIndicesCreate.should.not.be.called 460 | stubESIndicesPutMapping.should.be.called 461 | const putMappingConfig = stubESIndicesPutMapping.getCalls()[0].args[0] 462 | putMappingConfig.should.have.property("index", "log_2020-h2") 463 | putMappingConfig.should.have 464 | .property("body") 465 | .that.deep.equal(defaultMapping) 466 | }) 467 | }) 468 | }) 469 | --------------------------------------------------------------------------------