├── eslint.config.js ├── docker-compose.yml ├── test ├── fixture │ └── wrap-transport.js ├── log.js └── end-to-end │ ├── pipe-usage.js │ └── transport.js ├── .github ├── dependabot.yml └── workflows │ └── ci.yml ├── lib ├── log.js ├── makeInsert.js └── pino-transport.js ├── trial ├── run.sh └── assert.js ├── HELP.md ├── LICENSE ├── package.json ├── pino-mongodb.js ├── .gitignore └── README.md /eslint.config.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | module.exports = require('neostandard')({}) 4 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | services: 3 | mongo-serve: 4 | container_name: mongo-serve 5 | image: mongo:5 6 | restart: always 7 | ports: 8 | - 27017:27017 9 | environment: 10 | MONGO_INITDB_ROOT_USERNAME: one 11 | MONGO_INITDB_ROOT_PASSWORD: two 12 | -------------------------------------------------------------------------------- /test/fixture/wrap-transport.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const transport = require('../../lib/pino-transport') 4 | 5 | module.exports = async function (opts) { 6 | opts.parseLine = function (str) { 7 | const obj = JSON.parse(str) 8 | return obj 9 | } 10 | return transport(opts) 11 | } 12 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "monthly" 7 | open-pull-requests-limit: 10 8 | 9 | - package-ecosystem: "npm" 10 | directory: "/" 11 | schedule: 12 | interval: "weekly" 13 | open-pull-requests-limit: 10 14 | -------------------------------------------------------------------------------- /lib/log.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | module.exports = function log (data) { 4 | let log 5 | 6 | try { 7 | log = typeof data === 'string' ? JSON.parse(data) : data 8 | 9 | if (log.time) { 10 | log.time = new Date(log.time) 11 | } 12 | if (log.timestamp) { 13 | log.timestamp = new Date(log.timestamp) 14 | } 15 | } catch (e) { 16 | log = { 17 | msg: data 18 | } 19 | } 20 | 21 | return log 22 | } 23 | -------------------------------------------------------------------------------- /trial/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo Up... 4 | docker-compose up -d 5 | 6 | sleep 3 7 | 8 | url="mongodb://one:two@localhost:27017" 9 | container="mongo-serve" 10 | 11 | # trial 12 | echo '{"txt": "should work with valid json"}' | ./pino-mongodb.js -u $url & 13 | echo '{"time": "1990-09-16T09:00:00.009Z"}' | ./pino-mongodb.js -u $url & 14 | echo 'should work with invalid json' | ./pino-mongodb.js -u $url & 15 | 16 | sleep 3 17 | 18 | node trial/assert.js $url && echo OK || echo Fail 19 | 20 | echo Down... 21 | docker-compose down 22 | -------------------------------------------------------------------------------- /HELP.md: -------------------------------------------------------------------------------- 1 | Usage: pino-mongodb [options] [mongo-url] 2 | 3 | Insert JSON from stdin into MongoDB 4 | 5 | Options: 6 | -V, --version output the version number 7 | -c, --collection database collection (default: "logs") 8 | -o, --stdout output inserted documents into stdout (default: 9 | false) 10 | -e, --errors output insertion errors into stderr (default: false) 11 | -u, --unified use mongodb unified topology (default: false) 12 | -h, --help display help for command 13 | -------------------------------------------------------------------------------- /trial/assert.js: -------------------------------------------------------------------------------- 1 | const MongoClient = require('mongodb').MongoClient 2 | const t = require('assert').strict 3 | 4 | async function main () { 5 | const url = process.argv.slice(2, 3)[0] 6 | const conn = await MongoClient.connect(url, { useUnifiedTopology: true }) 7 | const db = conn.db('admin') 8 | const logs = db.collection('logs') 9 | 10 | const data = await logs.find().toArray() 11 | 12 | const valid = data.filter(x => x.txt) 13 | console.log('valid', valid) 14 | t.ok(valid.length, 'should have found valid insertions') 15 | 16 | const notValid = data.filter(x => x.msg) 17 | console.log('notValid', notValid) 18 | t.ok(notValid.length, 'should have found invalid insertions') 19 | 20 | const time = data.filter(x => x.time) 21 | console.log('time', time) 22 | t.ok(time[0].time, 'should have found time-based insertions') 23 | 24 | await conn.close() 25 | } 26 | 27 | main() 28 | .catch((e) => { 29 | console.error(e) 30 | process.exit(1) 31 | }) 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Viktor Kuroljov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /lib/makeInsert.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { nextTick } = require('node:process') 4 | const { EOL } = require('node:os') 5 | const options = { 6 | forceServerObjectId: true 7 | } 8 | 9 | module.exports = function makeInsert (showErrors, showStdout) { 10 | let callback 11 | 12 | if (showErrors && showStdout) { 13 | callback = function (e, log) { 14 | if (e) { 15 | console.error(e) 16 | } else { 17 | process.stdout.write(JSON.stringify(log) + EOL) 18 | } 19 | } 20 | } else if (showErrors && !showStdout) { 21 | callback = function (e) { 22 | if (e) { 23 | console.error(e) 24 | } 25 | } 26 | } else if (!showErrors && showStdout) { 27 | callback = function (e, log) { 28 | if (!e) { 29 | process.stdout.write(JSON.stringify(log) + EOL) 30 | } 31 | } 32 | } 33 | 34 | return function insert (collection, log, cliCallback) { 35 | collection.insertOne(log, options) 36 | .then(() => { 37 | callback && nextTick(() => callback(null, log)) 38 | cliCallback && nextTick(() => cliCallback()) 39 | }) 40 | .catch(error => { 41 | callback && callback(error, log) 42 | cliCallback && cliCallback(error) 43 | }) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /test/log.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const test = require('node:test') 4 | const assert = require('node:assert') 5 | const log = require('../lib/log') 6 | 7 | test('valid input', t => { 8 | const expected = { a: 1 } 9 | const actual = log(JSON.stringify(expected)) 10 | 11 | assert.deepEqual(actual, expected) 12 | }) 13 | 14 | test('valid input with time as Date', t => { 15 | const now = Date.now() 16 | const expected = { 17 | a: 1, 18 | time: new Date(now) 19 | } 20 | const actual = log(JSON.stringify({ 21 | a: expected.a, 22 | time: now 23 | })) 24 | 25 | assert.deepEqual(actual, expected) 26 | }) 27 | test('valid input with timestamp as Date', t => { 28 | const now = Date.now() 29 | const expected = { 30 | a: 1, 31 | timestamp: new Date(now) 32 | } 33 | const actual = log(JSON.stringify({ 34 | a: expected.a, 35 | timestamp: now 36 | })) 37 | 38 | assert.deepEqual(actual, expected) 39 | }) 40 | 41 | test('invalid input', t => { 42 | const expected = { msg: 'message' } 43 | const actual = log('message') 44 | 45 | assert.deepEqual(actual, expected) 46 | }) 47 | 48 | test('do not mutate object', t => { 49 | const expected = { msg: 'message' } 50 | const actual = log(expected) 51 | 52 | assert.deepEqual(actual, expected) 53 | }) 54 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pino-mongodb", 3 | "version": "5.0.0", 4 | "description": "Insert JSON from stdin into MongoDB", 5 | "files": [ 6 | "lib" 7 | ], 8 | "scripts": { 9 | "test": "borp --coverage test/log.js", 10 | "test:end2end": "npm test && borp 'test/end-to-end/*.js'", 11 | "lint": "eslint .", 12 | "lint:fix": "npm run lint -- --fix", 13 | "help": "./pino-mongodb.js --help > HELP.md", 14 | "trial": "./trial/run.sh" 15 | }, 16 | "main": "pino-mongodb.js", 17 | "bin": "pino-mongodb.js", 18 | "repository": { 19 | "type": "git", 20 | "url": "https://github.com/pinojs/pino-mongodb.git" 21 | }, 22 | "keywords": [ 23 | "pino", 24 | "mongo", 25 | "mongodb", 26 | "pino-mongodb", 27 | "logs", 28 | "logger" 29 | ], 30 | "author": "Viktor Kuroljov", 31 | "license": "MIT", 32 | "bugs": { 33 | "url": "https://github.com/pinojs/pino-mongodb/issues" 34 | }, 35 | "homepage": "https://github.com/pinojs/pino-mongodb", 36 | "dependencies": { 37 | "carrier": "^0.3", 38 | "commander": "^14.0.2", 39 | "mongodb": "^6.20.0", 40 | "muri": "^1.3", 41 | "pino-abstract-transport": "^3.0.0" 42 | }, 43 | "devDependencies": { 44 | "borp": "^0.21.0", 45 | "eslint": "^9.39.1", 46 | "neostandard": "^0.12.2", 47 | "pino": "^10.1.0" 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /lib/pino-transport.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { nextTick } = require('node:process') 4 | const { Writable } = require('node:stream') 5 | const { MongoClient } = require('mongodb') 6 | const build = require('pino-abstract-transport') 7 | 8 | const log = require('./log') 9 | 10 | const defaultOption = { 11 | uri: 'mongodb://localhost:27017/logs', 12 | collection: 'logs', 13 | unified: false, 14 | mongoOptions: { 15 | appName: 'pino-mongodb' 16 | }, 17 | parseLine: null 18 | } 19 | 20 | async function mongodbTransport (opts) { 21 | const { 22 | uri, 23 | database: databaseName, 24 | collection: collectionName, 25 | mongoOptions, 26 | parseLine: customParseLine 27 | } = Object.assign({}, defaultOption, opts) 28 | 29 | /** 30 | * prevent passing custom parseLine that is not a function 31 | * we fallback to `null` but not `log` because `log` function provide more error handling 32 | * and it will be used before insert to provide more protection on input error 33 | */ 34 | const parseLine = typeof customParseLine === 'function' ? customParseLine : null 35 | 36 | const client = new MongoClient(uri, mongoOptions) 37 | await client.connect() 38 | 39 | const db = client.db(databaseName) 40 | const collection = db.collection(collectionName) 41 | 42 | const mongoStream = new Writable({ 43 | objectMode: true, 44 | autoDestroy: true, 45 | write (chunk, enc, cb) { 46 | // todo: bulk insert? 47 | collection.insertOne(log(chunk), { 48 | forceServerObjectId: true 49 | }).then(() => { nextTick(() => cb()) }) 50 | }, 51 | destroy (err, cb) { 52 | client.close() 53 | .then(() => { 54 | nextTick(() => cb()) 55 | }) 56 | .catch((closeErr) => { 57 | cb(err || closeErr) 58 | }) 59 | } 60 | }) 61 | 62 | return build(function (source) { 63 | source.pipe(mongoStream) 64 | }, { 65 | parseLine, 66 | close (err, cb) { 67 | mongoStream.end() 68 | mongoStream.once('close', cb.bind(null, err)) 69 | } 70 | }) 71 | } 72 | 73 | module.exports = mongodbTransport 74 | module.exports.defaultOption = defaultOption 75 | -------------------------------------------------------------------------------- /pino-mongodb.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 'use strict' 3 | 4 | const { nextTick } = require('node:process') 5 | const carrier = require('carrier') 6 | const { program } = require('commander') 7 | const { MongoClient } = require('mongodb') 8 | const parseMongoUrl = require('muri') 9 | const log = require('./lib/log') 10 | const pkg = require('./package.json') 11 | const makeInsert = require('./lib/makeInsert') 12 | const transport = require('./lib/pino-transport') 13 | 14 | module.exports = transport 15 | 16 | if (require.main === module) { 17 | // used as cli 18 | cli() 19 | } 20 | 21 | function cli () { 22 | program 23 | .version(pkg.version) 24 | .description(pkg.description) 25 | .arguments('[mongo-url]') 26 | .option('-c, --collection ', 'database collection', transport.defaultOption.collection) 27 | .option('-o, --stdout', 'output inserted documents into stdout', false) 28 | .option('-e, --errors', 'output insertion errors into stderr', false) 29 | .parse(process.argv) 30 | 31 | const cliOptions = program.opts() 32 | const mongoUrl = (program.args[0] || transport.defaultOption.uri) 33 | 34 | function handleConnection (e, mClient) { 35 | if (e) { 36 | throw e 37 | } 38 | 39 | const dbName = parseMongoUrl(mongoUrl).db 40 | 41 | const db = mClient.db(dbName) 42 | const emitter = carrier.carry(process.stdin) 43 | const collection = db.collection(cliOptions.collection) 44 | const insert = makeInsert(cliOptions.errors, cliOptions.stdout) 45 | 46 | let insertCounter = 0 47 | const insertCallback = function () { 48 | insertCounter-- 49 | if (process.stdin.destroyed && insertCounter === 0) { 50 | mClient.close(process.exit) 51 | } 52 | } 53 | 54 | emitter.on('line', (line) => { 55 | insertCounter++ 56 | insert(collection, log(line), insertCallback) 57 | }) 58 | 59 | process.stdin.on('close', () => { 60 | if (insertCounter === 0) { 61 | mClient.close(process.exit) 62 | } 63 | }) 64 | 65 | process.once('SIGINT', () => { 66 | mClient.close(process.exit) 67 | }) 68 | } 69 | 70 | const options = {} 71 | 72 | MongoClient.connect(mongoUrl, options) 73 | .then((client) => { 74 | nextTick(() => handleConnection(null, client)) 75 | }) 76 | .catch((error) => { 77 | handleConnection(error) 78 | }) 79 | } 80 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | .pnpm-debug.log* 9 | 10 | # Diagnostic reports (https://nodejs.org/api/report.html) 11 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 12 | 13 | # Runtime data 14 | pids 15 | *.pid 16 | *.seed 17 | *.pid.lock 18 | 19 | # Directory for instrumented libs generated by jscoverage/JSCover 20 | lib-cov 21 | 22 | # Coverage directory used by tools like istanbul 23 | coverage 24 | *.lcov 25 | 26 | # nyc test coverage 27 | .nyc_output 28 | 29 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 30 | .grunt 31 | 32 | # Bower dependency directory (https://bower.io/) 33 | bower_components 34 | 35 | # node-waf configuration 36 | .lock-wscript 37 | 38 | # Compiled binary addons (https://nodejs.org/api/addons.html) 39 | build/Release 40 | 41 | # Dependency directories 42 | node_modules/ 43 | jspm_packages/ 44 | 45 | # Snowpack dependency directory (https://snowpack.dev/) 46 | web_modules/ 47 | 48 | # TypeScript cache 49 | *.tsbuildinfo 50 | 51 | # Optional npm cache directory 52 | .npm 53 | 54 | # Optional eslint cache 55 | .eslintcache 56 | 57 | # Optional stylelint cache 58 | .stylelintcache 59 | 60 | # Microbundle cache 61 | .rpt2_cache/ 62 | .rts2_cache_cjs/ 63 | .rts2_cache_es/ 64 | .rts2_cache_umd/ 65 | 66 | # Optional REPL history 67 | .node_repl_history 68 | 69 | # Output of 'npm pack' 70 | *.tgz 71 | 72 | # Yarn Integrity file 73 | .yarn-integrity 74 | 75 | # dotenv environment variable files 76 | .env 77 | .env.development.local 78 | .env.test.local 79 | .env.production.local 80 | .env.local 81 | 82 | # parcel-bundler cache (https://parceljs.org/) 83 | .cache 84 | .parcel-cache 85 | 86 | # Next.js build output 87 | .next 88 | out 89 | 90 | # Nuxt.js build / generate output 91 | .nuxt 92 | dist 93 | 94 | # Gatsby files 95 | .cache/ 96 | # Comment in the public line in if your project uses Gatsby and not Next.js 97 | # https://nextjs.org/blog/next-9-1#public-directory-support 98 | # public 99 | 100 | # vuepress build output 101 | .vuepress/dist 102 | 103 | # vuepress v2.x temp and cache directory 104 | .temp 105 | .cache 106 | 107 | # Docusaurus cache and generated files 108 | .docusaurus 109 | 110 | # Serverless directories 111 | .serverless/ 112 | 113 | # FuseBox cache 114 | .fusebox/ 115 | 116 | # DynamoDB Local files 117 | .dynamodb/ 118 | 119 | # TernJS port file 120 | .tern-port 121 | 122 | # Stores VSCode versions used for testing VSCode extensions 123 | .vscode-test 124 | 125 | # yarn v2 126 | .yarn/cache 127 | .yarn/unplugged 128 | .yarn/build-state.yml 129 | .yarn/install-state.gz 130 | .pnp.* 131 | 132 | # Vim swap files 133 | *.swp 134 | 135 | # macOS files 136 | .DS_Store 137 | 138 | # editor files 139 | .vscode 140 | .idea 141 | 142 | # lock files 143 | package-lock.json 144 | pnpm-lock.yaml 145 | yarn.lock 146 | 147 | # 0x 148 | .__browserify* 149 | profile-* -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | paths-ignore: 6 | - 'docs/**' 7 | - '*.md' 8 | pull_request: 9 | paths-ignore: 10 | - 'docs/**' 11 | - '*.md' 12 | 13 | # This allows a subsequently queued workflow run to interrupt previous runs 14 | concurrency: 15 | group: "${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}" 16 | cancel-in-progress: true 17 | 18 | jobs: 19 | dependency-review: 20 | name: Dependency Review 21 | if: github.event_name == 'pull_request' 22 | runs-on: ubuntu-latest 23 | permissions: 24 | contents: read 25 | steps: 26 | - name: Check out repo 27 | uses: actions/checkout@v3 28 | with: 29 | persist-credentials: false 30 | 31 | - name: Dependency review 32 | uses: actions/dependency-review-action@v4 33 | 34 | test: 35 | name: Test 36 | runs-on: ${{ matrix.os }} 37 | permissions: 38 | contents: read 39 | strategy: 40 | matrix: 41 | node-version: [20, 22, 24] 42 | os: [macos-latest, windows-latest] 43 | steps: 44 | - name: Check out repo 45 | uses: actions/checkout@v3 46 | with: 47 | persist-credentials: false 48 | 49 | - name: Setup Node ${{ matrix.node-version }} 50 | uses: actions/setup-node@v3 51 | with: 52 | node-version: ${{ matrix.node-version }} 53 | 54 | - name: Install dependencies 55 | run: npm i --ignore-scripts 56 | 57 | - name: Run tests 58 | run: npm test 59 | 60 | test-end2end: 61 | runs-on: ${{ matrix.os }} 62 | 63 | strategy: 64 | matrix: 65 | node-version: [20, 22, 24] 66 | os: [ubuntu-latest] 67 | 68 | services: 69 | mongodb: 70 | image: mongo:5 71 | ports: 72 | - 27017:27017 73 | env: 74 | MONGO_INITDB_ROOT_USERNAME: one 75 | MONGO_INITDB_ROOT_PASSWORD: two 76 | 77 | steps: 78 | - name: Check out repo 79 | uses: actions/checkout@v3 80 | with: 81 | persist-credentials: false 82 | 83 | - name: Setup Node ${{ matrix.node-version }} 84 | uses: actions/setup-node@v3 85 | with: 86 | node-version: ${{ matrix.node-version }} 87 | 88 | - name: Install dependencies 89 | run: npm i --ignore-scripts 90 | 91 | - name: Run tests 92 | run: npm run test:end2end 93 | 94 | automerge: 95 | name: Automerge Dependabot PRs 96 | if: > 97 | github.event_name == 'pull_request' && 98 | github.event.pull_request.user.login == 'dependabot[bot]' 99 | needs: [test, test-end2end] 100 | permissions: 101 | pull-requests: write 102 | contents: write 103 | runs-on: ubuntu-latest 104 | steps: 105 | - uses: fastify/github-action-merge-dependabot@v3 106 | with: 107 | github-token: ${{ secrets.GITHUB_TOKEN }} 108 | -------------------------------------------------------------------------------- /test/end-to-end/pipe-usage.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const test = require('node:test') 4 | const assert = require('node:assert') 5 | const { spawn } = require('node:child_process') 6 | const { once } = require('node:events') 7 | const { MongoClient } = require('mongodb') 8 | 9 | const mongoUrl = 'mongodb://one:two@localhost:27017/newdb?authSource=admin' 10 | 11 | test('must log to a custom collection', async (t) => { 12 | const customCollection = 'custom-collection' 13 | const childProcess = spawn('node', [ 14 | '../../pino-mongodb.js', 15 | mongoUrl, 16 | '-c', 17 | customCollection 18 | ], { 19 | cwd: __dirname, 20 | stdio: ['pipe', null, null] 21 | }) 22 | 23 | const client = new MongoClient(mongoUrl) 24 | await client.connect() 25 | t.after(client.close.bind(client)) 26 | const db = client.db() 27 | const collection = db.collection(customCollection) 28 | 29 | const rowsBefore = await collection.countDocuments() 30 | assert.ok(`rows count ${rowsBefore}`) 31 | 32 | childProcess.stdin.write('hello pino-mongo 1\n') 33 | childProcess.stdin.write(`${JSON.stringify({ hello: 'pino' })}\n`) 34 | childProcess.stdin.write('hello pino-mongo 2\n') 35 | childProcess.stdin.end() 36 | 37 | try { 38 | await once(childProcess, 'close') 39 | const rowsAfter = await collection.countDocuments() 40 | assert.equal(rowsAfter, rowsBefore + 3, 'logged 3 rows') 41 | } catch (error) { 42 | assert.fail(error.message) 43 | } 44 | }) 45 | 46 | test('must exit when the stdin is destroyed', async (t) => { 47 | const customCollection = 'custom-collection' 48 | const childProcess = spawn('node', [ 49 | '../../pino-mongodb.js', 50 | mongoUrl, 51 | '-c', 52 | customCollection 53 | ], { 54 | cwd: __dirname, 55 | stdio: ['pipe', null, null] 56 | }) 57 | 58 | childProcess.stdin.end() 59 | 60 | try { 61 | await once(childProcess, 'close') 62 | assert.ok('pino-mongo exits') 63 | } catch (error) { 64 | assert.fail(error.message) 65 | } 66 | }) 67 | 68 | test('must write logs to the console with -o option', async (t) => { 69 | const customCollection = 'custom-collection' 70 | const childProcess = spawn('node', [ 71 | '../../pino-mongodb.js', 72 | mongoUrl, 73 | '-o', 74 | '-c', 75 | customCollection 76 | ], { 77 | cwd: __dirname, 78 | stdio: ['pipe', 'pipe', process.stderr] 79 | }) 80 | 81 | const client = new MongoClient(mongoUrl) 82 | await client.connect() 83 | t.after(client.close.bind(client)) 84 | const db = client.db() 85 | const collection = db.collection(customCollection) 86 | 87 | const rowsBefore = await collection.countDocuments() 88 | assert.ok(`rows count ${rowsBefore}`) 89 | 90 | childProcess.stdin.write('hello pino-mongo 1\n') 91 | childProcess.stdin.write(`${JSON.stringify({ hello: 'pino' })}\n`) 92 | childProcess.stdin.write('hello pino-mongo 2\n') 93 | childProcess.stdin.end() 94 | 95 | // read stdout 96 | const chunks = [] 97 | for await (const chunk of childProcess.stdout) { 98 | chunks.push(chunk) 99 | } 100 | const output = Buffer.concat(chunks).toString() 101 | assert.equal(output.trim().split('\n').length, 3) 102 | 103 | try { 104 | await once(childProcess, 'close') 105 | const rowsAfter = await collection.countDocuments() 106 | assert.equal(rowsAfter, rowsBefore + 3, 'logged 3 rows') 107 | } catch (error) { 108 | assert.fail(error.message) 109 | } 110 | }) 111 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pino-mongodb 2 | [![npm version](https://img.shields.io/npm/v/pino-mongodb)](https://www.npmjs.com/package/pino-mongodb) 3 | [![Build Status](https://img.shields.io/github/workflow/status/pinojs/pino-mongodb/CI)](https://github.com/pinojs/pino-mongodb/actions) 4 | 5 | > Insert JSON from stdin into MongoDB 6 | 7 | This project is part of the `pino` logger family, however you can use it to parse and insert any 8 | `JSON` into the `mongo`. 9 | 10 | ## Install 11 | 12 | ```bash 13 | $ npm i pino-mongodb 14 | ``` 15 | 16 | ## Usage as Pino Transport 17 | 18 | You can use this module as a [pino transport](https://getpino.io/#/docs/transports?id=v7-transports) like so: 19 | 20 | ```js 21 | const pino = require('pino') 22 | const transport = pino.transport({ 23 | target: 'pino-mongodb', 24 | level: 'info', 25 | options: { 26 | uri: 'mongodb://localhost:27017/', 27 | database: 'logs', 28 | collection: 'log-collection', 29 | mongoOptions: { 30 | auth: { 31 | username: 'one', 32 | password: 'two' 33 | } 34 | } 35 | } 36 | }) 37 | 38 | pino(transport) 39 | ``` 40 | 41 | The `mongoOptions` is provided to the the standard mongodb client. All the available options are described on [its official documentation](https://mongodb.github.io/node-mongodb-native/4.1/interfaces/MongoClientOptions.html). 42 | 43 | Note that you may encouter missing logs in special cases: it dependes on data and mongo's version. Please checkout the [mongodb limitation](https://docs.mongodb.com/manual/reference/limits/) official documentation. 44 | For example on MongoDB 4: 45 | 46 | ```js 47 | // IT DOES NOT WORK: 48 | log.info({ $and: [{ a: 1 }, { b: 2 }] }, 'my query is') 49 | 50 | // IT WORKS: 51 | log.info({ query: { $and: [{ a: 1 }, { b: 2 }]} }, 'my query is') 52 | ``` 53 | 54 | If you want a custom parser to handle the above case. You need to wrap `pino-mongo` and pass a function through `option.parseLine`. Any value that is not a function will be ignored in this option. 55 | 56 | ```js 57 | // mongo-transport.js 58 | 'use strict' 59 | 60 | const transport = require('pino-mongodb') 61 | 62 | module.exports = function(opts) { 63 | opts.parseLine = function(str) { // `str` is passed from `pino` and expected to be a string 64 | const obj = JSON.parse(str) 65 | 66 | // do anything you want... 67 | 68 | return obj // return value is expected to be a json that will pass and save inside mongodb 69 | } 70 | return transport(opts) 71 | } 72 | 73 | // main.js 74 | const pino = require('pino') 75 | const transport = pino.transport({ 76 | target: './mongo-transport.js', 77 | uri: 'mongodb://localhost:27017/logs', 78 | collection: 'log-collection', 79 | }) 80 | pino(transport) 81 | ``` 82 | 83 | ## Usage as Pino Legacy Transport 84 | 85 | Pino supports a [legacy transport interface](https://getpino.io/#/docs/transports?id=legacy-transports) 86 | that is still supported by this module. 87 | 88 | ### Get started 89 | 90 | ```bash 91 | $ echo '{"name": "Viktor"}' | pino-mongodb [options] [mongo-url] 92 | ``` 93 | 94 | ```bash 95 | $ cat many.logs | pino-mongodb [options] [mongo-url] 96 | ``` 97 | 98 | ```bash 99 | $ node ./app.js | pino-mongodb [options] [mongo-url] 100 | ``` 101 | 102 | ### CLI Options 103 | 104 | ``` 105 | Usage: pino-mongodb [options] [mongo-url] 106 | 107 | Insert JSON from stdin into MongoDB 108 | 109 | Options: 110 | -V, --version output the version number 111 | -c, --collection database collection (default: "logs") 112 | -o, --stdout output inserted documents into stdout (default: 113 | false) 114 | -e, --errors output insertion errors into stderr (default: false) 115 | -h, --help display help for command 116 | ``` 117 | 118 | ## Tests 119 | 120 | To run unit tests: 121 | 122 | ```bash 123 | $ npm t 124 | ``` 125 | 126 | To run integrational tests with real mongo server: 127 | 128 | ```bash 129 | $ npm run trial 130 | ``` 131 | 132 | Note, you will have to have `docker` and `docker-compose` installed 133 | on your machine for that! 134 | 135 | ## License 136 | 137 | Licensed under [MIT](./LICENSE). 138 | -------------------------------------------------------------------------------- /test/end-to-end/transport.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const test = require('node:test') 4 | const assert = require('node:assert') 5 | const { once } = require('node:events') 6 | const { promisify } = require('node:util') 7 | const pino = require('pino') 8 | const { MongoClient } = require('mongodb') 9 | 10 | const setTimeout = promisify(global.setTimeout) 11 | 12 | test('auth transport test', async (t) => { 13 | const options = { 14 | uri: 'mongodb://localhost:27017/', 15 | database: 'logs', 16 | collection: 'log-test', 17 | mongoOptions: { 18 | authMechanism: 'SCRAM-SHA-1', 19 | auth: { 20 | username: 'one', 21 | password: 'two' 22 | } 23 | } 24 | } 25 | 26 | const client = new MongoClient(options.uri, options.mongoOptions) 27 | await client.connect() 28 | t.after(client.close.bind(client)) 29 | const db = client.db(options.database) 30 | const collection = db.collection(options.collection) 31 | 32 | const rowsBefore = await collection.countDocuments() 33 | 34 | const transport = pino.transport({ 35 | target: '../../pino-mongodb.js', 36 | level: 'info', 37 | options 38 | }) 39 | const log = pino(transport) 40 | await once(transport, 'ready') 41 | log.info('this is a string log') 42 | log.debug('ignored') 43 | log.info('this is a long string log'.repeat(1000)) 44 | log.fatal(new Error('ops'), 'not ignored') 45 | assert.ok('logged on mongo') 46 | 47 | await setTimeout(1000) 48 | const rowsAfter = await collection.countDocuments() 49 | assert.equal(rowsAfter, rowsBefore + 3, 'logged 3 rows') 50 | }) 51 | 52 | test('auth transport test', async (t) => { 53 | const options = { 54 | uri: 'mongodb://one:two@localhost:27017/dbname?authSource=admin', 55 | collection: 'log-test' 56 | } 57 | 58 | const client = new MongoClient(options.uri) 59 | await client.connect() 60 | t.after(client.close.bind(client)) 61 | const db = client.db() 62 | const collection = db.collection(options.collection) 63 | 64 | const rowsBefore = await collection.countDocuments() 65 | 66 | const transport = pino.transport({ 67 | target: '../../pino-mongodb.js', 68 | level: 'info', 69 | options 70 | }) 71 | const log = pino(transport) 72 | await once(transport, 'ready') 73 | log.info('this is a string log') 74 | log.debug('ignored') 75 | log.info('this is a long string log'.repeat(1000)) 76 | log.fatal(new Error('ops'), 'not ignored') 77 | assert.ok('logged on mongo') 78 | 79 | await setTimeout(1000) 80 | const rowsAfter = await collection.countDocuments() 81 | assert.equal(rowsAfter, rowsBefore + 3, 'logged 3 rows') 82 | }) 83 | 84 | test('log blocked items', async (t) => { 85 | const options = { 86 | uri: 'mongodb://one:two@localhost:27017/dbname?authSource=admin', 87 | collection: 'log-block' 88 | } 89 | 90 | const client = new MongoClient(options.uri) 91 | await client.connect() 92 | t.after(client.close.bind(client)) 93 | const db = client.db() 94 | const collection = db.collection(options.collection) 95 | 96 | const rowsBefore = await collection.countDocuments() 97 | 98 | const transport = pino.transport({ 99 | target: '../../pino-mongodb.js', 100 | level: 'info', 101 | options 102 | }) 103 | const log = pino(transport) 104 | 105 | await once(transport, 'ready') 106 | log.info({ query: { $and: [{ a: 1 }, { b: 2 }] } }, 'my query was') 107 | log.info({ 'foo.bar': 42 }, 'dot object') 108 | assert.ok('logged on mongo') 109 | 110 | await setTimeout(1000) 111 | 112 | const rowsAfter = await collection.countDocuments() 113 | assert.equal(rowsAfter, rowsBefore + 2, 'log not inserted due the mongo limitation') 114 | 115 | log.info('the stream is open') 116 | await setTimeout(1000) 117 | const rowsInserted = await collection.countDocuments() 118 | assert.equal(rowsInserted, rowsAfter + 1, 'logs are still working') 119 | }) 120 | 121 | test('custom parse line function', async (t) => { 122 | const options = { 123 | uri: 'mongodb://localhost:27017/', 124 | database: 'logs', 125 | collection: 'log-test', 126 | mongoOptions: { 127 | authMechanism: 'SCRAM-SHA-1', 128 | auth: { 129 | username: 'one', 130 | password: 'two' 131 | } 132 | } 133 | } 134 | 135 | const client = new MongoClient(options.uri, options.mongoOptions) 136 | await client.connect() 137 | t.after(client.close.bind(client)) 138 | const db = client.db(options.database) 139 | const collection = db.collection(options.collection) 140 | 141 | const rowsBefore = await collection.countDocuments() 142 | 143 | const transport = pino.transport({ 144 | target: '../fixture/wrap-transport.js', 145 | level: 'info', 146 | options 147 | }) 148 | const log = pino(transport) 149 | await once(transport, 'ready') 150 | log.info('this is a string log') 151 | log.debug('ignored') 152 | log.info('this is a long string log'.repeat(1000)) 153 | log.fatal(new Error('ops'), 'not ignored') 154 | assert.ok('logged on mongo') 155 | 156 | await setTimeout(1000) 157 | const rowsAfter = await collection.countDocuments() 158 | assert.equal(rowsAfter, rowsBefore + 3, 'logged 3 rows') 159 | }) 160 | 161 | test('invalid custom parse line function', async (t) => { 162 | const options = { 163 | uri: 'mongodb://localhost:27017/', 164 | database: 'logs', 165 | collection: 'log-test', 166 | mongoOptions: { 167 | authMechanism: 'SCRAM-SHA-1', 168 | auth: { 169 | username: 'one', 170 | password: 'two' 171 | } 172 | }, 173 | parseLine: false 174 | } 175 | 176 | const client = new MongoClient(options.uri, options.mongoOptions) 177 | await client.connect() 178 | t.after(client.close.bind(client)) 179 | const db = client.db(options.database) 180 | const collection = db.collection(options.collection) 181 | 182 | const rowsBefore = await collection.countDocuments() 183 | 184 | const transport = pino.transport({ 185 | target: '../../pino-mongodb.js', 186 | level: 'info', 187 | options 188 | }) 189 | const log = pino(transport) 190 | await once(transport, 'ready') 191 | log.info('this is a string log') 192 | log.debug('ignored') 193 | log.info('this is a long string log'.repeat(1000)) 194 | log.fatal(new Error('ops'), 'not ignored') 195 | assert.ok('logged on mongo') 196 | 197 | await setTimeout(1000) 198 | const rowsAfter = await collection.countDocuments() 199 | assert.equal(rowsAfter, rowsBefore + 3, 'logged 3 rows') 200 | }) 201 | --------------------------------------------------------------------------------