├── .env.example ├── .github └── workflows │ ├── ci.yml │ └── release.yml ├── .gitignore ├── LICENSE ├── README.md ├── bin └── test.ts ├── compose.yaml ├── eslint.config.js ├── examples ├── adonisjs │ ├── ace.js │ ├── adonisrc.ts │ ├── app │ │ ├── exceptions │ │ │ └── handler.ts │ │ └── middleware │ │ │ └── container_bindings_middleware.ts │ ├── bin │ │ ├── console.ts │ │ └── server.ts │ ├── config │ │ ├── app.ts │ │ ├── bodyparser.ts │ │ ├── hash.ts │ │ └── logger.ts │ ├── package.json │ ├── pnpm-lock.yaml │ ├── start │ │ ├── env.ts │ │ ├── kernel.ts │ │ └── routes.ts │ └── tsconfig.json ├── basic.ts ├── batching.ts ├── cli.ts ├── custom_timestamp.ts ├── module_usage.ts └── multiple_transports.ts ├── loki_config.yaml ├── package.json ├── pnpm-lock.yaml ├── src ├── cli │ ├── args.ts │ ├── index.ts │ └── print_help.ts ├── constants.ts ├── debug.ts ├── format_mesage.ts ├── get.ts ├── index.ts ├── log_builder.ts ├── log_pusher.ts └── types.ts ├── tests ├── fixtures │ └── custom_pino_loki.js ├── helpers.ts ├── integration │ └── loki.spec.ts └── unit │ ├── cli.spec.ts │ ├── format_message.spec.ts │ ├── log_builder.spec.ts │ └── log_pusher.spec.ts ├── tsconfig.json └── tsdown.config.ts /.env.example: -------------------------------------------------------------------------------- 1 | LOKI_HOST=http://localhost:3100 2 | LOKI_USERNAME= 3 | LOKI_PASSWORD= 4 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | - push 5 | - pull_request 6 | - workflow_call 7 | 8 | jobs: 9 | lint: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - name: Checkout 14 | uses: actions/checkout@v4 15 | 16 | - name: Install pnpm 17 | uses: pnpm/action-setup@v4 18 | 19 | - name: Setup node 20 | uses: actions/setup-node@v4 21 | with: 22 | node-version: lts/* 23 | cache: pnpm 24 | 25 | - name: Install dependencies 26 | run: pnpm install 27 | 28 | - name: Lint 29 | run: pnpm run lint 30 | 31 | - name: Typecheck 32 | run: pnpm run typecheck 33 | 34 | test: 35 | runs-on: ubuntu-latest 36 | strategy: 37 | matrix: 38 | node: [22.x, 24.x] 39 | env: 40 | LOKI_HOST: ${{ secrets.LOKI_HOST }} 41 | LOKI_USERNAME: ${{ secrets.LOKI_USERNAME }} 42 | LOKI_PASSWORD: ${{ secrets.LOKI_PASSWORD }} 43 | 44 | steps: 45 | - uses: actions/checkout@v4 46 | 47 | - name: Install pnpm 48 | uses: pnpm/action-setup@v4 49 | 50 | - name: Set node version to ${{ matrix.node }} 51 | uses: actions/setup-node@v4 52 | with: 53 | node-version: ${{ matrix.node }} 54 | cache: pnpm 55 | 56 | - name: Install 57 | run: pnpm install 58 | 59 | - name: Build 60 | run: pnpm build 61 | 62 | - name: Test 63 | run: pnpm test 64 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | release: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v4 13 | with: 14 | fetch-depth: 0 15 | 16 | - name: Install pnpm 17 | run: | 18 | corepack enable 19 | corepack prepare pnpm@latest --activate 20 | 21 | - name: Set node 22 | uses: actions/setup-node@v4 23 | with: 24 | node-version: 22.x 25 | cache: pnpm 26 | 27 | - run: npx changelogithub 28 | env: 29 | GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .cache 2 | .DS_Store 3 | .idea 4 | *.log 5 | *.tgz 6 | coverage 7 | dist 8 | lib-cov 9 | logs 10 | node_modules 11 | temp 12 | .env 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Julien Ripouteau 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 | 5 | This module provides a transport for pino that forwards messages to a Loki instance. 6 | 7 | ## Why pino-loki 8 | Pino-loki is based upon the highly performant logging library pino. Loki usually gets the logs through Grafana Agent which reads system logs from files. This setup may not always be possible or require additional infrastructure, especially in situations where logs are gathered application code deployed as a SaaS in the cloud. Pino-loki sends the pino logs directly to Loki. 9 | 10 | Pino-loki is for Pino v7.0.0 and above, so the module can be configured to operate in a worker thread, which is the recommended way to use it. 11 | 12 | ## Usage 13 | 14 | ### In a worker thread 15 | 16 | ```ts 17 | import pino from 'pino' 18 | import type { LokiOptions } from 'pino-loki' 19 | 20 | const transport = pino.transport({ 21 | target: "pino-loki", 22 | options: { 23 | batching: true, 24 | interval: 5, 25 | 26 | host: 'https://my-loki-instance:3100', 27 | basicAuth: { 28 | username: "username", 29 | password: "password", 30 | }, 31 | }, 32 | }); 33 | 34 | const logger = pino(transport); 35 | logger.error({ foo: 'bar' }) 36 | ``` 37 | 38 | ### In main process 39 | 40 | See [the example](./examples/module_usage.ts) 41 | 42 | ### Library options 43 | 44 | #### `labels` 45 | 46 | Additional labels to be added to all Loki logs. This can be used to add additional context to all logs, such as the application name, environment, etc. 47 | 48 | #### `propsToLabels` 49 | 50 | A list of properties to be converted to loki labels. 51 | 52 | #### `levelMap` 53 | 54 | A map of pino log levels to Loki log levels. This can be used to map pino log levels to different Loki log levels. This is the default map. Left is pino log level, right is Loki log level. 55 | 56 | ```ts 57 | { 58 | 10: LokiLogLevel.Debug, 59 | 20: LokiLogLevel.Debug, 60 | 30: LokiLogLevel.Info, 61 | 40: LokiLogLevel.Warning, 62 | 50: LokiLogLevel.Error, 63 | 60: LokiLogLevel.Critical, 64 | }, 65 | ``` 66 | 67 | #### `host` 68 | 69 | The URL for Loki. This is required. 70 | 71 | #### `endpoint` 72 | 73 | The path to the Loki push API. Defaults to `/loki/api/v1/push`. 74 | 75 | #### `basicAuth` 76 | 77 | Basic auth credentials for Loki. An object with the following shape: 78 | 79 | ```ts 80 | { 81 | username: "username", 82 | password: "password", 83 | } 84 | ``` 85 | #### `headers` 86 | 87 | A list of headers to be sent to Loki. This can be useful for adding the `X-Scope-OrgID` header for Grafana Cloud Loki : 88 | 89 | ```ts 90 | { 91 | "X-Scope-OrgID": "your-id", 92 | }) 93 | ``` 94 | 95 | #### `timeout` 96 | 97 | A max timeout in miliseconds when sending logs to Loki. Defaults to `30_000`. 98 | 99 | #### `silenceErrors` 100 | 101 | If false, errors when sending logs to Loki will be displayed in the console. Defaults to `false`. 102 | 103 | #### `batching` 104 | 105 | Should logs be sent in batch mode. Defaults to `true`. 106 | 107 | #### `interval` 108 | 109 | The interval at which batched logs are sent in seconds. Defaults to `5`. 110 | 111 | #### `replaceTimestamp` 112 | 113 | Defaults to `false`. If true, the timestamp in the pino log will be replaced with `Date.now()`. Be careful when using this option with `batching` enabled, as the logs will be sent in batches, and the timestamp will be the time of the batch, not the time of the log. 114 | 115 | #### `structuredMetaKey` 116 | 117 | The key in the pino log object that contains [structured metadata](https://grafana.com/docs/loki/latest/get-started/labels/structured-metadata/). Defaults to `undefined` which means that structured metadata will not be sent to Loki. If set to `meta` for example, `{ recordId: 123, traceId: 456 }` will be sent if using the following log : 118 | 119 | ```ts 120 | logger.info({ meta: { recordId: 123, traceId: 456 } }, 'Hello') 121 | ``` 122 | 123 | #### `convertArrays` 124 | 125 | Defaults to `false`. As documented in the [Loki documentation](https://grafana.com/docs/loki/latest/query/log_queries/#json), Loki JSON parser will skip arrays. Setting this options to `true` will convert arrays to object with index as key. For example, `["foo", "bar"]` will be converted to `{ "0": "foo", "1": "bar" }`. 126 | 127 | 128 | #### `logFormat` 129 | 130 | Defaults to `false`. This option will let you convert the JSON pino log into a single string in a format that you set. 131 | The template can be either a string template ( not a string literal ! ) or a function that returns a string. 132 | You can use dot notation to access nested properties in the pino log object, such as `{req.method}` or `{req.url}`. 133 | 134 | ```typescript 135 | const transport = pino.transport({ 136 | target: 'pino-loki', 137 | options: { 138 | // String template 139 | logFormat: '{time} | {level} | {msg} {req.method} {req.url}', 140 | // Or a function ⚠️ Will not work out-of-the-box 141 | // with worker threads. Read the warning below ! 142 | logFormat: ({ time, level, msg, req }) => { 143 | return `${time} | ${level} | ${msg} ${req.method} ${req.url}`; 144 | }, 145 | }, 146 | }) 147 | ``` 148 | > [!NOTE] 149 | > Want to use the `logFormat` option with worker threads? Check the below section about [Handling non-serializable options](#handling-non-serializable-options). 150 | 151 | The log object has the following options: 152 | 153 | - `lokiLevel`: The pino log level parsed to Loki log level ( 'debug', 'info', 'warning' etc.. ) 154 | - `{key}`: Any other key in the pino log object, such as `pid`, `hostname`, `msg` etc. 155 | 156 | 157 | 158 | ### Handling non-serializable options 159 | 160 | Using the new pino v7+ transports not all options are serializable, for example if you want to use `logFormat` as a function you will need to wrap `pino-loki` in a custom module like this : 161 | 162 | ```ts 163 | // main.ts 164 | import pino from 'pino' 165 | 166 | const logger = pino({ 167 | transport: { 168 | target: './my-custom-pino-loki.js', 169 | options: { labels: { application: 'MY-APP' } } 170 | }, 171 | }) 172 | ``` 173 | 174 | ```ts 175 | // my-custom-pino-loki.js 176 | import { pinoLoki } from 'pino-loki' 177 | 178 | export default function customPinoLoki(options) { 179 | return pinoLoki({ 180 | ...options, 181 | logFormat: (log) => { 182 | return `hello ${log.msg} ${log.lokilevel} ${log.req.id} ${log.level}` 183 | }, 184 | }) 185 | } 186 | ``` 187 | 188 | This way you can use the `logFormat` option as a function, or any other non-serializable option. 189 | ## CLI usage 190 | ```shell 191 | npm install -g pino-loki 192 | node foo | pino-loki --hostname=http://hostname:3100 193 | ``` 194 | 195 | ``` 196 | $ pino-loki -h 197 | Options: 198 | -V, --version output the version number 199 | -u, --user Loki username 200 | -p, --password Loki password 201 | --hostname URL for Loki 202 | --endpoint Path to the Loki push API 203 | --headers Headers to be sent to Loki (Example: "X-Scope-OrgID=your-id,another-header=another-value") 204 | -b, --batch Should logs be sent in batch mode 205 | -i, --interval The interval at which batched logs are sent in seconds 206 | -t, --timeout Timeout for request to Loki 207 | -s, --silenceErrors If false, errors will be displayed in the console 208 | -r, --replaceTimestamp Replace pino logs timestamps with Date.now() 209 | -l, --labels